|
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512 |
- /*
- Tokenizer for MWParserFromHell
- Copyright (C) 2012-2013 Ben Kurtovic <ben.kurtovic@verizon.net>
-
- Permission is hereby granted, free of charge, to any person obtaining a copy of
- this software and associated documentation files (the "Software"), to deal in
- the Software without restriction, including without limitation the rights to
- use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
- of the Software, and to permit persons to whom the Software is furnished to do
- so, subject to the following conditions:
-
- The above copyright notice and this permission notice shall be included in all
- copies or substantial portions of the Software.
-
- THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
- AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- SOFTWARE.
- */
-
- #include "tokenizer.h"
-
- /*
- Given a context, return the heading level encoded within it.
- */
- static int heading_level_from_context(int n)
- {
- int level;
- n /= LC_HEADING_LEVEL_1;
- for (level = 1; n > 1; n >>= 1)
- level++;
- return level;
- }
-
- static PyObject*
- Tokenizer_new(PyTypeObject* type, PyObject* args, PyObject* kwds)
- {
- Tokenizer* self = (Tokenizer*) type->tp_alloc(type, 0);
- return (PyObject*) self;
- }
-
- static struct Textbuffer*
- Textbuffer_new(void)
- {
- struct Textbuffer* buffer = malloc(sizeof(struct Textbuffer));
- if (!buffer) {
- PyErr_NoMemory();
- return NULL;
- }
- buffer->size = 0;
- buffer->data = malloc(sizeof(Py_UNICODE) * TEXTBUFFER_BLOCKSIZE);
- if (!buffer->data) {
- free(buffer);
- PyErr_NoMemory();
- return NULL;
- }
- buffer->next = NULL;
- return buffer;
- }
-
- static void
- Tokenizer_dealloc(Tokenizer* self)
- {
- struct Stack *this = self->topstack, *next;
- Py_XDECREF(self->text);
-
- while (this) {
- Py_DECREF(this->stack);
- Textbuffer_dealloc(this->textbuffer);
- next = this->next;
- free(this);
- this = next;
- }
- self->ob_type->tp_free((PyObject*) self);
- }
-
- static void
- Textbuffer_dealloc(struct Textbuffer* this)
- {
- struct Textbuffer* next;
- while (this) {
- free(this->data);
- next = this->next;
- free(this);
- this = next;
- }
- }
-
- static int
- Tokenizer_init(Tokenizer* self, PyObject* args, PyObject* kwds)
- {
- static char* kwlist[] = {NULL};
- if (!PyArg_ParseTupleAndKeywords(args, kwds, "", kwlist))
- return -1;
- self->text = Py_None;
- Py_INCREF(Py_None);
- self->topstack = NULL;
- self->head = 0;
- self->length = 0;
- self->global = 0;
- return 0;
- }
-
- /*
- Add a new token stack, context, and textbuffer to the list.
- */
- static int
- Tokenizer_push(Tokenizer* self, int context)
- {
- struct Stack* top = malloc(sizeof(struct Stack));
- if (!top) {
- PyErr_NoMemory();
- return -1;
- }
- top->stack = PyList_New(0);
- top->context = context;
- top->textbuffer = Textbuffer_new();
- if (!top->textbuffer)
- return -1;
- top->next = self->topstack;
- self->topstack = top;
- self->depth++;
- self->cycles++;
- return 0;
- }
-
- /*
- Return the contents of the textbuffer as a Python Unicode object.
- */
- static PyObject*
- Textbuffer_render(struct Textbuffer* self)
- {
- PyObject *result = PyUnicode_FromUnicode(self->data, self->size);
- PyObject *left, *concat;
- while (self->next) {
- self = self->next;
- left = PyUnicode_FromUnicode(self->data, self->size);
- concat = PyUnicode_Concat(left, result);
- Py_DECREF(left);
- Py_DECREF(result);
- result = concat;
- }
- return result;
- }
-
- /*
- Push the textbuffer onto the stack as a Text node and clear it.
- */
- static int
- Tokenizer_push_textbuffer(Tokenizer* self)
- {
- PyObject *text, *kwargs, *token;
- struct Textbuffer* buffer = self->topstack->textbuffer;
- if (buffer->size == 0 && !buffer->next)
- return 0;
- text = Textbuffer_render(buffer);
- if (!text)
- return -1;
- kwargs = PyDict_New();
- if (!kwargs) {
- Py_DECREF(text);
- return -1;
- }
- PyDict_SetItemString(kwargs, "text", text);
- Py_DECREF(text);
- token = PyObject_Call(Text, NOARGS, kwargs);
- Py_DECREF(kwargs);
- if (!token)
- return -1;
- if (PyList_Append(self->topstack->stack, token)) {
- Py_DECREF(token);
- return -1;
- }
- Py_DECREF(token);
- Textbuffer_dealloc(buffer);
- self->topstack->textbuffer = Textbuffer_new();
- if (!self->topstack->textbuffer)
- return -1;
- return 0;
- }
-
- /*
- Pop and deallocate the top token stack/context/textbuffer.
- */
- static void
- Tokenizer_delete_top_of_stack(Tokenizer* self)
- {
- struct Stack* top = self->topstack;
- Py_DECREF(top->stack);
- Textbuffer_dealloc(top->textbuffer);
- self->topstack = top->next;
- free(top);
- self->depth--;
- }
-
- /*
- Pop the current stack/context/textbuffer, returing the stack.
- */
- static PyObject*
- Tokenizer_pop(Tokenizer* self)
- {
- PyObject* stack;
- if (Tokenizer_push_textbuffer(self))
- return NULL;
- stack = self->topstack->stack;
- Py_INCREF(stack);
- Tokenizer_delete_top_of_stack(self);
- return stack;
- }
-
- /*
- Pop the current stack/context/textbuffer, returing the stack. We will also
- replace the underlying stack's context with the current stack's.
- */
- static PyObject*
- Tokenizer_pop_keeping_context(Tokenizer* self)
- {
- PyObject* stack;
- int context;
- if (Tokenizer_push_textbuffer(self))
- return NULL;
- stack = self->topstack->stack;
- Py_INCREF(stack);
- context = self->topstack->context;
- Tokenizer_delete_top_of_stack(self);
- self->topstack->context = context;
- return stack;
- }
-
- /*
- Fail the current tokenization route. Discards the current
- stack/context/textbuffer and raises a BadRoute exception.
- */
- static void*
- Tokenizer_fail_route(Tokenizer* self)
- {
- PyObject* stack = Tokenizer_pop(self);
- Py_XDECREF(stack);
- FAIL_ROUTE();
- return NULL;
- }
-
- /*
- Write a token to the end of the current token stack.
- */
- static int
- Tokenizer_write(Tokenizer* self, PyObject* token)
- {
- if (Tokenizer_push_textbuffer(self))
- return -1;
- if (PyList_Append(self->topstack->stack, token))
- return -1;
- return 0;
- }
-
- /*
- Write a token to the beginning of the current token stack.
- */
- static int
- Tokenizer_write_first(Tokenizer* self, PyObject* token)
- {
- if (Tokenizer_push_textbuffer(self))
- return -1;
- if (PyList_Insert(self->topstack->stack, 0, token))
- return -1;
- return 0;
- }
-
- /*
- Write text to the current textbuffer.
- */
- static int
- Tokenizer_write_text(Tokenizer* self, Py_UNICODE text)
- {
- struct Textbuffer* buf = self->topstack->textbuffer;
- if (buf->size == TEXTBUFFER_BLOCKSIZE) {
- struct Textbuffer* new = Textbuffer_new();
- if (!new)
- return -1;
- new->next = buf;
- self->topstack->textbuffer = new;
- buf = new;
- }
- buf->data[buf->size] = text;
- buf->size++;
- return 0;
- }
-
- /*
- Write a series of tokens to the current stack at once.
- */
- static int
- Tokenizer_write_all(Tokenizer* self, PyObject* tokenlist)
- {
- int pushed = 0;
- PyObject *stack, *token, *left, *right, *text;
- struct Textbuffer* buffer;
- Py_ssize_t size;
-
- if (PyList_GET_SIZE(tokenlist) > 0) {
- token = PyList_GET_ITEM(tokenlist, 0);
- switch (PyObject_IsInstance(token, Text)) {
- case 0:
- break;
- case 1: {
- pushed = 1;
- buffer = self->topstack->textbuffer;
- if (buffer->size == 0 && !buffer->next)
- break;
- left = Textbuffer_render(buffer);
- if (!left)
- return -1;
- right = PyObject_GetAttrString(token, "text");
- if (!right)
- return -1;
- text = PyUnicode_Concat(left, right);
- Py_DECREF(left);
- Py_DECREF(right);
- if (!text)
- return -1;
- if (PyObject_SetAttrString(token, "text", text)) {
- Py_DECREF(text);
- return -1;
- }
- Py_DECREF(text);
- Textbuffer_dealloc(buffer);
- self->topstack->textbuffer = Textbuffer_new();
- if (!self->topstack->textbuffer)
- return -1;
- break;
- }
- case -1:
- return -1;
- }
- }
- if (!pushed) {
- if (Tokenizer_push_textbuffer(self))
- return -1;
- }
- stack = self->topstack->stack;
- size = PyList_GET_SIZE(stack);
- if (PyList_SetSlice(stack, size, size, tokenlist))
- return -1;
- return 0;
- }
-
- /*
- Pop the current stack, write text, and then write the stack. 'text' is a
- NULL-terminated array of chars.
- */
- static int
- Tokenizer_write_text_then_stack(Tokenizer* self, const char* text)
- {
- PyObject* stack = Tokenizer_pop(self);
- int i = 0;
- while (1) {
- if (!text[i])
- break;
- if (Tokenizer_write_text(self, (Py_UNICODE) text[i])) {
- Py_XDECREF(stack);
- return -1;
- }
- i++;
- }
- if (stack) {
- if (PyList_GET_SIZE(stack) > 0) {
- if (Tokenizer_write_all(self, stack)) {
- Py_DECREF(stack);
- return -1;
- }
- }
- Py_DECREF(stack);
- }
- self->head--;
- return 0;
- }
-
- /*
- Read the value at a relative point in the wikicode, forwards.
- */
- static PyObject*
- Tokenizer_read(Tokenizer* self, Py_ssize_t delta)
- {
- Py_ssize_t index = self->head + delta;
- if (index >= self->length)
- return EMPTY;
- return PyList_GET_ITEM(self->text, index);
- }
-
- /*
- Read the value at a relative point in the wikicode, backwards.
- */
- static PyObject*
- Tokenizer_read_backwards(Tokenizer* self, Py_ssize_t delta)
- {
- Py_ssize_t index;
- if (delta > self->head)
- return EMPTY;
- index = self->head - delta;
- return PyList_GET_ITEM(self->text, index);
- }
-
- /*
- Parse a template or argument at the head of the wikicode string.
- */
- static int
- Tokenizer_parse_template_or_argument(Tokenizer* self)
- {
- unsigned int braces = 2, i;
- PyObject *tokenlist;
-
- self->head += 2;
- while (Tokenizer_READ(self, 0) == *"{" && braces < MAX_BRACES) {
- self->head++;
- braces++;
- }
- if (Tokenizer_push(self, 0))
- return -1;
- while (braces) {
- if (braces == 1) {
- if (Tokenizer_write_text_then_stack(self, "{"))
- return -1;
- return 0;
- }
- if (braces == 2) {
- if (Tokenizer_parse_template(self))
- return -1;
-
- if (BAD_ROUTE) {
- RESET_ROUTE();
- if (Tokenizer_write_text_then_stack(self, "{{"))
- return -1;
- return 0;
- }
- break;
- }
- if (Tokenizer_parse_argument(self))
- return -1;
- if (BAD_ROUTE) {
- RESET_ROUTE();
- if (Tokenizer_parse_template(self))
- return -1;
- if (BAD_ROUTE) {
- char text[MAX_BRACES + 1];
- RESET_ROUTE();
- for (i = 0; i < braces; i++) text[i] = *"{";
- text[braces] = *"";
- if (Tokenizer_write_text_then_stack(self, text)) {
- Py_XDECREF(text);
- return -1;
- }
- Py_XDECREF(text);
- return 0;
- }
- else
- braces -= 2;
- }
- else
- braces -= 3;
- if (braces)
- self->head++;
- }
- tokenlist = Tokenizer_pop(self);
- if (!tokenlist)
- return -1;
- if (Tokenizer_write_all(self, tokenlist)) {
- Py_DECREF(tokenlist);
- return -1;
- }
- Py_DECREF(tokenlist);
- return 0;
- }
-
- /*
- Parse a template at the head of the wikicode string.
- */
- static int
- Tokenizer_parse_template(Tokenizer* self)
- {
- PyObject *template, *token;
- Py_ssize_t reset = self->head;
-
- template = Tokenizer_parse(self, LC_TEMPLATE_NAME);
- if (BAD_ROUTE) {
- self->head = reset;
- return 0;
- }
- if (!template)
- return -1;
- token = PyObject_CallObject(TemplateOpen, NULL);
- if (!token) {
- Py_DECREF(template);
- return -1;
- }
- if (Tokenizer_write_first(self, token)) {
- Py_DECREF(token);
- Py_DECREF(template);
- return -1;
- }
- Py_DECREF(token);
- if (Tokenizer_write_all(self, template)) {
- Py_DECREF(template);
- return -1;
- }
- Py_DECREF(template);
- token = PyObject_CallObject(TemplateClose, NULL);
- if (!token)
- return -1;
- if (Tokenizer_write(self, token)) {
- Py_DECREF(token);
- return -1;
- }
- Py_DECREF(token);
- return 0;
- }
-
- /*
- Parse an argument at the head of the wikicode string.
- */
- static int
- Tokenizer_parse_argument(Tokenizer* self)
- {
- PyObject *argument, *token;
- Py_ssize_t reset = self->head;
-
- argument = Tokenizer_parse(self, LC_ARGUMENT_NAME);
- if (BAD_ROUTE) {
- self->head = reset;
- return 0;
- }
- if (!argument)
- return -1;
- token = PyObject_CallObject(ArgumentOpen, NULL);
- if (!token) {
- Py_DECREF(argument);
- return -1;
- }
- if (Tokenizer_write_first(self, token)) {
- Py_DECREF(token);
- Py_DECREF(argument);
- return -1;
- }
- Py_DECREF(token);
- if (Tokenizer_write_all(self, argument)) {
- Py_DECREF(argument);
- return -1;
- }
- Py_DECREF(argument);
- token = PyObject_CallObject(ArgumentClose, NULL);
- if (!token)
- return -1;
- if (Tokenizer_write(self, token)) {
- Py_DECREF(token);
- return -1;
- }
- Py_DECREF(token);
- return 0;
- }
-
- /*
- Handle a template parameter at the head of the string.
- */
- static int
- Tokenizer_handle_template_param(Tokenizer* self)
- {
- PyObject *stack, *token;
-
- if (self->topstack->context & LC_TEMPLATE_NAME)
- self->topstack->context ^= LC_TEMPLATE_NAME;
- else if (self->topstack->context & LC_TEMPLATE_PARAM_VALUE)
- self->topstack->context ^= LC_TEMPLATE_PARAM_VALUE;
- if (self->topstack->context & LC_TEMPLATE_PARAM_KEY) {
- stack = Tokenizer_pop_keeping_context(self);
- if (!stack)
- return -1;
- if (Tokenizer_write_all(self, stack)) {
- Py_DECREF(stack);
- return -1;
- }
- Py_DECREF(stack);
- }
- else
- self->topstack->context |= LC_TEMPLATE_PARAM_KEY;
-
- token = PyObject_CallObject(TemplateParamSeparator, NULL);
- if (!token)
- return -1;
- if (Tokenizer_write(self, token)) {
- Py_DECREF(token);
- return -1;
- }
- Py_DECREF(token);
- if (Tokenizer_push(self, self->topstack->context))
- return -1;
- return 0;
- }
-
- /*
- Handle a template parameter's value at the head of the string.
- */
- static int
- Tokenizer_handle_template_param_value(Tokenizer* self)
- {
- PyObject *stack, *token;
-
- stack = Tokenizer_pop_keeping_context(self);
- if (!stack)
- return -1;
- if (Tokenizer_write_all(self, stack)) {
- Py_DECREF(stack);
- return -1;
- }
- Py_DECREF(stack);
- self->topstack->context ^= LC_TEMPLATE_PARAM_KEY;
- self->topstack->context |= LC_TEMPLATE_PARAM_VALUE;
- token = PyObject_CallObject(TemplateParamEquals, NULL);
- if (!token)
- return -1;
- if (Tokenizer_write(self, token)) {
- Py_DECREF(token);
- return -1;
- }
- Py_DECREF(token);
- return 0;
- }
-
- /*
- Handle the end of a template at the head of the string.
- */
- static PyObject*
- Tokenizer_handle_template_end(Tokenizer* self)
- {
- PyObject* stack;
-
- if (self->topstack->context & LC_TEMPLATE_PARAM_KEY) {
- stack = Tokenizer_pop_keeping_context(self);
- if (!stack)
- return NULL;
- if (Tokenizer_write_all(self, stack)) {
- Py_DECREF(stack);
- return NULL;
- }
- Py_DECREF(stack);
- }
- self->head++;
- stack = Tokenizer_pop(self);
- return stack;
- }
-
- /*
- Handle the separator between an argument's name and default.
- */
- static int
- Tokenizer_handle_argument_separator(Tokenizer* self)
- {
- PyObject* token;
- self->topstack->context ^= LC_ARGUMENT_NAME;
- self->topstack->context |= LC_ARGUMENT_DEFAULT;
- token = PyObject_CallObject(ArgumentSeparator, NULL);
- if (!token)
- return -1;
- if (Tokenizer_write(self, token)) {
- Py_DECREF(token);
- return -1;
- }
- Py_DECREF(token);
- return 0;
- }
-
- /*
- Handle the end of an argument at the head of the string.
- */
- static PyObject*
- Tokenizer_handle_argument_end(Tokenizer* self)
- {
- PyObject* stack = Tokenizer_pop(self);
- self->head += 2;
- return stack;
- }
-
- /*
- Parse an internal wikilink at the head of the wikicode string.
- */
- static int
- Tokenizer_parse_wikilink(Tokenizer* self)
- {
- Py_ssize_t reset;
- PyObject *wikilink, *token;
- int i;
-
- self->head += 2;
- reset = self->head - 1;
- wikilink = Tokenizer_parse(self, LC_WIKILINK_TITLE);
- if (BAD_ROUTE) {
- RESET_ROUTE();
- self->head = reset;
- for (i = 0; i < 2; i++) {
- if (Tokenizer_write_text(self, *"["))
- return -1;
- }
- return 0;
- }
- if (!wikilink)
- return -1;
- token = PyObject_CallObject(WikilinkOpen, NULL);
- if (!token) {
- Py_DECREF(wikilink);
- return -1;
- }
- if (Tokenizer_write(self, token)) {
- Py_DECREF(token);
- Py_DECREF(wikilink);
- return -1;
- }
- Py_DECREF(token);
- if (Tokenizer_write_all(self, wikilink)) {
- Py_DECREF(wikilink);
- return -1;
- }
- Py_DECREF(wikilink);
- token = PyObject_CallObject(WikilinkClose, NULL);
- if (!token)
- return -1;
- if (Tokenizer_write(self, token)) {
- Py_DECREF(token);
- return -1;
- }
- Py_DECREF(token);
- return 0;
- }
-
- /*
- Handle the separator between a wikilink's title and its text.
- */
- static int
- Tokenizer_handle_wikilink_separator(Tokenizer* self)
- {
- PyObject* token;
- self->topstack->context ^= LC_WIKILINK_TITLE;
- self->topstack->context |= LC_WIKILINK_TEXT;
- token = PyObject_CallObject(WikilinkSeparator, NULL);
- if (!token)
- return -1;
- if (Tokenizer_write(self, token)) {
- Py_DECREF(token);
- return -1;
- }
- Py_DECREF(token);
- return 0;
- }
-
- /*
- Handle the end of a wikilink at the head of the string.
- */
- static PyObject*
- Tokenizer_handle_wikilink_end(Tokenizer* self)
- {
- PyObject* stack = Tokenizer_pop(self);
- self->head += 1;
- return stack;
- }
-
- /*
- Parse a section heading at the head of the wikicode string.
- */
- static int
- Tokenizer_parse_heading(Tokenizer* self)
- {
- Py_ssize_t reset = self->head;
- int best = 1, i, context, diff;
- HeadingData *heading;
- PyObject *level, *kwargs, *token;
-
- self->global |= GL_HEADING;
- self->head += 1;
- while (Tokenizer_READ(self, 0) == *"=") {
- best++;
- self->head++;
- }
- context = LC_HEADING_LEVEL_1 << (best > 5 ? 5 : best - 1);
- heading = (HeadingData*) Tokenizer_parse(self, context);
- if (BAD_ROUTE) {
- RESET_ROUTE();
- self->head = reset + best - 1;
- for (i = 0; i < best; i++) {
- if (Tokenizer_write_text(self, *"="))
- return -1;
- }
- self->global ^= GL_HEADING;
- return 0;
- }
-
- level = PyInt_FromSsize_t(heading->level);
- if (!level) {
- Py_DECREF(heading->title);
- free(heading);
- return -1;
- }
- kwargs = PyDict_New();
- if (!kwargs) {
- Py_DECREF(level);
- Py_DECREF(heading->title);
- free(heading);
- return -1;
- }
- PyDict_SetItemString(kwargs, "level", level);
- Py_DECREF(level);
- token = PyObject_Call(HeadingStart, NOARGS, kwargs);
- Py_DECREF(kwargs);
- if (!token) {
- Py_DECREF(heading->title);
- free(heading);
- return -1;
- }
- if (Tokenizer_write(self, token)) {
- Py_DECREF(token);
- Py_DECREF(heading->title);
- free(heading);
- return -1;
- }
- Py_DECREF(token);
- if (heading->level < best) {
- diff = best - heading->level;
- for (i = 0; i < diff; i++) {
- if (Tokenizer_write_text(self, *"=")) {
- Py_DECREF(heading->title);
- free(heading);
- return -1;
- }
- }
- }
- if (Tokenizer_write_all(self, heading->title)) {
- Py_DECREF(heading->title);
- free(heading);
- return -1;
- }
- Py_DECREF(heading->title);
- free(heading);
- token = PyObject_CallObject(HeadingEnd, NULL);
- if (!token)
- return -1;
- if (Tokenizer_write(self, token)) {
- Py_DECREF(token);
- return -1;
- }
- Py_DECREF(token);
- self->global ^= GL_HEADING;
- return 0;
- }
-
- /*
- Handle the end of a section heading at the head of the string.
- */
- static HeadingData*
- Tokenizer_handle_heading_end(Tokenizer* self)
- {
- Py_ssize_t reset = self->head, best;
- int i, current, level, diff;
- HeadingData *after, *heading;
- PyObject *stack;
-
- self->head += 1;
- best = 1;
- while (Tokenizer_READ(self, 0) == *"=") {
- best++;
- self->head++;
- }
- current = heading_level_from_context(self->topstack->context);
- level = current > best ? (best > 6 ? 6 : best) :
- (current > 6 ? 6 : current);
- after = (HeadingData*) Tokenizer_parse(self, self->topstack->context);
- if (BAD_ROUTE) {
- RESET_ROUTE();
- if (level < best) {
- diff = best - level;
- for (i = 0; i < diff; i++) {
- if (Tokenizer_write_text(self, *"="))
- return NULL;
- }
- }
- self->head = reset + best - 1;
- }
- else {
- for (i = 0; i < best; i++) {
- if (Tokenizer_write_text(self, *"=")) {
- Py_DECREF(after->title);
- free(after);
- return NULL;
- }
- }
- if (Tokenizer_write_all(self, after->title)) {
- Py_DECREF(after->title);
- free(after);
- return NULL;
- }
- Py_DECREF(after->title);
- level = after->level;
- free(after);
- }
- stack = Tokenizer_pop(self);
- if (!stack)
- return NULL;
- heading = malloc(sizeof(HeadingData));
- if (!heading) {
- PyErr_NoMemory();
- return NULL;
- }
- heading->title = stack;
- heading->level = level;
- return heading;
- }
-
- /*
- Actually parse an HTML entity and ensure that it is valid.
- */
- static int
- Tokenizer_really_parse_entity(Tokenizer* self)
- {
- PyObject *token, *kwargs, *textobj;
- Py_UNICODE this;
- int numeric, hexadecimal, i, j, zeroes, test;
- char *valid, *text, *buffer, *def;
-
- #define FAIL_ROUTE_AND_EXIT() { \
- Tokenizer_fail_route(self); \
- free(text); \
- return 0; \
- }
-
- token = PyObject_CallObject(HTMLEntityStart, NULL);
- if (!token)
- return -1;
- if (Tokenizer_write(self, token)) {
- Py_DECREF(token);
- return -1;
- }
- Py_DECREF(token);
- self->head++;
- this = Tokenizer_READ(self, 0);
- if (this == *"") {
- Tokenizer_fail_route(self);
- return 0;
- }
- if (this == *"#") {
- numeric = 1;
- token = PyObject_CallObject(HTMLEntityNumeric, NULL);
- if (!token)
- return -1;
- if (Tokenizer_write(self, token)) {
- Py_DECREF(token);
- return -1;
- }
- Py_DECREF(token);
- self->head++;
- this = Tokenizer_READ(self, 0);
- if (this == *"") {
- Tokenizer_fail_route(self);
- return 0;
- }
- if (this == *"x" || this == *"X") {
- hexadecimal = 1;
- kwargs = PyDict_New();
- if (!kwargs)
- return -1;
- PyDict_SetItemString(kwargs, "char", Tokenizer_read(self, 0));
- token = PyObject_Call(HTMLEntityHex, NOARGS, kwargs);
- Py_DECREF(kwargs);
- if (!token)
- return -1;
- if (Tokenizer_write(self, token)) {
- Py_DECREF(token);
- return -1;
- }
- Py_DECREF(token);
- self->head++;
- }
- else
- hexadecimal = 0;
- }
- else
- numeric = hexadecimal = 0;
- if (hexadecimal)
- valid = HEXDIGITS;
- else if (numeric)
- valid = DIGITS;
- else
- valid = ALPHANUM;
- text = calloc(MAX_ENTITY_SIZE, sizeof(char));
- if (!text) {
- PyErr_NoMemory();
- return -1;
- }
- i = 0;
- zeroes = 0;
- while (1) {
- this = Tokenizer_READ(self, 0);
- if (this == *";") {
- if (i == 0)
- FAIL_ROUTE_AND_EXIT()
- break;
- }
- if (i == 0 && this == *"0") {
- zeroes++;
- self->head++;
- continue;
- }
- if (i >= 8)
- FAIL_ROUTE_AND_EXIT()
- for (j = 0; j < NUM_MARKERS; j++) {
- if (this == *MARKERS[j])
- FAIL_ROUTE_AND_EXIT()
- }
- j = 0;
- while (1) {
- if (!valid[j])
- FAIL_ROUTE_AND_EXIT()
- if (this == valid[j])
- break;
- j++;
- }
- text[i] = this;
- self->head++;
- i++;
- }
- if (numeric) {
- sscanf(text, (hexadecimal ? "%x" : "%d"), &test);
- if (test < 1 || test > 0x10FFFF)
- FAIL_ROUTE_AND_EXIT()
- }
- else {
- i = 0;
- while (1) {
- def = entitydefs[i];
- if (!def) // We've reached the end of the defs without finding it
- FAIL_ROUTE_AND_EXIT()
- if (strcmp(text, def) == 0)
- break;
- i++;
- }
- }
- if (zeroes) {
- buffer = calloc(strlen(text) + zeroes + 1, sizeof(char));
- if (!buffer) {
- free(text);
- PyErr_NoMemory();
- return -1;
- }
- for (i = 0; i < zeroes; i++)
- strcat(buffer, "0");
- strcat(buffer, text);
- free(text);
- text = buffer;
- }
- textobj = PyUnicode_FromString(text);
- if (!textobj) {
- free(text);
- return -1;
- }
- free(text);
- kwargs = PyDict_New();
- if (!kwargs) {
- Py_DECREF(textobj);
- return -1;
- }
- PyDict_SetItemString(kwargs, "text", textobj);
- Py_DECREF(textobj);
- token = PyObject_Call(Text, NOARGS, kwargs);
- Py_DECREF(kwargs);
- if (!token)
- return -1;
- if (Tokenizer_write(self, token)) {
- Py_DECREF(token);
- return -1;
- }
- Py_DECREF(token);
- token = PyObject_CallObject(HTMLEntityEnd, NULL);
- if (!token)
- return -1;
- if (Tokenizer_write(self, token)) {
- Py_DECREF(token);
- return -1;
- }
- Py_DECREF(token);
- return 0;
- }
-
- /*
- Parse an HTML entity at the head of the wikicode string.
- */
- static int
- Tokenizer_parse_entity(Tokenizer* self)
- {
- Py_ssize_t reset = self->head;
- PyObject *tokenlist;
-
- if (Tokenizer_push(self, 0))
- return -1;
- if (Tokenizer_really_parse_entity(self))
- return -1;
- if (BAD_ROUTE) {
- RESET_ROUTE();
- self->head = reset;
- if (Tokenizer_write_text(self, *"&"))
- return -1;
- return 0;
- }
- tokenlist = Tokenizer_pop(self);
- if (!tokenlist)
- return -1;
- if (Tokenizer_write_all(self, tokenlist)) {
- Py_DECREF(tokenlist);
- return -1;
- }
- Py_DECREF(tokenlist);
- return 0;
- }
-
- /*
- Parse an HTML comment at the head of the wikicode string.
- */
- static int
- Tokenizer_parse_comment(Tokenizer* self)
- {
- Py_ssize_t reset = self->head + 3;
- PyObject *token, *comment;
- int i;
-
- self->head += 4;
- comment = Tokenizer_parse(self, LC_COMMENT);
- if (BAD_ROUTE) {
- const char* text = "<!--";
- RESET_ROUTE();
- self->head = reset;
- i = 0;
- while (1) {
- if (!text[i])
- return 0;
- if (Tokenizer_write_text(self, (Py_UNICODE) text[i])) {
- Py_XDECREF(text);
- return -1;
- }
- i++;
- }
- return 0;
- }
- if (!comment)
- return -1;
- token = PyObject_CallObject(CommentStart, NULL);
- if (!token) {
- Py_DECREF(comment);
- return -1;
- }
- if (Tokenizer_write(self, token)) {
- Py_DECREF(token);
- Py_DECREF(comment);
- return -1;
- }
- Py_DECREF(token);
- if (Tokenizer_write_all(self, comment)) {
- Py_DECREF(comment);
- return -1;
- }
- Py_DECREF(comment);
- token = PyObject_CallObject(CommentEnd, NULL);
- if (!token)
- return -1;
- if (Tokenizer_write(self, token)) {
- Py_DECREF(token);
- return -1;
- }
- Py_DECREF(token);
- self->head += 2;
- return 0;
- }
-
- /*
- Make sure we are not trying to write an invalid character. Return 0 if
- everything is safe, or -1 if the route must be failed.
- */
- static int
- Tokenizer_verify_safe(Tokenizer* self, int context, Py_UNICODE data)
- {
- if (context & LC_FAIL_NEXT) {
- return -1;
- }
- if (context & LC_WIKILINK_TITLE) {
- if (data == *"]" || data == *"{")
- self->topstack->context |= LC_FAIL_NEXT;
- else if (data == *"\n" || data == *"[" || data == *"}")
- return -1;
- return 0;
- }
- if (context & LC_TEMPLATE_NAME) {
- if (data == *"{" || data == *"}" || data == *"[") {
- self->topstack->context |= LC_FAIL_NEXT;
- return 0;
- }
- if (data == *"]") {
- return -1;
- }
- if (data == *"|")
- return 0;
- if (context & LC_HAS_TEXT) {
- if (context & LC_FAIL_ON_TEXT) {
- if (!Py_UNICODE_ISSPACE(data))
- return -1;
- }
- else {
- if (data == *"\n")
- self->topstack->context |= LC_FAIL_ON_TEXT;
- }
- }
- else if (!Py_UNICODE_ISSPACE(data))
- self->topstack->context |= LC_HAS_TEXT;
- }
- else {
- if (context & LC_FAIL_ON_EQUALS) {
- if (data == *"=") {
- return -1;
- }
- }
- else if (context & LC_FAIL_ON_LBRACE) {
- if (data == *"{" || (Tokenizer_READ(self, -1) == *"{" &&
- Tokenizer_READ(self, -2) == *"{")) {
- if (context & LC_TEMPLATE)
- self->topstack->context |= LC_FAIL_ON_EQUALS;
- else
- self->topstack->context |= LC_FAIL_NEXT;
- return 0;
- }
- self->topstack->context ^= LC_FAIL_ON_LBRACE;
- }
- else if (context & LC_FAIL_ON_RBRACE) {
- if (data == *"}") {
- if (context & LC_TEMPLATE)
- self->topstack->context |= LC_FAIL_ON_EQUALS;
- else
- self->topstack->context |= LC_FAIL_NEXT;
- return 0;
- }
- self->topstack->context ^= LC_FAIL_ON_RBRACE;
- }
- else if (data == *"{")
- self->topstack->context |= LC_FAIL_ON_LBRACE;
- else if (data == *"}")
- self->topstack->context |= LC_FAIL_ON_RBRACE;
- }
- return 0;
- }
-
- /*
- Parse the wikicode string, using context for when to stop.
- */
- static PyObject*
- Tokenizer_parse(Tokenizer* self, int context)
- {
- static int fail_contexts = (LC_TEMPLATE | LC_ARGUMENT | LC_WIKILINK |
- LC_HEADING | LC_COMMENT);
- static int unsafe_contexts = (LC_TEMPLATE_NAME | LC_WIKILINK_TITLE |
- LC_TEMPLATE_PARAM_KEY | LC_ARGUMENT_NAME);
- int this_context, is_marker, i;
- Py_UNICODE this, next, next_next, last;
- PyObject *trash;
-
- if (Tokenizer_push(self, context))
- return NULL;
- while (1) {
- this = Tokenizer_READ(self, 0);
- this_context = self->topstack->context;
- if (this_context & unsafe_contexts) {
- if (Tokenizer_verify_safe(self, this_context, this) < 0) {
- if (this_context & LC_TEMPLATE_PARAM_KEY) {
- trash = Tokenizer_pop(self);
- Py_XDECREF(trash);
- }
- Tokenizer_fail_route(self);
- return NULL;
- }
- }
- is_marker = 0;
- for (i = 0; i < NUM_MARKERS; i++) {
- if (*MARKERS[i] == this) {
- is_marker = 1;
- break;
- }
- }
- if (!is_marker) {
- Tokenizer_write_text(self, this);
- self->head++;
- continue;
- }
- if (this == *"") {
- if (this_context & LC_TEMPLATE_PARAM_KEY) {
- trash = Tokenizer_pop(self);
- Py_XDECREF(trash);
- }
- if (this_context & fail_contexts)
- return Tokenizer_fail_route(self);
- return Tokenizer_pop(self);
- }
- next = Tokenizer_READ(self, 1);
- if (this_context & LC_COMMENT) {
- if (this == next && next == *"-") {
- if (Tokenizer_READ(self, 2) == *">")
- return Tokenizer_pop(self);
- }
- Tokenizer_write_text(self, this);
- }
- else if (this == next && next == *"{") {
- if (Tokenizer_CAN_RECURSE(self)) {
- if (Tokenizer_parse_template_or_argument(self))
- return NULL;
- if (self->topstack->context & LC_FAIL_NEXT)
- self->topstack->context ^= LC_FAIL_NEXT;
- }
- else
- Tokenizer_write_text(self, this);
- }
- else if (this == *"|" && this_context & LC_TEMPLATE) {
- if (Tokenizer_handle_template_param(self))
- return NULL;
- }
- else if (this == *"=" && this_context & LC_TEMPLATE_PARAM_KEY) {
- if (Tokenizer_handle_template_param_value(self))
- return NULL;
- }
- else if (this == next && next == *"}" && this_context & LC_TEMPLATE)
- return Tokenizer_handle_template_end(self);
- else if (this == *"|" && this_context & LC_ARGUMENT_NAME) {
- if (Tokenizer_handle_argument_separator(self))
- return NULL;
- }
- else if (this == next && next == *"}" && this_context & LC_ARGUMENT) {
- if (Tokenizer_READ(self, 2) == *"}") {
- return Tokenizer_handle_argument_end(self);
- }
- Tokenizer_write_text(self, this);
- }
- else if (this == next && next == *"[") {
- if (!(this_context & LC_WIKILINK_TITLE) &&
- Tokenizer_CAN_RECURSE(self)) {
- if (Tokenizer_parse_wikilink(self))
- return NULL;
- if (self->topstack->context & LC_FAIL_NEXT)
- self->topstack->context ^= LC_FAIL_NEXT;
- }
- else
- Tokenizer_write_text(self, this);
- }
- else if (this == *"|" && this_context & LC_WIKILINK_TITLE) {
- if (Tokenizer_handle_wikilink_separator(self))
- return NULL;
- }
- else if (this == next && next == *"]" && this_context & LC_WIKILINK)
- return Tokenizer_handle_wikilink_end(self);
- else if (this == *"=" && !(self->global & GL_HEADING)) {
- last = *PyUnicode_AS_UNICODE(Tokenizer_read_backwards(self, 1));
- if (last == *"\n" || last == *"") {
- if (Tokenizer_parse_heading(self))
- return NULL;
- }
- else
- Tokenizer_write_text(self, this);
- }
- else if (this == *"=" && this_context & LC_HEADING)
- return (PyObject*) Tokenizer_handle_heading_end(self);
- else if (this == *"\n" && this_context & LC_HEADING)
- return Tokenizer_fail_route(self);
- else if (this == *"&") {
- if (Tokenizer_parse_entity(self))
- return NULL;
- }
- else if (this == *"<" && next == *"!") {
- next_next = Tokenizer_READ(self, 2);
- if (next_next == Tokenizer_READ(self, 3) && next_next == *"-") {
- if (Tokenizer_parse_comment(self))
- return NULL;
- }
- else
- Tokenizer_write_text(self, this);
- }
- else
- Tokenizer_write_text(self, this);
- self->head++;
- }
- }
-
- /*
- Build a list of tokens from a string of wikicode and return it.
- */
- static PyObject*
- Tokenizer_tokenize(Tokenizer* self, PyObject* args)
- {
- PyObject *text, *temp;
-
- if (!PyArg_ParseTuple(args, "U", &text)) {
- const char* encoded;
- Py_ssize_t size;
- /* Failed to parse a Unicode object; try a string instead. */
- PyErr_Clear();
- if (!PyArg_ParseTuple(args, "s#", &encoded, &size))
- return NULL;
- temp = PyUnicode_FromStringAndSize(encoded, size);
- if (!text)
- return NULL;
- Py_XDECREF(self->text);
- text = PySequence_Fast(temp, "expected a sequence");
- Py_XDECREF(temp);
- self->text = text;
- }
- else {
- Py_XDECREF(self->text);
- self->text = PySequence_Fast(text, "expected a sequence");
- }
- self->length = PyList_GET_SIZE(self->text);
- return Tokenizer_parse(self, 0);
- }
-
- PyMODINIT_FUNC
- init_tokenizer(void)
- {
- PyObject *module, *tempmod, *defmap, *deflist, *globals, *locals,
- *fromlist, *modname;
- unsigned numdefs, i;
- char *name;
-
- TokenizerType.tp_new = PyType_GenericNew;
- if (PyType_Ready(&TokenizerType) < 0)
- return;
- module = Py_InitModule("_tokenizer", module_methods);
- Py_INCREF(&TokenizerType);
- PyModule_AddObject(module, "CTokenizer", (PyObject*) &TokenizerType);
- Py_INCREF(Py_True);
- PyDict_SetItemString(TokenizerType.tp_dict, "USES_C", Py_True);
-
- tempmod = PyImport_ImportModule("htmlentitydefs");
- if (!tempmod)
- return;
- defmap = PyObject_GetAttrString(tempmod, "entitydefs");
- if (!defmap)
- return;
- Py_DECREF(tempmod);
- deflist = PyDict_Keys(defmap);
- if (!deflist)
- return;
- Py_DECREF(defmap);
- numdefs = (unsigned) PyList_GET_SIZE(defmap);
- entitydefs = calloc(numdefs + 1, sizeof(char*));
- for (i = 0; i < numdefs; i++)
- entitydefs[i] = PyBytes_AsString(PyList_GET_ITEM(deflist, i));
- Py_DECREF(deflist);
-
- EMPTY = PyUnicode_FromString("");
- NOARGS = PyTuple_New(0);
-
- name = "mwparserfromhell.parser";
- globals = PyEval_GetGlobals();
- locals = PyEval_GetLocals();
- fromlist = PyList_New(1);
- if (!fromlist)
- return;
- modname = PyBytes_FromString("tokens");
- if (!modname)
- return;
- PyList_SET_ITEM(fromlist, 0, modname);
- tempmod = PyImport_ImportModuleLevel(name, globals, locals, fromlist, 0);
- Py_DECREF(fromlist);
- if (!tempmod)
- return;
- tokens = PyObject_GetAttrString(tempmod, "tokens");
- Py_DECREF(tempmod);
-
- Text = PyObject_GetAttrString(tokens, "Text");
-
- TemplateOpen = PyObject_GetAttrString(tokens, "TemplateOpen");
- TemplateParamSeparator = PyObject_GetAttrString(tokens,
- "TemplateParamSeparator");
- TemplateParamEquals = PyObject_GetAttrString(tokens,
- "TemplateParamEquals");
- TemplateClose = PyObject_GetAttrString(tokens, "TemplateClose");
-
- ArgumentOpen = PyObject_GetAttrString(tokens, "ArgumentOpen");
- ArgumentSeparator = PyObject_GetAttrString(tokens, "ArgumentSeparator");
- ArgumentClose = PyObject_GetAttrString(tokens, "ArgumentClose");
-
- WikilinkOpen = PyObject_GetAttrString(tokens, "WikilinkOpen");
- WikilinkSeparator = PyObject_GetAttrString(tokens, "WikilinkSeparator");
- WikilinkClose = PyObject_GetAttrString(tokens, "WikilinkClose");
-
- HTMLEntityStart = PyObject_GetAttrString(tokens, "HTMLEntityStart");
- HTMLEntityNumeric = PyObject_GetAttrString(tokens, "HTMLEntityNumeric");
- HTMLEntityHex = PyObject_GetAttrString(tokens, "HTMLEntityHex");
- HTMLEntityEnd = PyObject_GetAttrString(tokens, "HTMLEntityEnd");
-
- HeadingStart = PyObject_GetAttrString(tokens, "HeadingStart");
- HeadingEnd = PyObject_GetAttrString(tokens, "HeadingEnd");
-
- CommentStart = PyObject_GetAttrString(tokens, "CommentStart");
- CommentEnd = PyObject_GetAttrString(tokens, "CommentEnd");
-
- TagOpenOpen = PyObject_GetAttrString(tokens, "TagOpenOpen");
- TagAttrStart = PyObject_GetAttrString(tokens, "TagAttrStart");
- TagAttrEquals = PyObject_GetAttrString(tokens, "TagAttrEquals");
- TagAttrQuote = PyObject_GetAttrString(tokens, "TagAttrQuote");
- TagCloseOpen = PyObject_GetAttrString(tokens, "TagCloseOpen");
- TagCloseSelfclose = PyObject_GetAttrString(tokens, "TagCloseSelfclose");
- TagOpenClose = PyObject_GetAttrString(tokens, "TagOpenClose");
- TagCloseClose = PyObject_GetAttrString(tokens, "TagCloseClose");
- }
|