|
12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874187518761877187818791880188118821883188418851886188718881889189018911892189318941895189618971898189919001901190219031904190519061907190819091910191119121913191419151916191719181919192019211922192319241925192619271928192919301931193219331934193519361937193819391940194119421943194419451946194719481949195019511952195319541955195619571958195919601961196219631964196519661967196819691970197119721973197419751976197719781979198019811982198319841985198619871988198919901991199219931994199519961997199819992000200120022003200420052006200720082009201020112012201320142015201620172018201920202021202220232024202520262027202820292030203120322033203420352036203720382039204020412042204320442045204620472048204920502051205220532054205520562057205820592060206120622063206420652066206720682069207020712072207320742075207620772078207920802081208220832084208520862087208820892090209120922093209420952096209720982099210021012102210321042105210621072108210921102111211221132114211521162117211821192120212121222123212421252126212721282129213021312132213321342135213621372138213921402141214221432144214521462147214821492150215121522153215421552156215721582159216021612162216321642165216621672168216921702171217221732174217521762177217821792180218121822183218421852186218721882189219021912192219321942195219621972198219922002201220222032204220522062207220822092210221122122213221422152216221722182219222022212222222322242225222622272228222922302231223222332234223522362237223822392240224122422243224422452246224722482249225022512252225322542255225622572258225922602261226222632264226522662267226822692270227122722273227422752276227722782279228022812282228322842285228622872288228922902291229222932294229522962297229822992300230123022303230423052306230723082309231023112312231323142315231623172318231923202321232223232324232523262327232823292330233123322333233423352336233723382339234023412342234323442345234623472348234923502351235223532354235523562357235823592360236123622363236423652366236723682369237023712372237323742375237623772378237923802381238223832384238523862387238823892390239123922393239423952396239723982399240024012402240324042405240624072408240924102411241224132414241524162417241824192420242124222423242424252426242724282429243024312432243324342435243624372438243924402441244224432444244524462447244824492450245124522453245424552456245724582459246024612462246324642465246624672468246924702471247224732474247524762477247824792480248124822483248424852486248724882489249024912492249324942495249624972498249925002501250225032504250525062507250825092510251125122513251425152516251725182519252025212522252325242525252625272528252925302531253225332534253525362537253825392540254125422543254425452546254725482549255025512552255325542555255625572558255925602561256225632564256525662567256825692570257125722573257425752576257725782579258025812582258325842585258625872588258925902591259225932594259525962597259825992600260126022603260426052606260726082609261026112612261326142615261626172618261926202621262226232624262526262627262826292630263126322633263426352636263726382639264026412642264326442645264626472648264926502651265226532654265526562657265826592660266126622663266426652666266726682669267026712672267326742675267626772678267926802681268226832684268526862687268826892690269126922693269426952696269726982699270027012702270327042705270627072708270927102711271227132714271527162717271827192720272127222723272427252726272727282729273027312732273327342735273627372738273927402741274227432744274527462747274827492750275127522753275427552756275727582759276027612762276327642765276627672768276927702771277227732774277527762777277827792780278127822783278427852786278727882789279027912792279327942795279627972798279928002801280228032804280528062807280828092810281128122813281428152816281728182819282028212822282328242825282628272828282928302831283228332834283528362837283828392840284128422843284428452846284728482849285028512852285328542855 |
- /*
- Tokenizer for MWParserFromHell
- Copyright (C) 2012-2014 Ben Kurtovic <ben.kurtovic@gmail.com>
-
- Permission is hereby granted, free of charge, to any person obtaining a copy of
- this software and associated documentation files (the "Software"), to deal in
- the Software without restriction, including without limitation the rights to
- use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
- of the Software, and to permit persons to whom the Software is furnished to do
- so, subject to the following conditions:
-
- The above copyright notice and this permission notice shall be included in all
- copies or substantial portions of the Software.
-
- THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
- AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- SOFTWARE.
- */
-
- #include "tokenizer.h"
-
- /*
- Determine whether the given Py_UNICODE is a marker.
- */
- static int is_marker(Py_UNICODE this)
- {
- int i;
-
- for (i = 0; i < NUM_MARKERS; i++) {
- if (MARKERS[i] == this)
- return 1;
- }
- return 0;
- }
-
- /*
- Given a context, return the heading level encoded within it.
- */
- static int heading_level_from_context(int n)
- {
- int level;
-
- n /= LC_HEADING_LEVEL_1;
- for (level = 1; n > 1; n >>= 1)
- level++;
- return level;
- }
-
- /*
- Call the given function in definitions.py, using 'in1', 'in2', and 'in3' as
- parameters, and return its output as a bool.
- */
- static int call_def_func(const char* funcname, PyObject* in1, PyObject* in2,
- PyObject* in3)
- {
- PyObject* func = PyObject_GetAttrString(definitions, funcname);
- PyObject* result = PyObject_CallFunctionObjArgs(func, in1, in2, in3, NULL);
- int ans = (result == Py_True) ? 1 : 0;
-
- Py_DECREF(func);
- Py_DECREF(result);
- return ans;
- }
-
- /*
- Sanitize the name of a tag so it can be compared with others for equality.
- */
- static PyObject* strip_tag_name(PyObject* token)
- {
- PyObject *text, *rstripped, *lowered;
-
- text = PyObject_GetAttrString(token, "text");
- if (!text)
- return NULL;
- rstripped = PyObject_CallMethod(text, "rstrip", NULL);
- Py_DECREF(text);
- if (!rstripped)
- return NULL;
- lowered = PyObject_CallMethod(rstripped, "lower", NULL);
- Py_DECREF(rstripped);
- return lowered;
- }
-
- static Textbuffer* Textbuffer_new(void)
- {
- Textbuffer* buffer = malloc(sizeof(Textbuffer));
-
- if (!buffer) {
- PyErr_NoMemory();
- return NULL;
- }
- buffer->size = 0;
- buffer->data = malloc(sizeof(Py_UNICODE) * TEXTBUFFER_BLOCKSIZE);
- if (!buffer->data) {
- free(buffer);
- PyErr_NoMemory();
- return NULL;
- }
- buffer->prev = buffer->next = NULL;
- return buffer;
- }
-
- static void Textbuffer_dealloc(Textbuffer* self)
- {
- Textbuffer* next;
-
- while (self) {
- free(self->data);
- next = self->next;
- free(self);
- self = next;
- }
- }
-
- /*
- Write a Unicode codepoint to the given textbuffer.
- */
- static int Textbuffer_write(Textbuffer** this, Py_UNICODE code)
- {
- Textbuffer* self = *this;
-
- if (self->size == TEXTBUFFER_BLOCKSIZE) {
- Textbuffer* new = Textbuffer_new();
- if (!new)
- return -1;
- new->next = self;
- self->prev = new;
- *this = self = new;
- }
- self->data[self->size++] = code;
- return 0;
- }
-
- /*
- Return the contents of the textbuffer as a Python Unicode object.
- */
- static PyObject* Textbuffer_render(Textbuffer* self)
- {
- PyObject *result = PyUnicode_FromUnicode(self->data, self->size);
- PyObject *left, *concat;
-
- while (self->next) {
- self = self->next;
- left = PyUnicode_FromUnicode(self->data, self->size);
- concat = PyUnicode_Concat(left, result);
- Py_DECREF(left);
- Py_DECREF(result);
- result = concat;
- }
- return result;
- }
-
- static TagData* TagData_new(void)
- {
- TagData *self = malloc(sizeof(TagData));
-
- #define ALLOC_BUFFER(name) \
- name = Textbuffer_new(); \
- if (!name) { \
- TagData_dealloc(self); \
- return NULL; \
- }
-
- if (!self) {
- PyErr_NoMemory();
- return NULL;
- }
- self->context = TAG_NAME;
- ALLOC_BUFFER(self->pad_first)
- ALLOC_BUFFER(self->pad_before_eq)
- ALLOC_BUFFER(self->pad_after_eq)
- self->reset = 0;
- return self;
- }
-
- static void TagData_dealloc(TagData* self)
- {
- #define DEALLOC_BUFFER(name) \
- if (name) \
- Textbuffer_dealloc(name);
-
- DEALLOC_BUFFER(self->pad_first);
- DEALLOC_BUFFER(self->pad_before_eq);
- DEALLOC_BUFFER(self->pad_after_eq);
- free(self);
- }
-
- static int TagData_reset_buffers(TagData* self)
- {
- #define RESET_BUFFER(name) \
- Textbuffer_dealloc(name); \
- name = Textbuffer_new(); \
- if (!name) \
- return -1;
-
- RESET_BUFFER(self->pad_first)
- RESET_BUFFER(self->pad_before_eq)
- RESET_BUFFER(self->pad_after_eq)
- return 0;
- }
-
- static PyObject*
- Tokenizer_new(PyTypeObject* type, PyObject* args, PyObject* kwds)
- {
- Tokenizer* self = (Tokenizer*) type->tp_alloc(type, 0);
- return (PyObject*) self;
- }
-
- static void Tokenizer_dealloc(Tokenizer* self)
- {
- Stack *this = self->topstack, *next;
- Py_XDECREF(self->text);
-
- while (this) {
- Py_DECREF(this->stack);
- Textbuffer_dealloc(this->textbuffer);
- next = this->next;
- free(this);
- this = next;
- }
- Py_TYPE(self)->tp_free((PyObject*) self);
- }
-
- static int Tokenizer_init(Tokenizer* self, PyObject* args, PyObject* kwds)
- {
- static char* kwlist[] = {NULL};
-
- if (!PyArg_ParseTupleAndKeywords(args, kwds, "", kwlist))
- return -1;
- self->text = Py_None;
- Py_INCREF(Py_None);
- self->topstack = NULL;
- self->head = self->length = self->global = self->depth = self->cycles = 0;
- return 0;
- }
-
- /*
- Add a new token stack, context, and textbuffer to the list.
- */
- static int Tokenizer_push(Tokenizer* self, int context)
- {
- Stack* top = malloc(sizeof(Stack));
-
- if (!top) {
- PyErr_NoMemory();
- return -1;
- }
- top->stack = PyList_New(0);
- top->context = context;
- top->textbuffer = Textbuffer_new();
- if (!top->textbuffer)
- return -1;
- top->next = self->topstack;
- self->topstack = top;
- self->depth++;
- self->cycles++;
- return 0;
- }
-
- /*
- Push the textbuffer onto the stack as a Text node and clear it.
- */
- static int Tokenizer_push_textbuffer(Tokenizer* self)
- {
- PyObject *text, *kwargs, *token;
- Textbuffer* buffer = self->topstack->textbuffer;
-
- if (buffer->size == 0 && !buffer->next)
- return 0;
- text = Textbuffer_render(buffer);
- if (!text)
- return -1;
- kwargs = PyDict_New();
- if (!kwargs) {
- Py_DECREF(text);
- return -1;
- }
- PyDict_SetItemString(kwargs, "text", text);
- Py_DECREF(text);
- token = PyObject_Call(Text, NOARGS, kwargs);
- Py_DECREF(kwargs);
- if (!token)
- return -1;
- if (PyList_Append(self->topstack->stack, token)) {
- Py_DECREF(token);
- return -1;
- }
- Py_DECREF(token);
- Textbuffer_dealloc(buffer);
- self->topstack->textbuffer = Textbuffer_new();
- if (!self->topstack->textbuffer)
- return -1;
- return 0;
- }
-
- /*
- Pop and deallocate the top token stack/context/textbuffer.
- */
- static void Tokenizer_delete_top_of_stack(Tokenizer* self)
- {
- Stack* top = self->topstack;
-
- Py_DECREF(top->stack);
- Textbuffer_dealloc(top->textbuffer);
- self->topstack = top->next;
- free(top);
- self->depth--;
- }
-
- /*
- Pop the current stack/context/textbuffer, returing the stack.
- */
- static PyObject* Tokenizer_pop(Tokenizer* self)
- {
- PyObject* stack;
-
- if (Tokenizer_push_textbuffer(self))
- return NULL;
- stack = self->topstack->stack;
- Py_INCREF(stack);
- Tokenizer_delete_top_of_stack(self);
- return stack;
- }
-
- /*
- Pop the current stack/context/textbuffer, returing the stack. We will also
- replace the underlying stack's context with the current stack's.
- */
- static PyObject* Tokenizer_pop_keeping_context(Tokenizer* self)
- {
- PyObject* stack;
- int context;
-
- if (Tokenizer_push_textbuffer(self))
- return NULL;
- stack = self->topstack->stack;
- Py_INCREF(stack);
- context = self->topstack->context;
- Tokenizer_delete_top_of_stack(self);
- self->topstack->context = context;
- return stack;
- }
-
- /*
- Fail the current tokenization route. Discards the current
- stack/context/textbuffer and raises a BadRoute exception.
- */
- static void* Tokenizer_fail_route(Tokenizer* self)
- {
- int context = self->topstack->context;
- PyObject* stack = Tokenizer_pop(self);
-
- Py_XDECREF(stack);
- FAIL_ROUTE(context);
- return NULL;
- }
-
- /*
- Write a token to the current token stack.
- */
- static int Tokenizer_emit_token(Tokenizer* self, PyObject* token, int first)
- {
- PyObject* instance;
-
- if (Tokenizer_push_textbuffer(self))
- return -1;
- instance = PyObject_CallObject(token, NULL);
- if (!instance)
- return -1;
- if (first ? PyList_Insert(self->topstack->stack, 0, instance) :
- PyList_Append(self->topstack->stack, instance)) {
- Py_DECREF(instance);
- return -1;
- }
- Py_DECREF(instance);
- return 0;
- }
-
- /*
- Write a token to the current token stack, with kwargs. Steals a reference
- to kwargs.
- */
- static int Tokenizer_emit_token_kwargs(Tokenizer* self, PyObject* token,
- PyObject* kwargs, int first)
- {
- PyObject* instance;
-
- if (Tokenizer_push_textbuffer(self)) {
- Py_DECREF(kwargs);
- return -1;
- }
- instance = PyObject_Call(token, NOARGS, kwargs);
- if (!instance) {
- Py_DECREF(kwargs);
- return -1;
- }
- if (first ? PyList_Insert(self->topstack->stack, 0, instance):
- PyList_Append(self->topstack->stack, instance)) {
- Py_DECREF(instance);
- Py_DECREF(kwargs);
- return -1;
- }
- Py_DECREF(instance);
- Py_DECREF(kwargs);
- return 0;
- }
-
- /*
- Write a Unicode codepoint to the current textbuffer.
- */
- static int Tokenizer_emit_char(Tokenizer* self, Py_UNICODE code)
- {
- return Textbuffer_write(&(self->topstack->textbuffer), code);
- }
-
- /*
- Write a string of text to the current textbuffer.
- */
- static int Tokenizer_emit_text(Tokenizer* self, const char* text)
- {
- int i = 0;
-
- while (text[i]) {
- if (Tokenizer_emit_char(self, text[i]))
- return -1;
- i++;
- }
- return 0;
- }
-
- /*
- Write the contents of another textbuffer to the current textbuffer,
- deallocating it in the process.
- */
- static int
- Tokenizer_emit_textbuffer(Tokenizer* self, Textbuffer* buffer, int reverse)
- {
- Textbuffer *original = buffer;
- long i;
-
- if (reverse) {
- do {
- for (i = buffer->size - 1; i >= 0; i--) {
- if (Tokenizer_emit_char(self, buffer->data[i])) {
- Textbuffer_dealloc(original);
- return -1;
- }
- }
- } while ((buffer = buffer->next));
- }
- else {
- while (buffer->next)
- buffer = buffer->next;
- do {
- for (i = 0; i < buffer->size; i++) {
- if (Tokenizer_emit_char(self, buffer->data[i])) {
- Textbuffer_dealloc(original);
- return -1;
- }
- }
- } while ((buffer = buffer->prev));
- }
- Textbuffer_dealloc(original);
- return 0;
- }
-
- /*
- Write a series of tokens to the current stack at once.
- */
- static int Tokenizer_emit_all(Tokenizer* self, PyObject* tokenlist)
- {
- int pushed = 0;
- PyObject *stack, *token, *left, *right, *text;
- Textbuffer* buffer;
- Py_ssize_t size;
-
- if (PyList_GET_SIZE(tokenlist) > 0) {
- token = PyList_GET_ITEM(tokenlist, 0);
- switch (PyObject_IsInstance(token, Text)) {
- case 0:
- break;
- case 1: {
- pushed = 1;
- buffer = self->topstack->textbuffer;
- if (buffer->size == 0 && !buffer->next)
- break;
- left = Textbuffer_render(buffer);
- if (!left)
- return -1;
- right = PyObject_GetAttrString(token, "text");
- if (!right)
- return -1;
- text = PyUnicode_Concat(left, right);
- Py_DECREF(left);
- Py_DECREF(right);
- if (!text)
- return -1;
- if (PyObject_SetAttrString(token, "text", text)) {
- Py_DECREF(text);
- return -1;
- }
- Py_DECREF(text);
- Textbuffer_dealloc(buffer);
- self->topstack->textbuffer = Textbuffer_new();
- if (!self->topstack->textbuffer)
- return -1;
- break;
- }
- case -1:
- return -1;
- }
- }
- if (!pushed) {
- if (Tokenizer_push_textbuffer(self))
- return -1;
- }
- stack = self->topstack->stack;
- size = PyList_GET_SIZE(stack);
- if (PyList_SetSlice(stack, size, size, tokenlist))
- return -1;
- return 0;
- }
-
- /*
- Pop the current stack, write text, and then write the stack. 'text' is a
- NULL-terminated array of chars.
- */
- static int Tokenizer_emit_text_then_stack(Tokenizer* self, const char* text)
- {
- PyObject* stack = Tokenizer_pop(self);
-
- if (Tokenizer_emit_text(self, text)) {
- Py_DECREF(stack);
- return -1;
- }
- if (stack) {
- if (PyList_GET_SIZE(stack) > 0) {
- if (Tokenizer_emit_all(self, stack)) {
- Py_DECREF(stack);
- return -1;
- }
- }
- Py_DECREF(stack);
- }
- self->head--;
- return 0;
- }
-
- /*
- Read the value at a relative point in the wikicode, forwards.
- */
- static PyObject* Tokenizer_read(Tokenizer* self, Py_ssize_t delta)
- {
- Py_ssize_t index = self->head + delta;
-
- if (index >= self->length)
- return EMPTY;
- return PyList_GET_ITEM(self->text, index);
- }
-
- /*
- Read the value at a relative point in the wikicode, backwards.
- */
- static PyObject* Tokenizer_read_backwards(Tokenizer* self, Py_ssize_t delta)
- {
- Py_ssize_t index;
-
- if (delta > self->head)
- return EMPTY;
- index = self->head - delta;
- return PyList_GET_ITEM(self->text, index);
- }
-
- /*
- Parse a template at the head of the wikicode string.
- */
- static int Tokenizer_parse_template(Tokenizer* self)
- {
- PyObject *template;
- Py_ssize_t reset = self->head;
-
- template = Tokenizer_parse(self, LC_TEMPLATE_NAME, 1);
- if (BAD_ROUTE) {
- self->head = reset;
- return 0;
- }
- if (!template)
- return -1;
- if (Tokenizer_emit_first(self, TemplateOpen)) {
- Py_DECREF(template);
- return -1;
- }
- if (Tokenizer_emit_all(self, template)) {
- Py_DECREF(template);
- return -1;
- }
- Py_DECREF(template);
- if (Tokenizer_emit(self, TemplateClose))
- return -1;
- return 0;
- }
-
- /*
- Parse an argument at the head of the wikicode string.
- */
- static int Tokenizer_parse_argument(Tokenizer* self)
- {
- PyObject *argument;
- Py_ssize_t reset = self->head;
-
- argument = Tokenizer_parse(self, LC_ARGUMENT_NAME, 1);
- if (BAD_ROUTE) {
- self->head = reset;
- return 0;
- }
- if (!argument)
- return -1;
- if (Tokenizer_emit_first(self, ArgumentOpen)) {
- Py_DECREF(argument);
- return -1;
- }
- if (Tokenizer_emit_all(self, argument)) {
- Py_DECREF(argument);
- return -1;
- }
- Py_DECREF(argument);
- if (Tokenizer_emit(self, ArgumentClose))
- return -1;
- return 0;
- }
-
- /*
- Parse a template or argument at the head of the wikicode string.
- */
- static int Tokenizer_parse_template_or_argument(Tokenizer* self)
- {
- unsigned int braces = 2, i;
- PyObject *tokenlist;
-
- self->head += 2;
- while (Tokenizer_READ(self, 0) == '{' && braces < MAX_BRACES) {
- self->head++;
- braces++;
- }
- if (Tokenizer_push(self, 0))
- return -1;
- while (braces) {
- if (braces == 1) {
- if (Tokenizer_emit_text_then_stack(self, "{"))
- return -1;
- return 0;
- }
- if (braces == 2) {
- if (Tokenizer_parse_template(self))
- return -1;
- if (BAD_ROUTE) {
- RESET_ROUTE();
- if (Tokenizer_emit_text_then_stack(self, "{{"))
- return -1;
- return 0;
- }
- break;
- }
- if (Tokenizer_parse_argument(self))
- return -1;
- if (BAD_ROUTE) {
- RESET_ROUTE();
- if (Tokenizer_parse_template(self))
- return -1;
- if (BAD_ROUTE) {
- char text[MAX_BRACES + 1];
- RESET_ROUTE();
- for (i = 0; i < braces; i++) text[i] = '{';
- text[braces] = '\0';
- if (Tokenizer_emit_text_then_stack(self, text)) {
- Py_XDECREF(text);
- return -1;
- }
- Py_XDECREF(text);
- return 0;
- }
- else
- braces -= 2;
- }
- else
- braces -= 3;
- if (braces)
- self->head++;
- }
- tokenlist = Tokenizer_pop(self);
- if (!tokenlist)
- return -1;
- if (Tokenizer_emit_all(self, tokenlist)) {
- Py_DECREF(tokenlist);
- return -1;
- }
- Py_DECREF(tokenlist);
- if (self->topstack->context & LC_FAIL_NEXT)
- self->topstack->context ^= LC_FAIL_NEXT;
- return 0;
- }
-
- /*
- Handle a template parameter at the head of the string.
- */
- static int Tokenizer_handle_template_param(Tokenizer* self)
- {
- PyObject *stack;
-
- if (self->topstack->context & LC_TEMPLATE_NAME)
- self->topstack->context ^= LC_TEMPLATE_NAME;
- else if (self->topstack->context & LC_TEMPLATE_PARAM_VALUE)
- self->topstack->context ^= LC_TEMPLATE_PARAM_VALUE;
- if (self->topstack->context & LC_TEMPLATE_PARAM_KEY) {
- stack = Tokenizer_pop_keeping_context(self);
- if (!stack)
- return -1;
- if (Tokenizer_emit_all(self, stack)) {
- Py_DECREF(stack);
- return -1;
- }
- Py_DECREF(stack);
- }
- else
- self->topstack->context |= LC_TEMPLATE_PARAM_KEY;
- if (Tokenizer_emit(self, TemplateParamSeparator))
- return -1;
- if (Tokenizer_push(self, self->topstack->context))
- return -1;
- return 0;
- }
-
- /*
- Handle a template parameter's value at the head of the string.
- */
- static int Tokenizer_handle_template_param_value(Tokenizer* self)
- {
- PyObject *stack;
-
- stack = Tokenizer_pop_keeping_context(self);
- if (!stack)
- return -1;
- if (Tokenizer_emit_all(self, stack)) {
- Py_DECREF(stack);
- return -1;
- }
- Py_DECREF(stack);
- self->topstack->context ^= LC_TEMPLATE_PARAM_KEY;
- self->topstack->context |= LC_TEMPLATE_PARAM_VALUE;
- if (Tokenizer_emit(self, TemplateParamEquals))
- return -1;
- return 0;
- }
-
- /*
- Handle the end of a template at the head of the string.
- */
- static PyObject* Tokenizer_handle_template_end(Tokenizer* self)
- {
- PyObject* stack;
-
- if (self->topstack->context & LC_TEMPLATE_PARAM_KEY) {
- stack = Tokenizer_pop_keeping_context(self);
- if (!stack)
- return NULL;
- if (Tokenizer_emit_all(self, stack)) {
- Py_DECREF(stack);
- return NULL;
- }
- Py_DECREF(stack);
- }
- self->head++;
- stack = Tokenizer_pop(self);
- return stack;
- }
-
- /*
- Handle the separator between an argument's name and default.
- */
- static int Tokenizer_handle_argument_separator(Tokenizer* self)
- {
- self->topstack->context ^= LC_ARGUMENT_NAME;
- self->topstack->context |= LC_ARGUMENT_DEFAULT;
- if (Tokenizer_emit(self, ArgumentSeparator))
- return -1;
- return 0;
- }
-
- /*
- Handle the end of an argument at the head of the string.
- */
- static PyObject* Tokenizer_handle_argument_end(Tokenizer* self)
- {
- PyObject* stack = Tokenizer_pop(self);
-
- self->head += 2;
- return stack;
- }
-
- /*
- Parse an internal wikilink at the head of the wikicode string.
- */
- static int Tokenizer_parse_wikilink(Tokenizer* self)
- {
- Py_ssize_t reset;
- PyObject *wikilink;
-
- self->head += 2;
- reset = self->head - 1;
- wikilink = Tokenizer_parse(self, LC_WIKILINK_TITLE, 1);
- if (BAD_ROUTE) {
- RESET_ROUTE();
- self->head = reset;
- if (Tokenizer_emit_text(self, "[["))
- return -1;
- return 0;
- }
- if (!wikilink)
- return -1;
- if (Tokenizer_emit(self, WikilinkOpen)) {
- Py_DECREF(wikilink);
- return -1;
- }
- if (Tokenizer_emit_all(self, wikilink)) {
- Py_DECREF(wikilink);
- return -1;
- }
- Py_DECREF(wikilink);
- if (Tokenizer_emit(self, WikilinkClose))
- return -1;
- if (self->topstack->context & LC_FAIL_NEXT)
- self->topstack->context ^= LC_FAIL_NEXT;
- return 0;
- }
-
- /*
- Handle the separator between a wikilink's title and its text.
- */
- static int Tokenizer_handle_wikilink_separator(Tokenizer* self)
- {
- self->topstack->context ^= LC_WIKILINK_TITLE;
- self->topstack->context |= LC_WIKILINK_TEXT;
- if (Tokenizer_emit(self, WikilinkSeparator))
- return -1;
- return 0;
- }
-
- /*
- Handle the end of a wikilink at the head of the string.
- */
- static PyObject* Tokenizer_handle_wikilink_end(Tokenizer* self)
- {
- PyObject* stack = Tokenizer_pop(self);
- self->head += 1;
- return stack;
- }
-
- /*
- Parse the URI scheme of a bracket-enclosed external link.
- */
- static int Tokenizer_parse_bracketed_uri_scheme(Tokenizer* self)
- {
- static const char* valid = "abcdefghijklmnopqrstuvwxyz0123456789+.-";
- Textbuffer* buffer;
- PyObject* scheme;
- Py_UNICODE this;
- int slashes, i;
-
- if (Tokenizer_push(self, LC_EXT_LINK_URI))
- return -1;
- if (Tokenizer_READ(self, 0) == '/' && Tokenizer_READ(self, 1) == '/') {
- if (Tokenizer_emit_text(self, "//"))
- return -1;
- self->head += 2;
- }
- else {
- buffer = Textbuffer_new();
- if (!buffer)
- return -1;
- while ((this = Tokenizer_READ(self, 0))) {
- i = 0;
- while (1) {
- if (!valid[i])
- goto end_of_loop;
- if (this == valid[i])
- break;
- i++;
- }
- Textbuffer_write(&buffer, this);
- if (Tokenizer_emit_char(self, this)) {
- Textbuffer_dealloc(buffer);
- return -1;
- }
- self->head++;
- }
- end_of_loop:
- if (this != ':') {
- Textbuffer_dealloc(buffer);
- Tokenizer_fail_route(self);
- return 0;
- }
- if (Tokenizer_emit_char(self, ':')) {
- Textbuffer_dealloc(buffer);
- return -1;
- }
- self->head++;
- slashes = (Tokenizer_READ(self, 0) == '/' &&
- Tokenizer_READ(self, 1) == '/');
- if (slashes) {
- if (Tokenizer_emit_text(self, "//")) {
- Textbuffer_dealloc(buffer);
- return -1;
- }
- self->head += 2;
- }
- scheme = Textbuffer_render(buffer);
- Textbuffer_dealloc(buffer);
- if (!scheme)
- return -1;
- if (!IS_SCHEME(scheme, slashes, 0)) {
- Py_DECREF(scheme);
- Tokenizer_fail_route(self);
- return 0;
- }
- Py_DECREF(scheme);
- }
- return 0;
- }
-
- /*
- Parse the URI scheme of a free (no brackets) external link.
- */
- static int Tokenizer_parse_free_uri_scheme(Tokenizer* self)
- {
- static const char* valid = "abcdefghijklmnopqrstuvwxyz0123456789+.-";
- Textbuffer *scheme_buffer = Textbuffer_new(), *temp_buffer;
- PyObject *scheme;
- Py_UNICODE chunk;
- long i;
- int slashes, j;
-
- if (!scheme_buffer)
- return -1;
- // We have to backtrack through the textbuffer looking for our scheme since
- // it was just parsed as text:
- temp_buffer = self->topstack->textbuffer;
- while (temp_buffer) {
- for (i = temp_buffer->size - 1; i >= 0; i--) {
- chunk = temp_buffer->data[i];
- if (Py_UNICODE_ISSPACE(chunk) || is_marker(chunk))
- goto end_of_loop;
- j = 0;
- while (1) {
- if (!valid[j]) {
- Textbuffer_dealloc(scheme_buffer);
- FAIL_ROUTE(0);
- return 0;
- }
- if (chunk == valid[j])
- break;
- j++;
- }
- Textbuffer_write(&scheme_buffer, chunk);
- }
- temp_buffer = temp_buffer->next;
- }
- end_of_loop:
- scheme = Textbuffer_render(scheme_buffer);
- if (!scheme) {
- Textbuffer_dealloc(scheme_buffer);
- return -1;
- }
- slashes = (Tokenizer_READ(self, 0) == '/' &&
- Tokenizer_READ(self, 1) == '/');
- if (!IS_SCHEME(scheme, slashes, 1)) {
- Py_DECREF(scheme);
- Textbuffer_dealloc(scheme_buffer);
- FAIL_ROUTE(0);
- return 0;
- }
- Py_DECREF(scheme);
- if (Tokenizer_push(self, self->topstack->context | LC_EXT_LINK_URI)) {
- Textbuffer_dealloc(scheme_buffer);
- return -1;
- }
- if (Tokenizer_emit_textbuffer(self, scheme_buffer, 1))
- return -1;
- if (Tokenizer_emit_char(self, ':'))
- return -1;
- if (slashes) {
- if (Tokenizer_emit_text(self, "//"))
- return -1;
- self->head += 2;
- }
- return 0;
- }
-
- /*
- Handle text in a free external link, including trailing punctuation.
- */
- static int
- Tokenizer_handle_free_link_text(Tokenizer* self, int* parens,
- Textbuffer** tail, Py_UNICODE this)
- {
- #define PUSH_TAIL_BUFFER(tail, error) \
- if ((tail)->size || (tail)->next) { \
- if (Tokenizer_emit_textbuffer(self, tail, 0)) \
- return error; \
- tail = Textbuffer_new(); \
- if (!(tail)) \
- return error; \
- }
-
- if (this == '(' && !(*parens)) {
- *parens = 1;
- PUSH_TAIL_BUFFER(*tail, -1)
- }
- else if (this == ',' || this == ';' || this == '\\' || this == '.' ||
- this == ':' || this == '!' || this == '?' ||
- (!(*parens) && this == ')'))
- return Textbuffer_write(tail, this);
- else
- PUSH_TAIL_BUFFER(*tail, -1)
- return Tokenizer_emit_char(self, this);
- }
-
- /*
- Return whether the current head is the end of a free link.
- */
- static int
- Tokenizer_is_free_link(Tokenizer* self, Py_UNICODE this, Py_UNICODE next)
- {
- // Built from Tokenizer_parse()'s end sentinels:
- Py_UNICODE after = Tokenizer_READ(self, 2);
- int ctx = self->topstack->context;
-
- return (!this || this == '\n' || this == '[' || this == ']' ||
- this == '<' || this == '>' || (this == '\'' && next == '\'') ||
- (this == '|' && ctx & LC_TEMPLATE) ||
- (this == '=' && ctx & (LC_TEMPLATE_PARAM_KEY | LC_HEADING)) ||
- (this == '}' && next == '}' &&
- (ctx & LC_TEMPLATE || (after == '}' && ctx & LC_ARGUMENT))));
- }
-
- /*
- Really parse an external link.
- */
- static PyObject*
- Tokenizer_really_parse_external_link(Tokenizer* self, int brackets,
- Textbuffer** extra)
- {
- Py_UNICODE this, next;
- int parens = 0;
-
- if (brackets ? Tokenizer_parse_bracketed_uri_scheme(self) :
- Tokenizer_parse_free_uri_scheme(self))
- return NULL;
- if (BAD_ROUTE)
- return NULL;
- this = Tokenizer_READ(self, 0);
- if (!this || this == '\n' || this == ' ' || this == ']')
- return Tokenizer_fail_route(self);
- if (!brackets && this == '[')
- return Tokenizer_fail_route(self);
- while (1) {
- this = Tokenizer_READ(self, 0);
- next = Tokenizer_READ(self, 1);
- if (this == '&') {
- PUSH_TAIL_BUFFER(*extra, NULL)
- if (Tokenizer_parse_entity(self))
- return NULL;
- }
- else if (this == '<' && next == '!'
- && Tokenizer_READ(self, 2) == '-'
- && Tokenizer_READ(self, 3) == '-') {
- PUSH_TAIL_BUFFER(*extra, NULL)
- if (Tokenizer_parse_comment(self))
- return NULL;
- }
- else if (!brackets && Tokenizer_is_free_link(self, this, next)) {
- self->head--;
- return Tokenizer_pop(self);
- }
- else if (!this || this == '\n')
- return Tokenizer_fail_route(self);
- else if (this == '{' && next == '{' && Tokenizer_CAN_RECURSE(self)) {
- PUSH_TAIL_BUFFER(*extra, NULL)
- if (Tokenizer_parse_template_or_argument(self))
- return NULL;
- }
- else if (this == ']')
- return Tokenizer_pop(self);
- else if (this == ' ') {
- if (brackets) {
- if (Tokenizer_emit(self, ExternalLinkSeparator))
- return NULL;
- self->topstack->context ^= LC_EXT_LINK_URI;
- self->topstack->context |= LC_EXT_LINK_TITLE;
- self->head++;
- return Tokenizer_parse(self, 0, 0);
- }
- if (Textbuffer_write(extra, ' '))
- return NULL;
- return Tokenizer_pop(self);
- }
- else if (!brackets) {
- if (Tokenizer_handle_free_link_text(self, &parens, extra, this))
- return NULL;
- }
- else {
- if (Tokenizer_emit_char(self, this))
- return NULL;
- }
- self->head++;
- }
- }
-
- /*
- Remove the URI scheme of a new external link from the textbuffer.
- */
- static int
- Tokenizer_remove_uri_scheme_from_textbuffer(Tokenizer* self, PyObject* link)
- {
- PyObject *text = PyObject_GetAttrString(PyList_GET_ITEM(link, 0), "text"),
- *split, *scheme;
- Py_ssize_t length;
- Textbuffer* temp;
-
- if (!text)
- return -1;
- split = PyObject_CallMethod(text, "split", "si", ":", 1);
- Py_DECREF(text);
- if (!split)
- return -1;
- scheme = PyList_GET_ITEM(split, 0);
- length = PyUnicode_GET_SIZE(scheme);
- while (length) {
- temp = self->topstack->textbuffer;
- if (length <= temp->size) {
- temp->size -= length;
- break;
- }
- length -= temp->size;
- self->topstack->textbuffer = temp->next;
- free(temp->data);
- free(temp);
- }
- Py_DECREF(split);
- return 0;
- }
-
- /*
- Parse an external link at the head of the wikicode string.
- */
- static int Tokenizer_parse_external_link(Tokenizer* self, int brackets)
- {
- #define INVALID_CONTEXT self->topstack->context & AGG_INVALID_LINK
- #define NOT_A_LINK \
- if (!brackets && self->topstack->context & LC_DLTERM) \
- return Tokenizer_handle_dl_term(self); \
- return Tokenizer_emit_char(self, Tokenizer_READ(self, 0))
-
- Py_ssize_t reset = self->head;
- PyObject *link, *kwargs;
- Textbuffer *extra = 0;
-
- if (INVALID_CONTEXT || !(Tokenizer_CAN_RECURSE(self))) {
- NOT_A_LINK;
- }
- extra = Textbuffer_new();
- if (!extra)
- return -1;
- self->head++;
- link = Tokenizer_really_parse_external_link(self, brackets, &extra);
- if (BAD_ROUTE) {
- RESET_ROUTE();
- self->head = reset;
- Textbuffer_dealloc(extra);
- NOT_A_LINK;
- }
- if (!link) {
- Textbuffer_dealloc(extra);
- return -1;
- }
- if (!brackets) {
- if (Tokenizer_remove_uri_scheme_from_textbuffer(self, link)) {
- Textbuffer_dealloc(extra);
- Py_DECREF(link);
- return -1;
- }
- }
- kwargs = PyDict_New();
- if (!kwargs) {
- Textbuffer_dealloc(extra);
- Py_DECREF(link);
- return -1;
- }
- PyDict_SetItemString(kwargs, "brackets", brackets ? Py_True : Py_False);
- if (Tokenizer_emit_kwargs(self, ExternalLinkOpen, kwargs)) {
- Textbuffer_dealloc(extra);
- Py_DECREF(link);
- return -1;
- }
- if (Tokenizer_emit_all(self, link)) {
- Textbuffer_dealloc(extra);
- Py_DECREF(link);
- return -1;
- }
- Py_DECREF(link);
- if (Tokenizer_emit(self, ExternalLinkClose)) {
- Textbuffer_dealloc(extra);
- return -1;
- }
- if (extra->size || extra->next)
- return Tokenizer_emit_textbuffer(self, extra, 0);
- Textbuffer_dealloc(extra);
- return 0;
- }
-
- /*
- Parse a section heading at the head of the wikicode string.
- */
- static int Tokenizer_parse_heading(Tokenizer* self)
- {
- Py_ssize_t reset = self->head;
- int best = 1, i, context, diff;
- HeadingData *heading;
- PyObject *level, *kwargs;
-
- self->global |= GL_HEADING;
- self->head += 1;
- while (Tokenizer_READ(self, 0) == '=') {
- best++;
- self->head++;
- }
- context = LC_HEADING_LEVEL_1 << (best > 5 ? 5 : best - 1);
- heading = (HeadingData*) Tokenizer_parse(self, context, 1);
- if (BAD_ROUTE) {
- RESET_ROUTE();
- self->head = reset + best - 1;
- for (i = 0; i < best; i++) {
- if (Tokenizer_emit_char(self, '='))
- return -1;
- }
- self->global ^= GL_HEADING;
- return 0;
- }
- level = NEW_INT_FUNC(heading->level);
- if (!level) {
- Py_DECREF(heading->title);
- free(heading);
- return -1;
- }
- kwargs = PyDict_New();
- if (!kwargs) {
- Py_DECREF(level);
- Py_DECREF(heading->title);
- free(heading);
- return -1;
- }
- PyDict_SetItemString(kwargs, "level", level);
- Py_DECREF(level);
- if (Tokenizer_emit_kwargs(self, HeadingStart, kwargs)) {
- Py_DECREF(heading->title);
- free(heading);
- return -1;
- }
- if (heading->level < best) {
- diff = best - heading->level;
- for (i = 0; i < diff; i++) {
- if (Tokenizer_emit_char(self, '=')) {
- Py_DECREF(heading->title);
- free(heading);
- return -1;
- }
- }
- }
- if (Tokenizer_emit_all(self, heading->title)) {
- Py_DECREF(heading->title);
- free(heading);
- return -1;
- }
- Py_DECREF(heading->title);
- free(heading);
- if (Tokenizer_emit(self, HeadingEnd))
- return -1;
- self->global ^= GL_HEADING;
- return 0;
- }
-
- /*
- Handle the end of a section heading at the head of the string.
- */
- static HeadingData* Tokenizer_handle_heading_end(Tokenizer* self)
- {
- Py_ssize_t reset = self->head;
- int best, i, current, level, diff;
- HeadingData *after, *heading;
- PyObject *stack;
-
- self->head += 1;
- best = 1;
- while (Tokenizer_READ(self, 0) == '=') {
- best++;
- self->head++;
- }
- current = heading_level_from_context(self->topstack->context);
- level = current > best ? (best > 6 ? 6 : best) :
- (current > 6 ? 6 : current);
- after = (HeadingData*) Tokenizer_parse(self, self->topstack->context, 1);
- if (BAD_ROUTE) {
- RESET_ROUTE();
- if (level < best) {
- diff = best - level;
- for (i = 0; i < diff; i++) {
- if (Tokenizer_emit_char(self, '='))
- return NULL;
- }
- }
- self->head = reset + best - 1;
- }
- else {
- for (i = 0; i < best; i++) {
- if (Tokenizer_emit_char(self, '=')) {
- Py_DECREF(after->title);
- free(after);
- return NULL;
- }
- }
- if (Tokenizer_emit_all(self, after->title)) {
- Py_DECREF(after->title);
- free(after);
- return NULL;
- }
- Py_DECREF(after->title);
- level = after->level;
- free(after);
- }
- stack = Tokenizer_pop(self);
- if (!stack)
- return NULL;
- heading = malloc(sizeof(HeadingData));
- if (!heading) {
- PyErr_NoMemory();
- return NULL;
- }
- heading->title = stack;
- heading->level = level;
- return heading;
- }
-
- /*
- Actually parse an HTML entity and ensure that it is valid.
- */
- static int Tokenizer_really_parse_entity(Tokenizer* self)
- {
- PyObject *kwargs, *textobj;
- Py_UNICODE this;
- int numeric, hexadecimal, i, j, zeroes, test;
- char *valid, *text, *buffer, *def;
-
- #define FAIL_ROUTE_AND_EXIT() { \
- Tokenizer_fail_route(self); \
- free(text); \
- return 0; \
- }
-
- if (Tokenizer_emit(self, HTMLEntityStart))
- return -1;
- self->head++;
- this = Tokenizer_READ(self, 0);
- if (!this) {
- Tokenizer_fail_route(self);
- return 0;
- }
- if (this == '#') {
- numeric = 1;
- if (Tokenizer_emit(self, HTMLEntityNumeric))
- return -1;
- self->head++;
- this = Tokenizer_READ(self, 0);
- if (!this) {
- Tokenizer_fail_route(self);
- return 0;
- }
- if (this == 'x' || this == 'X') {
- hexadecimal = 1;
- kwargs = PyDict_New();
- if (!kwargs)
- return -1;
- PyDict_SetItemString(kwargs, "char", Tokenizer_read(self, 0));
- if (Tokenizer_emit_kwargs(self, HTMLEntityHex, kwargs))
- return -1;
- self->head++;
- }
- else
- hexadecimal = 0;
- }
- else
- numeric = hexadecimal = 0;
- if (hexadecimal)
- valid = HEXDIGITS;
- else if (numeric)
- valid = DIGITS;
- else
- valid = ALPHANUM;
- text = calloc(MAX_ENTITY_SIZE, sizeof(char));
- if (!text) {
- PyErr_NoMemory();
- return -1;
- }
- i = 0;
- zeroes = 0;
- while (1) {
- this = Tokenizer_READ(self, 0);
- if (this == ';') {
- if (i == 0)
- FAIL_ROUTE_AND_EXIT()
- break;
- }
- if (i == 0 && this == '0') {
- zeroes++;
- self->head++;
- continue;
- }
- if (i >= MAX_ENTITY_SIZE)
- FAIL_ROUTE_AND_EXIT()
- if (is_marker(this))
- FAIL_ROUTE_AND_EXIT()
- j = 0;
- while (1) {
- if (!valid[j])
- FAIL_ROUTE_AND_EXIT()
- if (this == valid[j])
- break;
- j++;
- }
- text[i] = (char) this;
- self->head++;
- i++;
- }
- if (numeric) {
- sscanf(text, (hexadecimal ? "%x" : "%d"), &test);
- if (test < 1 || test > 0x10FFFF)
- FAIL_ROUTE_AND_EXIT()
- }
- else {
- i = 0;
- while (1) {
- def = entitydefs[i];
- if (!def) // We've reached the end of the defs without finding it
- FAIL_ROUTE_AND_EXIT()
- if (strcmp(text, def) == 0)
- break;
- i++;
- }
- }
- if (zeroes) {
- buffer = calloc(strlen(text) + zeroes + 1, sizeof(char));
- if (!buffer) {
- free(text);
- PyErr_NoMemory();
- return -1;
- }
- for (i = 0; i < zeroes; i++)
- strcat(buffer, "0");
- strcat(buffer, text);
- free(text);
- text = buffer;
- }
- textobj = PyUnicode_FromString(text);
- if (!textobj) {
- free(text);
- return -1;
- }
- free(text);
- kwargs = PyDict_New();
- if (!kwargs) {
- Py_DECREF(textobj);
- return -1;
- }
- PyDict_SetItemString(kwargs, "text", textobj);
- Py_DECREF(textobj);
- if (Tokenizer_emit_kwargs(self, Text, kwargs))
- return -1;
- if (Tokenizer_emit(self, HTMLEntityEnd))
- return -1;
- return 0;
- }
-
- /*
- Parse an HTML entity at the head of the wikicode string.
- */
- static int Tokenizer_parse_entity(Tokenizer* self)
- {
- Py_ssize_t reset = self->head;
- PyObject *tokenlist;
-
- if (Tokenizer_push(self, 0))
- return -1;
- if (Tokenizer_really_parse_entity(self))
- return -1;
- if (BAD_ROUTE) {
- RESET_ROUTE();
- self->head = reset;
- if (Tokenizer_emit_char(self, '&'))
- return -1;
- return 0;
- }
- tokenlist = Tokenizer_pop(self);
- if (!tokenlist)
- return -1;
- if (Tokenizer_emit_all(self, tokenlist)) {
- Py_DECREF(tokenlist);
- return -1;
- }
- Py_DECREF(tokenlist);
- return 0;
- }
-
- /*
- Parse an HTML comment at the head of the wikicode string.
- */
- static int Tokenizer_parse_comment(Tokenizer* self)
- {
- Py_ssize_t reset = self->head + 3;
- PyObject *comment;
- Py_UNICODE this;
-
- self->head += 4;
- if (Tokenizer_push(self, 0))
- return -1;
- while (1) {
- this = Tokenizer_READ(self, 0);
- if (!this) {
- comment = Tokenizer_pop(self);
- Py_XDECREF(comment);
- self->head = reset;
- return Tokenizer_emit_text(self, "<!--");
- }
- if (this == '-' && Tokenizer_READ(self, 1) == this &&
- Tokenizer_READ(self, 2) == '>') {
- if (Tokenizer_emit_first(self, CommentStart))
- return -1;
- if (Tokenizer_emit(self, CommentEnd))
- return -1;
- comment = Tokenizer_pop(self);
- if (!comment)
- return -1;
- if (Tokenizer_emit_all(self, comment))
- return -1;
- Py_DECREF(comment);
- self->head += 2;
- return 0;
- }
- if (Tokenizer_emit_char(self, this))
- return -1;
- self->head++;
- }
- }
-
- /*
- Write a pending tag attribute from data to the stack.
- */
- static int Tokenizer_push_tag_buffer(Tokenizer* self, TagData* data)
- {
- PyObject *tokens, *kwargs, *pad_first, *pad_before_eq, *pad_after_eq;
-
- if (data->context & TAG_QUOTED) {
- if (Tokenizer_emit_first(self, TagAttrQuote))
- return -1;
- tokens = Tokenizer_pop(self);
- if (!tokens)
- return -1;
- if (Tokenizer_emit_all(self, tokens)) {
- Py_DECREF(tokens);
- return -1;
- }
- Py_DECREF(tokens);
- }
- pad_first = Textbuffer_render(data->pad_first);
- pad_before_eq = Textbuffer_render(data->pad_before_eq);
- pad_after_eq = Textbuffer_render(data->pad_after_eq);
- if (!pad_first || !pad_before_eq || !pad_after_eq)
- return -1;
- kwargs = PyDict_New();
- if (!kwargs)
- return -1;
- PyDict_SetItemString(kwargs, "pad_first", pad_first);
- PyDict_SetItemString(kwargs, "pad_before_eq", pad_before_eq);
- PyDict_SetItemString(kwargs, "pad_after_eq", pad_after_eq);
- Py_DECREF(pad_first);
- Py_DECREF(pad_before_eq);
- Py_DECREF(pad_after_eq);
- if (Tokenizer_emit_first_kwargs(self, TagAttrStart, kwargs))
- return -1;
- tokens = Tokenizer_pop(self);
- if (!tokens)
- return -1;
- if (Tokenizer_emit_all(self, tokens)) {
- Py_DECREF(tokens);
- return -1;
- }
- Py_DECREF(tokens);
- if (TagData_reset_buffers(data))
- return -1;
- return 0;
- }
-
- /*
- Handle whitespace inside of an HTML open tag.
- */
- static int
- Tokenizer_handle_tag_space(Tokenizer* self, TagData* data, Py_UNICODE text)
- {
- int ctx = data->context;
- int end_of_value = (ctx & TAG_ATTR_VALUE &&
- !(ctx & (TAG_QUOTED | TAG_NOTE_QUOTE)));
-
- if (end_of_value || (ctx & TAG_QUOTED && ctx & TAG_NOTE_SPACE)) {
- if (Tokenizer_push_tag_buffer(self, data))
- return -1;
- data->context = TAG_ATTR_READY;
- }
- else if (ctx & TAG_NOTE_SPACE)
- data->context = TAG_ATTR_READY;
- else if (ctx & TAG_ATTR_NAME) {
- data->context |= TAG_NOTE_EQUALS;
- if (Textbuffer_write(&(data->pad_before_eq), text))
- return -1;
- }
- if (ctx & TAG_QUOTED && !(ctx & TAG_NOTE_SPACE)) {
- if (Tokenizer_emit_char(self, text))
- return -1;
- }
- else if (data->context & TAG_ATTR_READY)
- return Textbuffer_write(&(data->pad_first), text);
- else if (data->context & TAG_ATTR_VALUE)
- return Textbuffer_write(&(data->pad_after_eq), text);
- return 0;
- }
-
- /*
- Handle regular text inside of an HTML open tag.
- */
- static int Tokenizer_handle_tag_text(Tokenizer* self, Py_UNICODE text)
- {
- Py_UNICODE next = Tokenizer_READ(self, 1);
-
- if (!is_marker(text) || !Tokenizer_CAN_RECURSE(self))
- return Tokenizer_emit_char(self, text);
- else if (text == next && next == '{')
- return Tokenizer_parse_template_or_argument(self);
- else if (text == next && next == '[')
- return Tokenizer_parse_wikilink(self);
- else if (text == '<')
- return Tokenizer_parse_tag(self);
- return Tokenizer_emit_char(self, text);
- }
-
- /*
- Handle all sorts of text data inside of an HTML open tag.
- */
- static int
- Tokenizer_handle_tag_data(Tokenizer* self, TagData* data, Py_UNICODE chunk)
- {
- PyObject *trash;
- int first_time, escaped;
-
- if (data->context & TAG_NAME) {
- first_time = !(data->context & TAG_NOTE_SPACE);
- if (is_marker(chunk) || (Py_UNICODE_ISSPACE(chunk) && first_time)) {
- // Tags must start with text, not spaces
- Tokenizer_fail_route(self);
- return 0;
- }
- else if (first_time)
- data->context |= TAG_NOTE_SPACE;
- else if (Py_UNICODE_ISSPACE(chunk)) {
- data->context = TAG_ATTR_READY;
- return Tokenizer_handle_tag_space(self, data, chunk);
- }
- }
- else if (Py_UNICODE_ISSPACE(chunk))
- return Tokenizer_handle_tag_space(self, data, chunk);
- else if (data->context & TAG_NOTE_SPACE) {
- if (data->context & TAG_QUOTED) {
- data->context = TAG_ATTR_VALUE;
- trash = Tokenizer_pop(self);
- Py_XDECREF(trash);
- self->head = data->reset - 1; // Will be auto-incremented
- }
- else
- Tokenizer_fail_route(self);
- return 0;
- }
- else if (data->context & TAG_ATTR_READY) {
- data->context = TAG_ATTR_NAME;
- if (Tokenizer_push(self, LC_TAG_ATTR))
- return -1;
- }
- else if (data->context & TAG_ATTR_NAME) {
- if (chunk == '=') {
- data->context = TAG_ATTR_VALUE | TAG_NOTE_QUOTE;
- if (Tokenizer_emit(self, TagAttrEquals))
- return -1;
- return 0;
- }
- if (data->context & TAG_NOTE_EQUALS) {
- if (Tokenizer_push_tag_buffer(self, data))
- return -1;
- data->context = TAG_ATTR_NAME;
- if (Tokenizer_push(self, LC_TAG_ATTR))
- return -1;
- }
- }
- else if (data->context & TAG_ATTR_VALUE) {
- escaped = (Tokenizer_READ_BACKWARDS(self, 1) == '\\' &&
- Tokenizer_READ_BACKWARDS(self, 2) != '\\');
- if (data->context & TAG_NOTE_QUOTE) {
- data->context ^= TAG_NOTE_QUOTE;
- if (chunk == '"' && !escaped) {
- data->context |= TAG_QUOTED;
- if (Tokenizer_push(self, self->topstack->context))
- return -1;
- data->reset = self->head;
- return 0;
- }
- }
- else if (data->context & TAG_QUOTED) {
- if (chunk == '"' && !escaped) {
- data->context |= TAG_NOTE_SPACE;
- return 0;
- }
- }
- }
- return Tokenizer_handle_tag_text(self, chunk);
- }
-
- /*
- Handle the closing of a open tag (<foo>).
- */
- static int
- Tokenizer_handle_tag_close_open(Tokenizer* self, TagData* data, PyObject* cls)
- {
- PyObject *padding, *kwargs;
-
- if (data->context & (TAG_ATTR_NAME | TAG_ATTR_VALUE)) {
- if (Tokenizer_push_tag_buffer(self, data))
- return -1;
- }
- padding = Textbuffer_render(data->pad_first);
- if (!padding)
- return -1;
- kwargs = PyDict_New();
- if (!kwargs) {
- Py_DECREF(padding);
- return -1;
- }
- PyDict_SetItemString(kwargs, "padding", padding);
- Py_DECREF(padding);
- if (Tokenizer_emit_kwargs(self, cls, kwargs))
- return -1;
- self->head++;
- return 0;
- }
-
- /*
- Handle the opening of a closing tag (</foo>).
- */
- static int Tokenizer_handle_tag_open_close(Tokenizer* self)
- {
- if (Tokenizer_emit(self, TagOpenClose))
- return -1;
- if (Tokenizer_push(self, LC_TAG_CLOSE))
- return -1;
- self->head++;
- return 0;
- }
-
- /*
- Handle the ending of a closing tag (</foo>).
- */
- static PyObject* Tokenizer_handle_tag_close_close(Tokenizer* self)
- {
- PyObject *closing, *first, *so, *sc;
- int valid = 1;
-
- closing = Tokenizer_pop(self);
- if (!closing)
- return NULL;
- if (PyList_GET_SIZE(closing) != 1)
- valid = 0;
- else {
- first = PyList_GET_ITEM(closing, 0);
- switch (PyObject_IsInstance(first, Text)) {
- case 0:
- valid = 0;
- break;
- case 1: {
- so = strip_tag_name(first);
- sc = strip_tag_name(PyList_GET_ITEM(self->topstack->stack, 1));
- if (so && sc) {
- if (PyUnicode_Compare(so, sc))
- valid = 0;
- Py_DECREF(so);
- Py_DECREF(sc);
- break;
- }
- Py_XDECREF(so);
- Py_XDECREF(sc);
- }
- case -1:
- Py_DECREF(closing);
- return NULL;
- }
- }
- if (!valid) {
- Py_DECREF(closing);
- return Tokenizer_fail_route(self);
- }
- if (Tokenizer_emit_all(self, closing)) {
- Py_DECREF(closing);
- return NULL;
- }
- Py_DECREF(closing);
- if (Tokenizer_emit(self, TagCloseClose))
- return NULL;
- return Tokenizer_pop(self);
- }
-
- /*
- Handle the body of an HTML tag that is parser-blacklisted.
- */
- static PyObject* Tokenizer_handle_blacklisted_tag(Tokenizer* self)
- {
- Py_UNICODE this, next;
-
- while (1) {
- this = Tokenizer_READ(self, 0);
- next = Tokenizer_READ(self, 1);
- if (!this)
- return Tokenizer_fail_route(self);
- else if (this == '<' && next == '/') {
- if (Tokenizer_handle_tag_open_close(self))
- return NULL;
- self->head++;
- return Tokenizer_parse(self, 0, 0);
- }
- else if (this == '&') {
- if (Tokenizer_parse_entity(self))
- return NULL;
- }
- else if (Tokenizer_emit_char(self, this))
- return NULL;
- self->head++;
- }
- }
-
- /*
- Handle the end of an implicitly closing single-only HTML tag.
- */
- static PyObject* Tokenizer_handle_single_only_tag_end(Tokenizer* self)
- {
- PyObject *top, *padding, *kwargs;
-
- top = PyObject_CallMethod(self->topstack->stack, "pop", NULL);
- if (!top)
- return NULL;
- padding = PyObject_GetAttrString(top, "padding");
- Py_DECREF(top);
- if (!padding)
- return NULL;
- kwargs = PyDict_New();
- if (!kwargs) {
- Py_DECREF(padding);
- return NULL;
- }
- PyDict_SetItemString(kwargs, "padding", padding);
- PyDict_SetItemString(kwargs, "implicit", Py_True);
- Py_DECREF(padding);
- if (Tokenizer_emit_kwargs(self, TagCloseSelfclose, kwargs))
- return NULL;
- self->head--; // Offset displacement done by handle_tag_close_open
- return Tokenizer_pop(self);
- }
-
- /*
- Handle the stream end when inside a single-supporting HTML tag.
- */
- static PyObject* Tokenizer_handle_single_tag_end(Tokenizer* self)
- {
- PyObject *token = 0, *padding, *kwargs;
- Py_ssize_t len, index;
- int is_instance;
-
- len = PyList_GET_SIZE(self->topstack->stack);
- for (index = 0; index < len; index++) {
- token = PyList_GET_ITEM(self->topstack->stack, index);
- is_instance = PyObject_IsInstance(token, TagCloseOpen);
- if (is_instance == -1)
- return NULL;
- else if (is_instance == 1)
- break;
- }
- if (!token)
- return NULL;
- padding = PyObject_GetAttrString(token, "padding");
- if (!padding)
- return NULL;
- kwargs = PyDict_New();
- if (!kwargs) {
- Py_DECREF(padding);
- return NULL;
- }
- PyDict_SetItemString(kwargs, "padding", padding);
- PyDict_SetItemString(kwargs, "implicit", Py_True);
- Py_DECREF(padding);
- token = PyObject_Call(TagCloseSelfclose, NOARGS, kwargs);
- Py_DECREF(kwargs);
- if (!token)
- return NULL;
- if (PyList_SetItem(self->topstack->stack, index, token)) {
- Py_DECREF(token);
- return NULL;
- }
- return Tokenizer_pop(self);
- }
-
- /*
- Actually parse an HTML tag, starting with the open (<foo>).
- */
- static PyObject* Tokenizer_really_parse_tag(Tokenizer* self)
- {
- TagData *data = TagData_new();
- PyObject *token, *text, *trash;
- Py_UNICODE this, next;
- int can_exit;
-
- if (!data)
- return NULL;
- if (Tokenizer_push(self, LC_TAG_OPEN)) {
- TagData_dealloc(data);
- return NULL;
- }
- if (Tokenizer_emit(self, TagOpenOpen)) {
- TagData_dealloc(data);
- return NULL;
- }
- while (1) {
- this = Tokenizer_READ(self, 0);
- next = Tokenizer_READ(self, 1);
- can_exit = (!(data->context & (TAG_QUOTED | TAG_NAME)) ||
- data->context & TAG_NOTE_SPACE);
- if (!this) {
- if (self->topstack->context & LC_TAG_ATTR) {
- if (data->context & TAG_QUOTED) {
- // Unclosed attribute quote: reset, don't die
- data->context = TAG_ATTR_VALUE;
- trash = Tokenizer_pop(self);
- Py_XDECREF(trash);
- self->head = data->reset;
- continue;
- }
- trash = Tokenizer_pop(self);
- Py_XDECREF(trash);
- }
- TagData_dealloc(data);
- return Tokenizer_fail_route(self);
- }
- else if (this == '>' && can_exit) {
- if (Tokenizer_handle_tag_close_open(self, data, TagCloseOpen)) {
- TagData_dealloc(data);
- return NULL;
- }
- TagData_dealloc(data);
- self->topstack->context = LC_TAG_BODY;
- token = PyList_GET_ITEM(self->topstack->stack, 1);
- text = PyObject_GetAttrString(token, "text");
- if (!text)
- return NULL;
- if (IS_SINGLE_ONLY(text)) {
- Py_DECREF(text);
- return Tokenizer_handle_single_only_tag_end(self);
- }
- if (IS_PARSABLE(text)) {
- Py_DECREF(text);
- return Tokenizer_parse(self, 0, 0);
- }
- Py_DECREF(text);
- return Tokenizer_handle_blacklisted_tag(self);
- }
- else if (this == '/' && next == '>' && can_exit) {
- if (Tokenizer_handle_tag_close_open(self, data,
- TagCloseSelfclose)) {
- TagData_dealloc(data);
- return NULL;
- }
- TagData_dealloc(data);
- return Tokenizer_pop(self);
- }
- else {
- if (Tokenizer_handle_tag_data(self, data, this) || BAD_ROUTE) {
- TagData_dealloc(data);
- return NULL;
- }
- }
- self->head++;
- }
- }
-
- /*
- Handle the (possible) start of an implicitly closing single tag.
- */
- static int Tokenizer_handle_invalid_tag_start(Tokenizer* self)
- {
- Py_ssize_t reset = self->head + 1, pos = 0;
- Textbuffer* buf;
- PyObject *name, *tag;
- Py_UNICODE this;
-
- self->head += 2;
- buf = Textbuffer_new();
- if (!buf)
- return -1;
- while (1) {
- this = Tokenizer_READ(self, pos);
- if (Py_UNICODE_ISSPACE(this) || is_marker(this)) {
- name = Textbuffer_render(buf);
- if (!name) {
- Textbuffer_dealloc(buf);
- return -1;
- }
- if (!IS_SINGLE_ONLY(name))
- FAIL_ROUTE(0);
- Py_DECREF(name);
- break;
- }
- Textbuffer_write(&buf, this);
- pos++;
- }
- Textbuffer_dealloc(buf);
- if (!BAD_ROUTE)
- tag = Tokenizer_really_parse_tag(self);
- if (BAD_ROUTE) {
- RESET_ROUTE();
- self->head = reset;
- return Tokenizer_emit_text(self, "</");
- }
- if (!tag)
- return -1;
- // Set invalid=True flag of TagOpenOpen
- if (PyObject_SetAttrString(PyList_GET_ITEM(tag, 0), "invalid", Py_True))
- return -1;
- if (Tokenizer_emit_all(self, tag)) {
- Py_DECREF(tag);
- return -1;
- }
- Py_DECREF(tag);
- return 0;
- }
-
- /*
- Parse an HTML tag at the head of the wikicode string.
- */
- static int Tokenizer_parse_tag(Tokenizer* self)
- {
- Py_ssize_t reset = self->head;
- PyObject* tag;
-
- self->head++;
- tag = Tokenizer_really_parse_tag(self);
- if (BAD_ROUTE) {
- RESET_ROUTE();
- self->head = reset;
- return Tokenizer_emit_char(self, '<');
- }
- if (!tag) {
- return -1;
- }
- if (Tokenizer_emit_all(self, tag)) {
- Py_DECREF(tag);
- return -1;
- }
- Py_DECREF(tag);
- return 0;
- }
-
- /*
- Write the body of a tag and the tokens that should surround it.
- */
- static int Tokenizer_emit_style_tag(Tokenizer* self, const char* tag,
- const char* ticks, PyObject* body)
- {
- PyObject *markup, *kwargs;
-
- markup = PyUnicode_FromString(ticks);
- if (!markup)
- return -1;
- kwargs = PyDict_New();
- if (!kwargs) {
- Py_DECREF(markup);
- return -1;
- }
- PyDict_SetItemString(kwargs, "wiki_markup", markup);
- Py_DECREF(markup);
- if (Tokenizer_emit_kwargs(self, TagOpenOpen, kwargs))
- return -1;
- if (Tokenizer_emit_text(self, tag))
- return -1;
- if (Tokenizer_emit(self, TagCloseOpen))
- return -1;
- if (Tokenizer_emit_all(self, body))
- return -1;
- Py_DECREF(body);
- if (Tokenizer_emit(self, TagOpenClose))
- return -1;
- if (Tokenizer_emit_text(self, tag))
- return -1;
- if (Tokenizer_emit(self, TagCloseClose))
- return -1;
- return 0;
- }
-
- /*
- Parse wiki-style italics.
- */
- static int Tokenizer_parse_italics(Tokenizer* self)
- {
- Py_ssize_t reset = self->head;
- int context;
- PyObject *stack;
-
- stack = Tokenizer_parse(self, LC_STYLE_ITALICS, 1);
- if (BAD_ROUTE) {
- RESET_ROUTE();
- self->head = reset;
- if (BAD_ROUTE_CONTEXT & LC_STYLE_PASS_AGAIN) {
- context = LC_STYLE_ITALICS | LC_STYLE_SECOND_PASS;
- stack = Tokenizer_parse(self, context, 1);
- }
- else
- return Tokenizer_emit_text(self, "''");
- }
- if (!stack)
- return -1;
- return Tokenizer_emit_style_tag(self, "i", "''", stack);
- }
-
- /*
- Parse wiki-style bold.
- */
- static int Tokenizer_parse_bold(Tokenizer* self)
- {
- Py_ssize_t reset = self->head;
- PyObject *stack;
-
- stack = Tokenizer_parse(self, LC_STYLE_BOLD, 1);
- if (BAD_ROUTE) {
- RESET_ROUTE();
- self->head = reset;
- if (self->topstack->context & LC_STYLE_SECOND_PASS)
- return Tokenizer_emit_char(self, '\'') ? -1 : 1;
- if (self->topstack->context & LC_STYLE_ITALICS) {
- self->topstack->context |= LC_STYLE_PASS_AGAIN;
- return Tokenizer_emit_text(self, "'''");
- }
- if (Tokenizer_emit_char(self, '\''))
- return -1;
- return Tokenizer_parse_italics(self);
- }
- if (!stack)
- return -1;
- return Tokenizer_emit_style_tag(self, "b", "'''", stack);
- }
-
- /*
- Parse wiki-style italics and bold together (i.e., five ticks).
- */
- static int Tokenizer_parse_italics_and_bold(Tokenizer* self)
- {
- Py_ssize_t reset = self->head;
- PyObject *stack, *stack2;
-
- stack = Tokenizer_parse(self, LC_STYLE_BOLD, 1);
- if (BAD_ROUTE) {
- RESET_ROUTE();
- self->head = reset;
- stack = Tokenizer_parse(self, LC_STYLE_ITALICS, 1);
- if (BAD_ROUTE) {
- RESET_ROUTE();
- self->head = reset;
- return Tokenizer_emit_text(self, "'''''");
- }
- if (!stack)
- return -1;
- reset = self->head;
- stack2 = Tokenizer_parse(self, LC_STYLE_BOLD, 1);
- if (BAD_ROUTE) {
- RESET_ROUTE();
- self->head = reset;
- if (Tokenizer_emit_text(self, "'''"))
- return -1;
- return Tokenizer_emit_style_tag(self, "i", "''", stack);
- }
- if (!stack2)
- return -1;
- if (Tokenizer_push(self, 0))
- return -1;
- if (Tokenizer_emit_style_tag(self, "i", "''", stack))
- return -1;
- if (Tokenizer_emit_all(self, stack2))
- return -1;
- Py_DECREF(stack2);
- stack2 = Tokenizer_pop(self);
- if (!stack2)
- return -1;
- return Tokenizer_emit_style_tag(self, "b", "'''", stack2);
- }
- if (!stack)
- return -1;
- reset = self->head;
- stack2 = Tokenizer_parse(self, LC_STYLE_ITALICS, 1);
- if (BAD_ROUTE) {
- RESET_ROUTE();
- self->head = reset;
- if (Tokenizer_emit_text(self, "''"))
- return -1;
- return Tokenizer_emit_style_tag(self, "b", "'''", stack);
- }
- if (!stack2)
- return -1;
- if (Tokenizer_push(self, 0))
- return -1;
- if (Tokenizer_emit_style_tag(self, "b", "'''", stack))
- return -1;
- if (Tokenizer_emit_all(self, stack2))
- return -1;
- Py_DECREF(stack2);
- stack2 = Tokenizer_pop(self);
- if (!stack2)
- return -1;
- return Tokenizer_emit_style_tag(self, "i", "''", stack2);
- }
-
- /*
- Parse wiki-style formatting (''/''' for italics/bold).
- */
- static PyObject* Tokenizer_parse_style(Tokenizer* self)
- {
- int context = self->topstack->context, ticks = 2, i;
-
- self->head += 2;
- while (Tokenizer_READ(self, 0) == '\'') {
- self->head++;
- ticks++;
- }
- if (ticks > 5) {
- for (i = 0; i < ticks - 5; i++) {
- if (Tokenizer_emit_char(self, '\''))
- return NULL;
- }
- ticks = 5;
- }
- else if (ticks == 4) {
- if (Tokenizer_emit_char(self, '\''))
- return NULL;
- ticks = 3;
- }
- if ((context & LC_STYLE_ITALICS && (ticks == 2 || ticks == 5)) ||
- (context & LC_STYLE_BOLD && (ticks == 3 || ticks == 5))) {
- if (ticks == 5)
- self->head -= context & LC_STYLE_ITALICS ? 3 : 2;
- return Tokenizer_pop(self);
- }
- if (!Tokenizer_CAN_RECURSE(self)) {
- if (ticks == 3) {
- if (context & LC_STYLE_SECOND_PASS) {
- if (Tokenizer_emit_char(self, '\''))
- return NULL;
- return Tokenizer_pop(self);
- }
- if (context & LC_STYLE_ITALICS)
- self->topstack->context |= LC_STYLE_PASS_AGAIN;
- }
- for (i = 0; i < ticks; i++) {
- if (Tokenizer_emit_char(self, '\''))
- return NULL;
- }
- }
- else if (ticks == 2) {
- if (Tokenizer_parse_italics(self))
- return NULL;
- }
- else if (ticks == 3) {
- switch (Tokenizer_parse_bold(self)) {
- case 1:
- return Tokenizer_pop(self);
- case -1:
- return NULL;
- }
- }
- else {
- if (Tokenizer_parse_italics_and_bold(self))
- return NULL;
- }
- self->head--;
- return Py_None;
- }
-
- /*
- Handle a list marker at the head (#, *, ;, :).
- */
- static int Tokenizer_handle_list_marker(Tokenizer* self)
- {
- PyObject *markup = Tokenizer_read(self, 0), *kwargs;
- Py_UNICODE code = *PyUnicode_AS_UNICODE(markup);
-
- if (code == ';')
- self->topstack->context |= LC_DLTERM;
- kwargs = PyDict_New();
- if (!kwargs)
- return -1;
- PyDict_SetItemString(kwargs, "wiki_markup", markup);
- if (Tokenizer_emit_kwargs(self, TagOpenOpen, kwargs))
- return -1;
- if (Tokenizer_emit_text(self, GET_HTML_TAG(code)))
- return -1;
- if (Tokenizer_emit(self, TagCloseSelfclose))
- return -1;
- return 0;
- }
-
- /*
- Handle a wiki-style list (#, *, ;, :).
- */
- static int Tokenizer_handle_list(Tokenizer* self)
- {
- Py_UNICODE marker = Tokenizer_READ(self, 1);
-
- if (Tokenizer_handle_list_marker(self))
- return -1;
- while (marker == '#' || marker == '*' || marker == ';' ||
- marker == ':') {
- self->head++;
- if (Tokenizer_handle_list_marker(self))
- return -1;
- marker = Tokenizer_READ(self, 1);
- }
- return 0;
- }
-
- /*
- Handle a wiki-style horizontal rule (----) in the string.
- */
- static int Tokenizer_handle_hr(Tokenizer* self)
- {
- PyObject *markup, *kwargs;
- Textbuffer *buffer = Textbuffer_new();
- int i;
-
- if (!buffer)
- return -1;
- self->head += 3;
- for (i = 0; i < 4; i++) {
- if (Textbuffer_write(&buffer, '-'))
- return -1;
- }
- while (Tokenizer_READ(self, 1) == '-') {
- if (Textbuffer_write(&buffer, '-'))
- return -1;
- self->head++;
- }
- markup = Textbuffer_render(buffer);
- Textbuffer_dealloc(buffer);
- if (!markup)
- return -1;
- kwargs = PyDict_New();
- if (!kwargs)
- return -1;
- PyDict_SetItemString(kwargs, "wiki_markup", markup);
- Py_DECREF(markup);
- if (Tokenizer_emit_kwargs(self, TagOpenOpen, kwargs))
- return -1;
- if (Tokenizer_emit_text(self, "hr"))
- return -1;
- if (Tokenizer_emit(self, TagCloseSelfclose))
- return -1;
- return 0;
- }
-
- /*
- Handle the term in a description list ('foo' in ';foo:bar').
- */
- static int Tokenizer_handle_dl_term(Tokenizer* self)
- {
- self->topstack->context ^= LC_DLTERM;
- if (Tokenizer_READ(self, 0) == ':')
- return Tokenizer_handle_list_marker(self);
- return Tokenizer_emit_char(self, '\n');
- }
-
- /*
- Handle the end of the stream of wikitext.
- */
- static PyObject* Tokenizer_handle_end(Tokenizer* self, int context)
- {
- PyObject *token, *text, *trash;
- int single;
-
- if (context & AGG_FAIL) {
- if (context & LC_TAG_BODY) {
- token = PyList_GET_ITEM(self->topstack->stack, 1);
- text = PyObject_GetAttrString(token, "text");
- if (!text)
- return NULL;
- single = IS_SINGLE(text);
- Py_DECREF(text);
- if (single)
- return Tokenizer_handle_single_tag_end(self);
- }
- else if (context & AGG_DOUBLE) {
- trash = Tokenizer_pop(self);
- Py_XDECREF(trash);
- }
- return Tokenizer_fail_route(self);
- }
- return Tokenizer_pop(self);
- }
-
- /*
- Make sure we are not trying to write an invalid character. Return 0 if
- everything is safe, or -1 if the route must be failed.
- */
- static int Tokenizer_verify_safe(Tokenizer* self, int context, Py_UNICODE data)
- {
- if (context & LC_FAIL_NEXT)
- return -1;
- if (context & LC_WIKILINK) {
- if (context & LC_WIKILINK_TEXT)
- return (data == '[' && Tokenizer_READ(self, 1) == '[') ? -1 : 0;
- else if (data == ']' || data == '{')
- self->topstack->context |= LC_FAIL_NEXT;
- else if (data == '\n' || data == '[' || data == '}')
- return -1;
- return 0;
- }
- if (context & LC_EXT_LINK_TITLE)
- return (data == '\n') ? -1 : 0;
- if (context & LC_TAG_CLOSE)
- return (data == '<') ? -1 : 0;
- if (context & LC_TEMPLATE_NAME) {
- if (data == '{' || data == '}' || data == '[') {
- self->topstack->context |= LC_FAIL_NEXT;
- return 0;
- }
- if (data == ']') {
- return -1;
- }
- if (data == '|')
- return 0;
- if (context & LC_HAS_TEXT) {
- if (context & LC_FAIL_ON_TEXT) {
- if (!Py_UNICODE_ISSPACE(data))
- return -1;
- }
- else {
- if (data == '\n')
- self->topstack->context |= LC_FAIL_ON_TEXT;
- }
- }
- else if (!Py_UNICODE_ISSPACE(data))
- self->topstack->context |= LC_HAS_TEXT;
- }
- else {
- if (context & LC_FAIL_ON_EQUALS) {
- if (data == '=') {
- return -1;
- }
- }
- else if (context & LC_FAIL_ON_LBRACE) {
- if (data == '{' || (Tokenizer_READ(self, -1) == '{' &&
- Tokenizer_READ(self, -2) == '{')) {
- if (context & LC_TEMPLATE)
- self->topstack->context |= LC_FAIL_ON_EQUALS;
- else
- self->topstack->context |= LC_FAIL_NEXT;
- return 0;
- }
- self->topstack->context ^= LC_FAIL_ON_LBRACE;
- }
- else if (context & LC_FAIL_ON_RBRACE) {
- if (data == '}') {
- if (context & LC_TEMPLATE)
- self->topstack->context |= LC_FAIL_ON_EQUALS;
- else
- self->topstack->context |= LC_FAIL_NEXT;
- return 0;
- }
- self->topstack->context ^= LC_FAIL_ON_RBRACE;
- }
- else if (data == '{')
- self->topstack->context |= LC_FAIL_ON_LBRACE;
- else if (data == '}')
- self->topstack->context |= LC_FAIL_ON_RBRACE;
- }
- return 0;
- }
-
- /*
- Parse the wikicode string, using context for when to stop. If push is true,
- we will push a new context, otherwise we won't and context will be ignored.
- */
- static PyObject* Tokenizer_parse(Tokenizer* self, int context, int push)
- {
- int this_context;
- Py_UNICODE this, next, next_next, last;
- PyObject* temp;
-
- if (push) {
- if (Tokenizer_push(self, context))
- return NULL;
- }
- while (1) {
- this = Tokenizer_READ(self, 0);
- this_context = self->topstack->context;
- if (this_context & AGG_UNSAFE) {
- if (Tokenizer_verify_safe(self, this_context, this) < 0) {
- if (this_context & AGG_DOUBLE) {
- temp = Tokenizer_pop(self);
- Py_XDECREF(temp);
- }
- return Tokenizer_fail_route(self);
- }
- }
- if (!is_marker(this)) {
- if (Tokenizer_emit_char(self, this))
- return NULL;
- self->head++;
- continue;
- }
- if (!this)
- return Tokenizer_handle_end(self, this_context);
- next = Tokenizer_READ(self, 1);
- last = Tokenizer_READ_BACKWARDS(self, 1);
- if (this == next && next == '{') {
- if (Tokenizer_CAN_RECURSE(self)) {
- if (Tokenizer_parse_template_or_argument(self))
- return NULL;
- }
- else if (Tokenizer_emit_char(self, this))
- return NULL;
- }
- else if (this == '|' && this_context & LC_TEMPLATE) {
- if (Tokenizer_handle_template_param(self))
- return NULL;
- }
- else if (this == '=' && this_context & LC_TEMPLATE_PARAM_KEY) {
- if (Tokenizer_handle_template_param_value(self))
- return NULL;
- }
- else if (this == next && next == '}' && this_context & LC_TEMPLATE)
- return Tokenizer_handle_template_end(self);
- else if (this == '|' && this_context & LC_ARGUMENT_NAME) {
- if (Tokenizer_handle_argument_separator(self))
- return NULL;
- }
- else if (this == next && next == '}' && this_context & LC_ARGUMENT) {
- if (Tokenizer_READ(self, 2) == '}') {
- return Tokenizer_handle_argument_end(self);
- }
- if (Tokenizer_emit_char(self, this))
- return NULL;
- }
- else if (this == next && next == '[' && Tokenizer_CAN_RECURSE(self)) {
- if (!(this_context & AGG_INVALID_LINK)) {
- if (Tokenizer_parse_wikilink(self))
- return NULL;
- }
- else if (Tokenizer_emit_char(self, this))
- return NULL;
- }
- else if (this == '|' && this_context & LC_WIKILINK_TITLE) {
- if (Tokenizer_handle_wikilink_separator(self))
- return NULL;
- }
- else if (this == next && next == ']' && this_context & LC_WIKILINK)
- return Tokenizer_handle_wikilink_end(self);
- else if (this == '[') {
- if (Tokenizer_parse_external_link(self, 1))
- return NULL;
- }
- else if (this == ':' && !is_marker(last)) {
- if (Tokenizer_parse_external_link(self, 0))
- return NULL;
- }
- else if (this == ']' && this_context & LC_EXT_LINK_TITLE)
- return Tokenizer_pop(self);
- else if (this == '=' && !(self->global & GL_HEADING)) {
- if (!last || last == '\n') {
- if (Tokenizer_parse_heading(self))
- return NULL;
- }
- else if (Tokenizer_emit_char(self, this))
- return NULL;
- }
- else if (this == '=' && this_context & LC_HEADING)
- return (PyObject*) Tokenizer_handle_heading_end(self);
- else if (this == '\n' && this_context & LC_HEADING)
- return Tokenizer_fail_route(self);
- else if (this == '&') {
- if (Tokenizer_parse_entity(self))
- return NULL;
- }
- else if (this == '<' && next == '!') {
- next_next = Tokenizer_READ(self, 2);
- if (next_next == Tokenizer_READ(self, 3) && next_next == '-') {
- if (Tokenizer_parse_comment(self))
- return NULL;
- }
- else if (Tokenizer_emit_char(self, this))
- return NULL;
- }
- else if (this == '<' && next == '/' && Tokenizer_READ(self, 2)) {
- if (this_context & LC_TAG_BODY ?
- Tokenizer_handle_tag_open_close(self) :
- Tokenizer_handle_invalid_tag_start(self))
- return NULL;
- }
- else if (this == '<' && !(this_context & LC_TAG_CLOSE)) {
- if (Tokenizer_CAN_RECURSE(self)) {
- if (Tokenizer_parse_tag(self))
- return NULL;
- }
- else if (Tokenizer_emit_char(self, this))
- return NULL;
- }
- else if (this == '>' && this_context & LC_TAG_CLOSE)
- return Tokenizer_handle_tag_close_close(self);
- else if (this == next && next == '\'' && !self->skip_style_tags) {
- temp = Tokenizer_parse_style(self);
- if (temp != Py_None)
- return temp;
- }
- else if (!last || last == '\n') {
- if (this == '#' || this == '*' || this == ';' || this == ':') {
- if (Tokenizer_handle_list(self))
- return NULL;
- }
- else if (this == '-' && this == next &&
- this == Tokenizer_READ(self, 2) &&
- this == Tokenizer_READ(self, 3)) {
- if (Tokenizer_handle_hr(self))
- return NULL;
- }
- else if (Tokenizer_emit_char(self, this))
- return NULL;
- }
- else if ((this == '\n' || this == ':') && this_context & LC_DLTERM) {
- if (Tokenizer_handle_dl_term(self))
- return NULL;
- }
- else if (Tokenizer_emit_char(self, this))
- return NULL;
- self->head++;
- }
- }
-
- /*
- Build a list of tokens from a string of wikicode and return it.
- */
- static PyObject* Tokenizer_tokenize(Tokenizer* self, PyObject* args)
- {
- PyObject *text, *temp;
- int context = 0, skip_style_tags = 0;
-
- if (PyArg_ParseTuple(args, "U|ii", &text, &context, &skip_style_tags)) {
- Py_XDECREF(self->text);
- self->text = PySequence_Fast(text, "expected a sequence");
- }
- else {
- const char* encoded;
- Py_ssize_t size;
- /* Failed to parse a Unicode object; try a string instead. */
- PyErr_Clear();
- if (!PyArg_ParseTuple(args, "s#|ii", &encoded, &size, &context,
- &skip_style_tags))
- return NULL;
- temp = PyUnicode_FromStringAndSize(encoded, size);
- if (!text)
- return NULL;
- Py_XDECREF(self->text);
- text = PySequence_Fast(temp, "expected a sequence");
- Py_XDECREF(temp);
- self->text = text;
- }
- self->head = self->global = self->depth = self->cycles = 0;
- self->length = PyList_GET_SIZE(self->text);
- self->skip_style_tags = skip_style_tags;
- return Tokenizer_parse(self, context, 1);
- }
-
- static int load_entitydefs(void)
- {
- PyObject *tempmod, *defmap, *deflist;
- unsigned numdefs, i;
- #ifdef IS_PY3K
- PyObject *string;
- #endif
-
- tempmod = PyImport_ImportModule(ENTITYDEFS_MODULE);
- if (!tempmod)
- return -1;
- defmap = PyObject_GetAttrString(tempmod, "entitydefs");
- if (!defmap)
- return -1;
- Py_DECREF(tempmod);
- deflist = PyDict_Keys(defmap);
- if (!deflist)
- return -1;
- Py_DECREF(defmap);
- numdefs = (unsigned) PyList_GET_SIZE(defmap);
- entitydefs = calloc(numdefs + 1, sizeof(char*));
- if (!entitydefs)
- return -1;
- for (i = 0; i < numdefs; i++) {
- #ifdef IS_PY3K
- string = PyUnicode_AsASCIIString(PyList_GET_ITEM(deflist, i));
- if (!string)
- return -1;
- entitydefs[i] = PyBytes_AsString(string);
- #else
- entitydefs[i] = PyBytes_AsString(PyList_GET_ITEM(deflist, i));
- #endif
- if (!entitydefs[i])
- return -1;
- }
- Py_DECREF(deflist);
- return 0;
- }
-
- static int load_tokens(void)
- {
- PyObject *tempmod, *tokens,
- *globals = PyEval_GetGlobals(),
- *locals = PyEval_GetLocals(),
- *fromlist = PyList_New(1),
- *modname = IMPORT_NAME_FUNC("tokens");
- char *name = "mwparserfromhell.parser";
-
- if (!fromlist || !modname)
- return -1;
- PyList_SET_ITEM(fromlist, 0, modname);
- tempmod = PyImport_ImportModuleLevel(name, globals, locals, fromlist, 0);
- Py_DECREF(fromlist);
- if (!tempmod)
- return -1;
- tokens = PyObject_GetAttrString(tempmod, "tokens");
- Py_DECREF(tempmod);
-
- Text = PyObject_GetAttrString(tokens, "Text");
-
- TemplateOpen = PyObject_GetAttrString(tokens, "TemplateOpen");
- TemplateParamSeparator = PyObject_GetAttrString(tokens,
- "TemplateParamSeparator");
- TemplateParamEquals = PyObject_GetAttrString(tokens,
- "TemplateParamEquals");
- TemplateClose = PyObject_GetAttrString(tokens, "TemplateClose");
-
- ArgumentOpen = PyObject_GetAttrString(tokens, "ArgumentOpen");
- ArgumentSeparator = PyObject_GetAttrString(tokens, "ArgumentSeparator");
- ArgumentClose = PyObject_GetAttrString(tokens, "ArgumentClose");
-
- WikilinkOpen = PyObject_GetAttrString(tokens, "WikilinkOpen");
- WikilinkSeparator = PyObject_GetAttrString(tokens, "WikilinkSeparator");
- WikilinkClose = PyObject_GetAttrString(tokens, "WikilinkClose");
-
- ExternalLinkOpen = PyObject_GetAttrString(tokens, "ExternalLinkOpen");
- ExternalLinkSeparator = PyObject_GetAttrString(tokens,
- "ExternalLinkSeparator");
- ExternalLinkClose = PyObject_GetAttrString(tokens, "ExternalLinkClose");
-
- HTMLEntityStart = PyObject_GetAttrString(tokens, "HTMLEntityStart");
- HTMLEntityNumeric = PyObject_GetAttrString(tokens, "HTMLEntityNumeric");
- HTMLEntityHex = PyObject_GetAttrString(tokens, "HTMLEntityHex");
- HTMLEntityEnd = PyObject_GetAttrString(tokens, "HTMLEntityEnd");
-
- HeadingStart = PyObject_GetAttrString(tokens, "HeadingStart");
- HeadingEnd = PyObject_GetAttrString(tokens, "HeadingEnd");
-
- CommentStart = PyObject_GetAttrString(tokens, "CommentStart");
- CommentEnd = PyObject_GetAttrString(tokens, "CommentEnd");
-
- TagOpenOpen = PyObject_GetAttrString(tokens, "TagOpenOpen");
- TagAttrStart = PyObject_GetAttrString(tokens, "TagAttrStart");
- TagAttrEquals = PyObject_GetAttrString(tokens, "TagAttrEquals");
- TagAttrQuote = PyObject_GetAttrString(tokens, "TagAttrQuote");
- TagCloseOpen = PyObject_GetAttrString(tokens, "TagCloseOpen");
- TagCloseSelfclose = PyObject_GetAttrString(tokens, "TagCloseSelfclose");
- TagOpenClose = PyObject_GetAttrString(tokens, "TagOpenClose");
- TagCloseClose = PyObject_GetAttrString(tokens, "TagCloseClose");
-
- Py_DECREF(tokens);
- return 0;
- }
-
- static int load_definitions(void)
- {
- PyObject *tempmod,
- *globals = PyEval_GetGlobals(),
- *locals = PyEval_GetLocals(),
- *fromlist = PyList_New(1),
- *modname = IMPORT_NAME_FUNC("definitions");
- char *name = "mwparserfromhell";
-
- if (!fromlist || !modname)
- return -1;
- PyList_SET_ITEM(fromlist, 0, modname);
- tempmod = PyImport_ImportModuleLevel(name, globals, locals, fromlist, 0);
- Py_DECREF(fromlist);
- if (!tempmod)
- return -1;
- definitions = PyObject_GetAttrString(tempmod, "definitions");
- Py_DECREF(tempmod);
- return 0;
- }
-
- PyMODINIT_FUNC INIT_FUNC_NAME(void)
- {
- PyObject *module;
-
- TokenizerType.tp_new = PyType_GenericNew;
- if (PyType_Ready(&TokenizerType) < 0)
- INIT_ERROR;
- module = CREATE_MODULE;
- if (!module)
- INIT_ERROR;
- Py_INCREF(&TokenizerType);
- PyModule_AddObject(module, "CTokenizer", (PyObject*) &TokenizerType);
- Py_INCREF(Py_True);
- PyDict_SetItemString(TokenizerType.tp_dict, "USES_C", Py_True);
- EMPTY = PyUnicode_FromString("");
- NOARGS = PyTuple_New(0);
- if (!EMPTY || !NOARGS)
- INIT_ERROR;
- if (load_entitydefs() || load_tokens() || load_definitions())
- INIT_ERROR;
- #ifdef IS_PY3K
- return module;
- #endif
- }
|