From e0660f8bc31a00c3119d13d2d37bcf18042b3102 Mon Sep 17 00:00:00 2001
From: Ben Kurtovic <ben.kurtovic@verizon.net>
Date: Sat, 22 Sep 2012 22:47:05 -0400
Subject: [PATCH 01/47] Committing this C work for now.

---
 docs/conf.py                        |   5 +-
 mwparserfromhell/parser/builder.c   |  24 +++
 mwparserfromhell/parser/tokenizer.c | 322 ++++++++++++++++++++++++++++++++++++
 setup.py                            |   9 +-
 4 files changed, 357 insertions(+), 3 deletions(-)
 create mode 100644 mwparserfromhell/parser/builder.c
 create mode 100644 mwparserfromhell/parser/tokenizer.c

diff --git a/docs/conf.py b/docs/conf.py
index 6cc3664..cff089b 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -17,6 +17,7 @@ import sys, os
 # add these directories to sys.path here. If the directory is relative to the
 # documentation root, use os.path.abspath to make it absolute, like shown here.
 sys.path.insert(0, os.path.abspath('..'))
+import mwparserfromhell
 
 # -- General configuration -----------------------------------------------------
 
@@ -48,9 +49,9 @@ copyright = u'2012 Ben Kurtovic'
 # built documents.
 #
 # The short X.Y version.
-version = '0.2'
+version = ".".join(mwparserfromhell.__version__.split(".", 2)[:2])
 # The full version, including alpha/beta/rc tags.
-release = '0.2.dev'
+release = mwparserfromhell.__version__
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.
diff --git a/mwparserfromhell/parser/builder.c b/mwparserfromhell/parser/builder.c
new file mode 100644
index 0000000..7cbe236
--- /dev/null
+++ b/mwparserfromhell/parser/builder.c
@@ -0,0 +1,24 @@
+/*
+Builder for MWParserFromHell
+Copyright (C) 2012 Ben Kurtovic <ben.kurtovic@verizon.net>
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of
+this software and associated documentation files (the "Software"), to deal in
+the Software without restriction, including without limitation the rights to
+use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
+of the Software, and to permit persons to whom the Software is furnished to do
+so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+*/
+
+#include <Python.h>
diff --git a/mwparserfromhell/parser/tokenizer.c b/mwparserfromhell/parser/tokenizer.c
new file mode 100644
index 0000000..3fdc370
--- /dev/null
+++ b/mwparserfromhell/parser/tokenizer.c
@@ -0,0 +1,322 @@
+/*
+Tokenizer for MWParserFromHell
+Copyright (C) 2012 Ben Kurtovic <ben.kurtovic@verizon.net>
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of
+this software and associated documentation files (the "Software"), to deal in
+the Software without restriction, including without limitation the rights to
+use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
+of the Software, and to permit persons to whom the Software is furnished to do
+so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+*/
+
+#ifndef PY_SSIZE_T_CLEAN
+#define PY_SSIZE_T_CLEAN
+#endif
+
+#include <Python.h>
+#include "structmember.h"
+
+static const Py_UNICODE* OUT_OF_BOUNDS = "";
+static const Py_UNICODE* MARKERS[] = {"{", "}", "[", "]", "<", ">", "|", "=",
+                                      "&", "#", "*", ";", ":", "/", "-", "!",
+                                      "\n", OUT_OF_BOUNDS};
+
+static PyMethodDef
+module_methods[] = {
+    {NULL}
+};
+
+typedef struct {
+    PyObject_HEAD
+    PyObject* text;        /* text to tokenize */
+    PyObject* stacks;      /* token stacks */
+    PyObject* topstack;    /* topmost stack */
+    Py_ssize_t head;       /* current position in text */
+    Py_ssize_t length;     /* length of text */
+    Py_ssize_t global;     /* global context */
+} Tokenizer;
+
+static PyObject*
+Tokenizer_new(PyTypeObject* type, PyObject* args, PyObject* kwds)
+{
+    Tokenizer *self;
+
+    self = (Tokenizer*) type->tp_alloc(type, 0);
+    if (self != NULL) {
+
+        self->text = Py_None;
+        Py_INCREF(Py_None);
+
+        self->stacks = PyList_New(0);
+        if (self->stacks == NULL) {
+            Py_DECREF(self);
+            return NULL;
+        }
+
+        self->head = 0;
+        self->length = 0;
+        self->global = 0;
+    }
+
+    return (PyObject*) self;
+}
+
+static void
+Tokenizer_dealloc(Tokenizer* self)
+{
+    Py_XDECREF(self->text);
+    Py_XDECREF(self->stacks);
+    Py_XDECREF(self->topstack);
+    self->ob_type->tp_free((PyObject*) self);
+}
+
+static int
+Tokenizer_init(Tokenizer* self, PyObject* args, PyObject* kwds)
+{
+    static char* kwlist[] = {NULL};
+    if (!PyArg_ParseTupleAndKeywords(args, kwds, "", kwlist))
+        return -1;
+    return 0;
+}
+
+#define Tokenizer_STACK(self) PyList_GET_ITEM(self->topstack, 0)
+#define Tokenizer_CONTEXT(self) PyList_GET_ITEM(self->topstack, 1)
+#define Tokenizer_TEXTBUFFER(self) PyList_GET_ITEM(self->topstack, 2)
+
+static int
+Tokenizer_set_context(Tokenizer* self, Py_ssize_t value)
+{
+    if (PyList_SetItem(self->topstack, 1, PyInt_FromSsize_t(value)))
+        return -1;
+    return 0;
+}
+
+static int
+Tokenizer_set_textbuffer(Tokenizer* self, PyObject* value)
+{
+    if (PyList_SetItem(self->topstack, 2, value))
+        return -1;
+    return 0;
+}
+
+/*
+    Add a new token stack, context, and textbuffer to the list.
+*/
+static int
+Tokenizer_push(Tokenizer* self, int context)
+{
+    PyObject* top = PyList_New(3);
+    PyList_SET_ITEM(top, 0, PyList_New(0));
+    PyList_SET_ITEM(top, 1, PyInt_FromSsize_t(0));
+    PyList_SET_ITEM(top, 2, PyList_New(0));
+
+    Py_XDECREF(self->topstack);
+    self->topstack = top;
+
+    if (PyList_Append(self->stacks, top))
+        return -1;
+    return 0;
+}
+
+/*
+    Push the textbuffer onto the stack as a Text node and clear it.
+*/
+static int
+Tokenizer_push_textbuffer(Tokenizer* self)
+{
+    if (PyList_GET_SIZE(Tokenizer_TEXTBUFFER(self)) > 0) {
+
+        PyObject* text;
+        // tokens.Text(text="".join(self._textbuffer))
+
+        if (PyList_Append(Tokenizer_STACK(self), text)
+            return -1;
+
+        if (Tokenizer_set_textbuffer(self, PyList_New(0)))
+            return -1;
+
+        return 0;
+    }
+}
+
+/*
+    Pop the current stack/context/textbuffer, returing the stack.
+*/
+static PyObject*
+Tokenizer_pop(Tokenizer* self)
+{
+    if (Tokenizer_push_textbuffer(self))
+        return NULL;
+
+    self->stacks // POP!?
+}
+
+/*
+    Pop the current stack/context/textbuffer, returing the stack. We will also
+    replace the underlying stack's context with the current stack's.
+*/
+static PyObject*
+Tokenizer_pop_keeping_context(Tokenizer* self)
+{
+    if (Tokenizer_push_textbuffer(self))
+        return NULL;
+}
+
+/*
+    Read the value at a relative point in the wikicode.
+*/
+static Py_UNICODE*
+Tokenizer_read(Tokenizer* self, Py_ssize_t delta)
+{
+    Py_ssize_t index = self->head + delta;
+
+    if (index >= self->length) {
+        return OUT_OF_BOUNDS;
+    }
+
+    PyObject* item = PySequence_Fast_GET_ITEM(self->text, index);
+    return PyUnicode_AS_UNICODE(item);
+}
+
+/*
+    Parse the wikicode string, using *context* for when to stop.
+*/
+static PyObject*
+Tokenizer_parse(Tokenizer* self, int context)
+{
+    Py_UNICODE* this;
+
+    Tokenizer_push(self, context);
+
+    while (1) {
+        this = Tokenizer_read(self, 0);
+        if (this not in MARKERS) {
+            WRITE TEXT
+        }
+        if (this == OUT_OF_BOUNDS) {
+            return Tokenizer_push(self);
+        }
+        printf("%p %i %c\n", this, *this, *this);
+        self->head++;
+    }
+}
+
+/*
+    Build a list of tokens from a string of wikicode and return it.
+*/
+static PyObject*
+Tokenizer_tokenize(Tokenizer* self, PyObject *args)
+{
+    PyObject* text;
+
+    if (!PyArg_ParseTuple(args, "U", &text)) {
+        /* Failed to parse a Unicode object; try a string instead. */
+        PyErr_Clear();
+        const char* encoded;
+        Py_ssize_t size;
+
+        if (!PyArg_ParseTuple(args, "s#", &encoded, &size)) {
+            return NULL;
+        }
+
+        PyObject* temp;
+        temp = PyUnicode_FromStringAndSize(encoded, size);
+        if (text == NULL)
+            return NULL;
+
+        Py_XDECREF(self->text);
+        text = PySequence_Fast(temp, "expected a sequence");
+        Py_XDECREF(temp);
+        self->text = text;
+    }
+    else {
+        Py_XDECREF(self->text);
+        self->text = PySequence_Fast(text, "expected a sequence");
+    }
+
+    self->length = PySequence_Length(self->text);
+
+    return Tokenizer_parse(self, 0);
+}
+
+static PyMethodDef
+Tokenizer_methods[] = {
+    {"tokenize", (PyCFunction) Tokenizer_tokenize, METH_VARARGS,
+    "Build a list of tokens from a string of wikicode and return it."},
+    {NULL}
+};
+
+static PyMemberDef
+Tokenizer_members[] = {
+    {NULL}
+};
+
+static PyTypeObject
+TokenizerType = {
+    PyObject_HEAD_INIT(NULL)
+    0,                                                      /* ob_size */
+    "_tokenizer.CTokenizer",                                /* tp_name */
+    sizeof(Tokenizer),                                      /* tp_basicsize */
+    0,                                                      /* tp_itemsize */
+    (destructor) Tokenizer_dealloc,                         /* tp_dealloc */
+    0,                                                      /* tp_print */
+    0,                                                      /* tp_getattr */
+    0,                                                      /* tp_setattr */
+    0,                                                      /* tp_compare */
+    0,                                                      /* tp_repr */
+    0,                                                      /* tp_as_number */
+    0,                                                      /* tp_as_sequence */
+    0,                                                      /* tp_as_mapping */
+    0,                                                      /* tp_hash  */
+    0,                                                      /* tp_call */
+    0,                                                      /* tp_str */
+    0,                                                      /* tp_getattro */
+    0,                                                      /* tp_setattro */
+    0,                                                      /* tp_as_buffer */
+    Py_TPFLAGS_DEFAULT,                                     /* tp_flags */
+    "Creates a list of tokens from a string of wikicode.",  /* tp_doc */
+    0,                                                      /* tp_traverse */
+    0,                                                      /* tp_clear */
+    0,                                                      /* tp_richcompare */
+    0,                                                      /* tp_weaklistoffset */
+    0,                                                      /* tp_iter */
+    0,                                                      /* tp_iternext */
+    Tokenizer_methods,                                      /* tp_methods */
+    Tokenizer_members,                                      /* tp_members */
+    0,                                                      /* tp_getset */
+    0,                                                      /* tp_base */
+    0,                                                      /* tp_dict */
+    0,                                                      /* tp_descr_get */
+    0,                                                      /* tp_descr_set */
+    0,                                                      /* tp_dictoffset */
+    (initproc) Tokenizer_init,                              /* tp_init */
+    0,                                                      /* tp_alloc */
+    Tokenizer_new,                                          /* tp_new */
+};
+
+PyMODINIT_FUNC
+init_tokenizer(void)
+{
+    PyObject* module;
+
+    TokenizerType.tp_new = PyType_GenericNew;
+    if (PyType_Ready(&TokenizerType) < 0)
+        return;
+
+    module = Py_InitModule("_tokenizer", module_methods);
+
+    Py_INCREF(&TokenizerType);
+    PyModule_AddObject(module, "CTokenizer", (PyObject*) &TokenizerType);
+}
diff --git a/setup.py b/setup.py
index 9faa56c..3664626 100644
--- a/setup.py
+++ b/setup.py
@@ -21,16 +21,23 @@
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 # SOFTWARE.
 
-from setuptools import setup, find_packages
+from setuptools import setup, find_packages, Extension
 
 from mwparserfromhell import __version__
 
 with open("README.rst") as fp:
     long_docs = fp.read()
 
+builder = Extension("mwparserfromhell.parser._builder",
+                    sources = ["mwparserfromhell/parser/builder.c"])
+
+tokenizer = Extension("mwparserfromhell.parser._tokenizer",
+                    sources = ["mwparserfromhell/parser/tokenizer.c"])
+
 setup(
     name = "mwparserfromhell",
     packages = find_packages(exclude=("tests",)),
+    ext_modules = [builder, tokenizer],
     test_suite = "tests",
     version = __version__,
     author = "Ben Kurtovic",

From 4cc4791d4871b833454ade8d9f52ee35e8bca742 Mon Sep 17 00:00:00 2001
From: Ben Kurtovic <ben.kurtovic@verizon.net>
Date: Sun, 23 Sep 2012 01:29:27 -0400
Subject: [PATCH 02/47] Adding a bunch more, and implementing
 Tokenizer_push_textbuffer.

---
 mwparserfromhell/parser/tokenizer.c | 116 ++++++++++++++++++++++++++++++------
 setup.py                            |   2 +-
 2 files changed, 98 insertions(+), 20 deletions(-)

diff --git a/mwparserfromhell/parser/tokenizer.c b/mwparserfromhell/parser/tokenizer.c
index 3fdc370..aec7b1d 100644
--- a/mwparserfromhell/parser/tokenizer.c
+++ b/mwparserfromhell/parser/tokenizer.c
@@ -28,10 +28,15 @@ SOFTWARE.
 #include <Python.h>
 #include "structmember.h"
 
-static const Py_UNICODE* OUT_OF_BOUNDS = "";
-static const Py_UNICODE* MARKERS[] = {"{", "}", "[", "]", "<", ">", "|", "=",
-                                      "&", "#", "*", ";", ":", "/", "-", "!",
-                                      "\n", OUT_OF_BOUNDS};
+#define PU (Py_UNICODE*)
+static const Py_UNICODE* OUT_OF_BOUNDS = PU"";
+static const Py_UNICODE* MARKERS[] = {PU"{", PU"}", PU"[", PU"]", PU"<", PU">",
+                                      PU"|", PU"=", PU"&", PU"#", PU"*", PU";",
+                                      PU":", PU"/", PU"-", PU"!", PU"\n", PU""};
+#undef PU
+
+static PyObject* contexts;
+static PyObject* tokens;
 
 static PyMethodDef
 module_methods[] = {
@@ -60,7 +65,7 @@ Tokenizer_new(PyTypeObject* type, PyObject* args, PyObject* kwds)
         Py_INCREF(Py_None);
 
         self->stacks = PyList_New(0);
-        if (self->stacks == NULL) {
+        if (!self->stacks) {
             Py_DECREF(self);
             return NULL;
         }
@@ -91,9 +96,9 @@ Tokenizer_init(Tokenizer* self, PyObject* args, PyObject* kwds)
     return 0;
 }
 
-#define Tokenizer_STACK(self) PyList_GET_ITEM(self->topstack, 0)
-#define Tokenizer_CONTEXT(self) PyList_GET_ITEM(self->topstack, 1)
-#define Tokenizer_TEXTBUFFER(self) PyList_GET_ITEM(self->topstack, 2)
+#define Tokenizer_STACK(self) PySequence_Fast_GET_ITEM(self->topstack, 0)
+#define Tokenizer_CONTEXT(self) PySequence_Fast_GET_ITEM(self->topstack, 1)
+#define Tokenizer_TEXTBUFFER(self) PySequence_Fast_GET_ITEM(self->topstack, 2)
 
 static int
 Tokenizer_set_context(Tokenizer* self, Py_ssize_t value)
@@ -136,19 +141,65 @@ Tokenizer_push(Tokenizer* self, int context)
 static int
 Tokenizer_push_textbuffer(Tokenizer* self)
 {
-    if (PyList_GET_SIZE(Tokenizer_TEXTBUFFER(self)) > 0) {
+    if (PySequence_Fast_GET_SIZE(Tokenizer_TEXTBUFFER(self)) > 0) {
+        PyObject* sep = PyUnicode_FromString("");
+        if (!sep) return -1;
+        PyObject* text = PyUnicode_Join(sep, Tokenizer_TEXTBUFFER(self));
+        Py_DECREF(sep);
+        if (!text) return -1;
+
+        PyObject* klass = PyObject_GetAttrString(tokens, "Text");
+        if (!klass) return -1;
+        PyObject* args = PyTuple_New(0);
+        if (!args) return -1;
+        PyObject* kwargs = PyDict_New();
+        if (!kwargs) return -1;
+        PyDict_SetItemString(kwargs, "text", text);
+        Py_DECREF(text);
+
+        PyObject* token = PyInstance_New(klass, args, kwargs);
+        if (!token) {
+            Py_DECREF(klass);
+            Py_DECREF(args);
+            Py_DECREF(kwargs);
+            return -1;
+        }
 
-        PyObject* text;
-        // tokens.Text(text="".join(self._textbuffer))
+        Py_DECREF(klass);
+        Py_DECREF(args);
+        Py_DECREF(kwargs);
 
-        if (PyList_Append(Tokenizer_STACK(self), text)
+        if (PyList_Append(Tokenizer_STACK(self), token)) {
+            Py_XDECREF(token);
             return -1;
+        }
+
+        Py_XDECREF(token);
 
         if (Tokenizer_set_textbuffer(self, PyList_New(0)))
             return -1;
+    }
+    return 0;
+}
 
-        return 0;
+static int
+Tokenizer_delete_top_of_stack(Tokenizer* self)
+{
+    if (PySequence_DelItem(self->stacks, -1))
+        return -1;
+    Py_DECREF(self->topstack);
+
+    Py_ssize_t size = PySequence_Fast_GET_SIZE(self->stacks);
+    if (size > 0) {
+        PyObject* top = PySequence_Fast_GET_ITEM(self->stacks, size - 1);
+        self->topstack = top;
+        Py_INCREF(top);
+    }
+    else {
+        self->topstack = NULL;
     }
+
+    return 0;
 }
 
 /*
@@ -160,7 +211,13 @@ Tokenizer_pop(Tokenizer* self)
     if (Tokenizer_push_textbuffer(self))
         return NULL;
 
-    self->stacks // POP!?
+    PyObject* stack = Tokenizer_STACK(self);
+    Py_INCREF(stack);
+
+    if (Tokenizer_delete_top_of_stack(self))
+        return NULL;
+
+    return stack;
 }
 
 /*
@@ -172,6 +229,19 @@ Tokenizer_pop_keeping_context(Tokenizer* self)
 {
     if (Tokenizer_push_textbuffer(self))
         return NULL;
+
+    PyObject* stack = Tokenizer_STACK(self);
+    PyObject* context = Tokenizer_CONTEXT(self);
+    Py_INCREF(stack);
+    Py_INCREF(context);
+
+    if (Tokenizer_delete_top_of_stack(self))
+        return NULL;
+
+    if (PyList_SetItem(self->topstack, 1, context))
+        return NULL;
+
+    return stack;
 }
 
 /*
@@ -183,7 +253,7 @@ Tokenizer_read(Tokenizer* self, Py_ssize_t delta)
     Py_ssize_t index = self->head + delta;
 
     if (index >= self->length) {
-        return OUT_OF_BOUNDS;
+        return (Py_UNICODE*) OUT_OF_BOUNDS;
     }
 
     PyObject* item = PySequence_Fast_GET_ITEM(self->text, index);
@@ -202,11 +272,11 @@ Tokenizer_parse(Tokenizer* self, int context)
 
     while (1) {
         this = Tokenizer_read(self, 0);
-        if (this not in MARKERS) {
+     /*   if (this not in MARKERS) {
             WRITE TEXT
-        }
+        } */
         if (this == OUT_OF_BOUNDS) {
-            return Tokenizer_push(self);
+            return Tokenizer_pop(self);
         }
         printf("%p %i %c\n", this, *this, *this);
         self->head++;
@@ -233,7 +303,7 @@ Tokenizer_tokenize(Tokenizer* self, PyObject *args)
 
         PyObject* temp;
         temp = PyUnicode_FromStringAndSize(encoded, size);
-        if (text == NULL)
+        if (!text)
             return NULL;
 
         Py_XDECREF(self->text);
@@ -319,4 +389,12 @@ init_tokenizer(void)
 
     Py_INCREF(&TokenizerType);
     PyModule_AddObject(module, "CTokenizer", (PyObject*) &TokenizerType);
+
+    PyObject* globals = PyEval_GetGlobals();
+    PyObject* locals = PyEval_GetLocals();
+    PyObject* fromlist = PyList_New(0);
+
+    contexts = PyImport_ImportModuleLevel("contexts", globals, locals, fromlist, 1);
+    tokens = PyImport_ImportModuleLevel("tokens", globals, locals, fromlist, 1);
+    Py_DECREF(fromlist);
 }
diff --git a/setup.py b/setup.py
index 3664626..e348ce5 100644
--- a/setup.py
+++ b/setup.py
@@ -32,7 +32,7 @@ builder = Extension("mwparserfromhell.parser._builder",
                     sources = ["mwparserfromhell/parser/builder.c"])
 
 tokenizer = Extension("mwparserfromhell.parser._tokenizer",
-                    sources = ["mwparserfromhell/parser/tokenizer.c"])
+                      sources = ["mwparserfromhell/parser/tokenizer.c"])
 
 setup(
     name = "mwparserfromhell",

From 9c4aba13912c9d5b274a61a5f7c6d9945f72c0b6 Mon Sep 17 00:00:00 2001
From: Ben Kurtovic <ben.kurtovic@verizon.net>
Date: Sun, 23 Sep 2012 03:40:19 -0400
Subject: [PATCH 03/47] Adding a few more functions.

---
 mwparserfromhell/parser/tokenizer.c | 114 ++++++++++++++++++++++++++++++++----
 1 file changed, 101 insertions(+), 13 deletions(-)

diff --git a/mwparserfromhell/parser/tokenizer.c b/mwparserfromhell/parser/tokenizer.c
index aec7b1d..99c9bfc 100644
--- a/mwparserfromhell/parser/tokenizer.c
+++ b/mwparserfromhell/parser/tokenizer.c
@@ -26,15 +26,20 @@ SOFTWARE.
 #endif
 
 #include <Python.h>
+#include "setjmp.h"
 #include "structmember.h"
 
+static PyObject* EMPTY;
+
 #define PU (Py_UNICODE*)
-static const Py_UNICODE* OUT_OF_BOUNDS = PU"";
 static const Py_UNICODE* MARKERS[] = {PU"{", PU"}", PU"[", PU"]", PU"<", PU">",
                                       PU"|", PU"=", PU"&", PU"#", PU"*", PU";",
                                       PU":", PU"/", PU"-", PU"!", PU"\n", PU""};
 #undef PU
 
+static jmp_buf exception_env;
+static const int BAD_ROUTE = 1;
+
 static PyObject* contexts;
 static PyObject* tokens;
 
@@ -142,10 +147,7 @@ static int
 Tokenizer_push_textbuffer(Tokenizer* self)
 {
     if (PySequence_Fast_GET_SIZE(Tokenizer_TEXTBUFFER(self)) > 0) {
-        PyObject* sep = PyUnicode_FromString("");
-        if (!sep) return -1;
-        PyObject* text = PyUnicode_Join(sep, Tokenizer_TEXTBUFFER(self));
-        Py_DECREF(sep);
+        PyObject* text = PyUnicode_Join(EMPTY, Tokenizer_TEXTBUFFER(self));
         if (!text) return -1;
 
         PyObject* klass = PyObject_GetAttrString(tokens, "Text");
@@ -174,7 +176,7 @@ Tokenizer_push_textbuffer(Tokenizer* self)
             return -1;
         }
 
-        Py_XDECREF(token);
+        Py_DECREF(token);
 
         if (Tokenizer_set_textbuffer(self, PyList_New(0)))
             return -1;
@@ -245,19 +247,104 @@ Tokenizer_pop_keeping_context(Tokenizer* self)
 }
 
 /*
+    Fail the current tokenization route.
+
+    Discards the current stack/context/textbuffer and "raises a BAD_ROUTE
+    exception", which is implemented using longjmp().
+*/
+static void
+Tokenizer_fail_route(Tokenizer* self)
+{
+    Tokenizer_pop(self);
+    longjmp(exception_env, BAD_ROUTE);
+}
+
+/*
+    Write a token to the end of the current token stack.
+*/
+static int
+Tokenizer_write(Tokenizer* self, PyObject* token)
+{
+    if (Tokenizer_push_textbuffer(self))
+        return -1;
+
+    if (PyList_Append(Tokenizer_STACK(self), token)) {
+        Py_XDECREF(token);
+        return -1;
+    }
+
+    Py_XDECREF(token);
+    return 0;
+}
+
+/*
+    Write a token to the beginning of the current token stack.
+*/
+static int
+Tokenizer_write_first(Tokenizer* self, PyObject* token)
+{
+    if (Tokenizer_push_textbuffer(self))
+        return -1;
+
+    if (PyList_Insert(Tokenizer_STACK(self), 0, token)) {
+        Py_XDECREF(token);
+        return -1;
+    }
+
+    Py_XDECREF(token);
+    return 0;
+}
+
+/*
+    Write text to the current textbuffer.
+*/
+static int
+Tokenizer_write_text(Tokenizer* self, PyObject* text)
+{
+    if (PyList_Append(Tokenizer_TEXTBUFFER(self), text)) {
+        Py_XDECREF(text);
+        return -1;
+    }
+
+    Py_XDECREF(text);
+    return 0;
+}
+
+/*
+    Write a series of tokens to the current stack at once.
+*/
+static int
+Tokenizer_write_all(Tokenizer* self, PyObject* tokenlist)
+{
+    if (Tokenizer_push_textbuffer(self))
+        Py_XDECREF(tokenlist);
+        return -1;
+
+    PyObject* stack = Tokenizer_STACK(self);
+    Py_ssize_t size = PySequence_Fast_GET_SIZE(stack);
+
+    if (PyList_SetSlice(stack, size, size, tokenlist)) {
+        Py_XDECREF(tokenlist);
+        return -1;
+    }
+
+    Py_XDECREF(tokenlist);
+    return 0;
+}
+
+/*
     Read the value at a relative point in the wikicode.
 */
-static Py_UNICODE*
+static PyObject*
 Tokenizer_read(Tokenizer* self, Py_ssize_t delta)
 {
     Py_ssize_t index = self->head + delta;
 
     if (index >= self->length) {
-        return (Py_UNICODE*) OUT_OF_BOUNDS;
+        return EMPTY;
     }
 
-    PyObject* item = PySequence_Fast_GET_ITEM(self->text, index);
-    return PyUnicode_AS_UNICODE(item);
+    return PySequence_Fast_GET_ITEM(self->text, index);
 }
 
 /*
@@ -266,7 +353,7 @@ Tokenizer_read(Tokenizer* self, Py_ssize_t delta)
 static PyObject*
 Tokenizer_parse(Tokenizer* self, int context)
 {
-    Py_UNICODE* this;
+    PyObject* this;
 
     Tokenizer_push(self, context);
 
@@ -275,10 +362,9 @@ Tokenizer_parse(Tokenizer* self, int context)
      /*   if (this not in MARKERS) {
             WRITE TEXT
         } */
-        if (this == OUT_OF_BOUNDS) {
+        if (this == EMPTY) {
             return Tokenizer_pop(self);
         }
-        printf("%p %i %c\n", this, *this, *this);
         self->head++;
     }
 }
@@ -390,6 +476,8 @@ init_tokenizer(void)
     Py_INCREF(&TokenizerType);
     PyModule_AddObject(module, "CTokenizer", (PyObject*) &TokenizerType);
 
+    EMPTY = PyUnicode_FromString("");
+
     PyObject* globals = PyEval_GetGlobals();
     PyObject* locals = PyEval_GetLocals();
     PyObject* fromlist = PyList_New(0);

From 5267c30cf60b9c03cdf908112f8bffc390a87ac1 Mon Sep 17 00:00:00 2001
From: Ben Kurtovic <ben.kurtovic@verizon.net>
Date: Sun, 23 Sep 2012 03:57:04 -0400
Subject: [PATCH 04/47] Fix refcount handling; implement
 Tokenizer_write_text_then_stack.

---
 mwparserfromhell/parser/tokenizer.c | 52 +++++++++++++++++++++++--------------
 1 file changed, 32 insertions(+), 20 deletions(-)

diff --git a/mwparserfromhell/parser/tokenizer.c b/mwparserfromhell/parser/tokenizer.c
index 99c9bfc..3f7e84e 100644
--- a/mwparserfromhell/parser/tokenizer.c
+++ b/mwparserfromhell/parser/tokenizer.c
@@ -247,10 +247,9 @@ Tokenizer_pop_keeping_context(Tokenizer* self)
 }
 
 /*
-    Fail the current tokenization route.
-
-    Discards the current stack/context/textbuffer and "raises a BAD_ROUTE
-    exception", which is implemented using longjmp().
+    Fail the current tokenization route. Discards the current
+    stack/context/textbuffer and "raises a BAD_ROUTE exception", which is
+    implemented using longjmp().
 */
 static void
 Tokenizer_fail_route(Tokenizer* self)
@@ -268,12 +267,9 @@ Tokenizer_write(Tokenizer* self, PyObject* token)
     if (Tokenizer_push_textbuffer(self))
         return -1;
 
-    if (PyList_Append(Tokenizer_STACK(self), token)) {
-        Py_XDECREF(token);
+    if (PyList_Append(Tokenizer_STACK(self), token))
         return -1;
-    }
 
-    Py_XDECREF(token);
     return 0;
 }
 
@@ -286,12 +282,9 @@ Tokenizer_write_first(Tokenizer* self, PyObject* token)
     if (Tokenizer_push_textbuffer(self))
         return -1;
 
-    if (PyList_Insert(Tokenizer_STACK(self), 0, token)) {
-        Py_XDECREF(token);
+    if (PyList_Insert(Tokenizer_STACK(self), 0, token))
         return -1;
-    }
 
-    Py_XDECREF(token);
     return 0;
 }
 
@@ -301,12 +294,9 @@ Tokenizer_write_first(Tokenizer* self, PyObject* token)
 static int
 Tokenizer_write_text(Tokenizer* self, PyObject* text)
 {
-    if (PyList_Append(Tokenizer_TEXTBUFFER(self), text)) {
-        Py_XDECREF(text);
+    if (PyList_Append(Tokenizer_TEXTBUFFER(self), text))
         return -1;
-    }
 
-    Py_XDECREF(text);
     return 0;
 }
 
@@ -317,18 +307,40 @@ static int
 Tokenizer_write_all(Tokenizer* self, PyObject* tokenlist)
 {
     if (Tokenizer_push_textbuffer(self))
-        Py_XDECREF(tokenlist);
         return -1;
 
     PyObject* stack = Tokenizer_STACK(self);
     Py_ssize_t size = PySequence_Fast_GET_SIZE(stack);
 
-    if (PyList_SetSlice(stack, size, size, tokenlist)) {
-        Py_XDECREF(tokenlist);
+    if (PyList_SetSlice(stack, size, size, tokenlist))
+        return -1;
+
+    return 0;
+}
+
+/*
+    Pop the current stack, write text, and then write the stack.
+*/
+static int
+Tokenizer_write_text_then_stack(Tokenizer* self, PyObject* text)
+{
+    PyObject* stack = Tokenizer_pop(self);
+    if (Tokenizer_write_text(self, text)) {
+        Py_XDECREF(stack);
         return -1;
     }
 
-    Py_XDECREF(tokenlist);
+    if (stack) {
+        if (PySequence_Fast_GET_SIZE(stack) > 0) {
+            if (Tokenizer_write_all(self, stack)) {
+                Py_DECREF(stack);
+                return -1;
+            }
+        }
+        Py_DECREF(stack);
+    }
+
+    self->head--;
     return 0;
 }
 

From 8729d20f078df40c50a70ee7cbd392b534173a88 Mon Sep 17 00:00:00 2001
From: Ben Kurtovic <ben.kurtovic@verizon.net>
Date: Sun, 23 Sep 2012 17:40:46 -0400
Subject: [PATCH 05/47] Fill out Tokenizer_parse(); build a bunch of empty
 function definitions.

---
 mwparserfromhell/parser/tokenizer.c | 252 ++++++++++++++++++++++++++++++++++--
 1 file changed, 240 insertions(+), 12 deletions(-)

diff --git a/mwparserfromhell/parser/tokenizer.c b/mwparserfromhell/parser/tokenizer.c
index 3f7e84e..0d18473 100644
--- a/mwparserfromhell/parser/tokenizer.c
+++ b/mwparserfromhell/parser/tokenizer.c
@@ -26,8 +26,8 @@ SOFTWARE.
 #endif
 
 #include <Python.h>
-#include "setjmp.h"
-#include "structmember.h"
+#include <setjmp.h>
+#include <structmember.h>
 
 static PyObject* EMPTY;
 
@@ -35,7 +35,10 @@ static PyObject* EMPTY;
 static const Py_UNICODE* MARKERS[] = {PU"{", PU"}", PU"[", PU"]", PU"<", PU">",
                                       PU"|", PU"=", PU"&", PU"#", PU"*", PU";",
                                       PU":", PU"/", PU"-", PU"!", PU"\n", PU""};
-#undef PU
+static const int NUM_MARKERS = 17;
+
+#define CONTEXT(name) PyInt_AsSsize_t((PyIntObject*) \
+                                          PyObject_GetAttrString(contexts, name))
 
 static jmp_buf exception_env;
 static const int BAD_ROUTE = 1;
@@ -103,6 +106,7 @@ Tokenizer_init(Tokenizer* self, PyObject* args, PyObject* kwds)
 
 #define Tokenizer_STACK(self) PySequence_Fast_GET_ITEM(self->topstack, 0)
 #define Tokenizer_CONTEXT(self) PySequence_Fast_GET_ITEM(self->topstack, 1)
+#define Tokenizer_CONTEXT_VAL(self) PyInt_AsSsize_t((PyIntObject*) Tokenizer_CONTEXT(self))
 #define Tokenizer_TEXTBUFFER(self) PySequence_Fast_GET_ITEM(self->topstack, 2)
 
 static int
@@ -125,11 +129,11 @@ Tokenizer_set_textbuffer(Tokenizer* self, PyObject* value)
     Add a new token stack, context, and textbuffer to the list.
 */
 static int
-Tokenizer_push(Tokenizer* self, int context)
+Tokenizer_push(Tokenizer* self, Py_ssize_t context)
 {
     PyObject* top = PyList_New(3);
     PyList_SET_ITEM(top, 0, PyList_New(0));
-    PyList_SET_ITEM(top, 1, PyInt_FromSsize_t(0));
+    PyList_SET_ITEM(top, 1, PyInt_FromSsize_t(context));
     PyList_SET_ITEM(top, 2, PyList_New(0));
 
     Py_XDECREF(self->topstack);
@@ -345,7 +349,7 @@ Tokenizer_write_text_then_stack(Tokenizer* self, PyObject* text)
 }
 
 /*
-    Read the value at a relative point in the wikicode.
+    Read the value at a relative point in the wikicode, forwards.
 */
 static PyObject*
 Tokenizer_read(Tokenizer* self, Py_ssize_t delta)
@@ -360,23 +364,247 @@ Tokenizer_read(Tokenizer* self, Py_ssize_t delta)
 }
 
 /*
-    Parse the wikicode string, using *context* for when to stop.
+    Read the value at a relative point in the wikicode, backwards.
 */
 static PyObject*
-Tokenizer_parse(Tokenizer* self, int context)
+Tokenizer_read_backwards(Tokenizer* self, Py_ssize_t delta)
+{
+    if (delta > self->head) {
+        return EMPTY;
+    }
+
+    Py_ssize_t index = self->head - delta;
+    return PySequence_Fast_GET_ITEM(self->text, index);
+}
+
+static int
+Tokenizer_parse_template_or_argument(Tokenizer* self)
+{
+
+}
+
+static int
+Tokenizer_parse_template(Tokenizer* self)
+{
+
+}
+
+static int
+Tokenizer_parse_argument(Tokenizer* self)
+{
+
+}
+
+static int
+Tokenizer_verify_safe(Tokenizer* self)
+{
+
+}
+
+static int
+Tokenizer_handle_template_param(Tokenizer* self)
+{
+
+}
+
+static int
+Tokenizer_handle_template_param_value(Tokenizer* self)
 {
-    PyObject* this;
+
+}
+
+static PyObject*
+Tokenizer_handle_template_end(Tokenizer* self)
+{
+
+}
+
+static int
+Tokenizer_handle_argument_separator(Tokenizer* self)
+{
+
+}
+
+static PyObject*
+Tokenizer_handle_argument_end(Tokenizer* self)
+{
+
+}
+
+static int
+Tokenizer_parse_wikilink(Tokenizer* self)
+{
+
+}
+
+static int
+Tokenizer_handle_wikilink_separator(Tokenizer* self)
+{
+
+}
+
+static PyObject*
+Tokenizer_handle_wikilink_end(Tokenizer* self)
+{
+
+}
+
+static int
+Tokenizer_parse_heading(Tokenizer* self)
+{
+
+}
+
+static PyObject*
+Tokenizer_handle_heading_end(Tokenizer* self)
+{
+
+}
+
+static int
+Tokenizer_really_parse_entity(Tokenizer* self)
+{
+
+}
+
+static int
+Tokenizer_parse_entity(Tokenizer* self)
+{
+
+}
+
+static int
+Tokenizer_parse_comment(Tokenizer* self)
+{
+
+}
+
+
+/*
+    Parse the wikicode string, using context for when to stop.
+*/
+static PyObject*
+Tokenizer_parse(Tokenizer* self, Py_ssize_t context)
+{
+    Py_ssize_t fail_contexts = (
+        CONTEXT("TEMPLATE") | CONTEXT("ARGUMENT") | CONTEXT("HEADING") |
+        CONTEXT("COMMENT"));
+
+    PyObject *this, *next;
+    Py_UNICODE *this_data, *next_data, *next_next_data, *last_data;
+    Py_ssize_t this_context;
+    int is_marker, i;
 
     Tokenizer_push(self, context);
 
     while (1) {
         this = Tokenizer_read(self, 0);
-     /*   if (this not in MARKERS) {
-            WRITE TEXT
-        } */
+        this_data = PyUnicode_AS_UNICODE(this);
+
+        is_marker = 0;
+        for (i = 0; i < NUM_MARKERS; i++) {
+            if (MARKERS[i] == this_data) {
+                is_marker = 1;
+                break;
+            }
+        }
+
+        if (!is_marker) {
+            Tokenizer_write_text(self, this);
+            self->head++;
+            continue;
+        }
+
+        this_context = Tokenizer_CONTEXT_VAL(self);
+
         if (this == EMPTY) {
+            if (this_context & fail_contexts) {
+                Tokenizer_fail_route(self);
+            }
             return Tokenizer_pop(self);
         }
+
+        next = Tokenizer_read(self, 1);
+        next_data = PyUnicode_AS_UNICODE(next);
+
+        if (this_context & CONTEXT("COMMENT")) {
+            if (this_data == next_data && next_data == PU "-") {
+                if (PyUnicode_AS_UNICODE(Tokenizer_read(self, 2)) == PU ">") {
+                    return Tokenizer_pop(self);
+                }
+            }
+            Tokenizer_write_text(self, this);
+        }
+        else if (this_data == next_data && next_data == PU "{") {
+            Tokenizer_parse_template_or_argument(self);
+        }
+        else if (this_data == PU "|" && this_context & CONTEXT("TEMPLATE")) {
+            Tokenizer_handle_template_param(self);
+        }
+        else if (this_data == PU "=" && this_context & CONTEXT("TEMPLATE_PARAM_KEY")) {
+            Tokenizer_handle_template_param_value(self);
+        }
+        else if (this_data == next_data && next_data == PU "}" &&
+                 this_context & CONTEXT("TEMPLATE")) {
+            Tokenizer_handle_template_end(self);
+        }
+        else if (this_data == PU "|" && this_context & CONTEXT("ARGUMENT_NAME")) {
+            Tokenizer_handle_argument_separator(self);
+        }
+        else if (this_data == next_data && next_data == PU "}" &&
+                 this_context & CONTEXT("ARGUMENT")) {
+            if (PyUnicode_AS_UNICODE(Tokenizer_read(self, 2)) == PU "}") {
+                return Tokenizer_handle_argument_end(self);
+            }
+            Tokenizer_write_text(self, this);
+        }
+        else if (this_data == next_data && next_data == PU "[") {
+            if (!(this_context & CONTEXT("WIKILINK_TITLE"))) {
+                Tokenizer_parse_wikilink(self);
+            }
+            else {
+                Tokenizer_write_text(self, this);
+            }
+        }
+        else if (this_data == PU "|" && this_context & CONTEXT("WIKILINK_TITLE")) {
+            Tokenizer_handle_wikilink_separator(self);
+        }
+        else if (this_data == next_data && next_data == PU "]" &&
+                 this_context & CONTEXT("WIKILINK")) {
+            return Tokenizer_handle_wikilink_end(self);
+        }
+        else if (this_data == PU "=" && !(self->global & CONTEXT("GL_HEADING"))) {
+            last_data = PyUnicode_AS_UNICODE(Tokenizer_read_backwards(self, 1));
+            if (last_data == PU "\n" || last_data == PU "") {
+                Tokenizer_parse_heading(self);
+            }
+            else {
+                Tokenizer_write_text(self, this);
+            }
+        }
+        else if (this_data == PU "=" && this_context & CONTEXT("HEADING")) {
+            return Tokenizer_handle_heading_end(self);
+        }
+        else if (this_data == PU "\n" && this_context & CONTEXT("HEADING")) {
+            Tokenizer_fail_route(self);
+        }
+        else if (this_data == PU "&") {
+            Tokenizer_parse_entity(self);
+        }
+        else if (this_data == PU "<" && next_data == PU "!") {
+            next_next_data = PyUnicode_AS_UNICODE(Tokenizer_read(self, 2));
+            if (next_next_data == PyUnicode_AS_UNICODE(Tokenizer_read(self, 3)) &&
+                    next_next_data == PU "-") {
+                Tokenizer_parse_comment(self);
+            }
+            else {
+                Tokenizer_write_text(self, this);
+            }
+        }
+        else {
+            Tokenizer_write_text(self, this);
+        }
+
         self->head++;
     }
 }

From 1ecb0e0d4485e71f9d49555d114df56ac9f0acff Mon Sep 17 00:00:00 2001
From: Ben Kurtovic <ben.kurtovic@verizon.net>
Date: Sun, 23 Sep 2012 17:48:57 -0400
Subject: [PATCH 06/47] Fix Tokenizer_verify_safe()'s prototype; add
 documentation.

---
 mwparserfromhell/parser/tokenizer.c | 57 +++++++++++++++++++++++++++++++++++--
 1 file changed, 55 insertions(+), 2 deletions(-)

diff --git a/mwparserfromhell/parser/tokenizer.c b/mwparserfromhell/parser/tokenizer.c
index 0d18473..ad013cb 100644
--- a/mwparserfromhell/parser/tokenizer.c
+++ b/mwparserfromhell/parser/tokenizer.c
@@ -377,109 +377,162 @@ Tokenizer_read_backwards(Tokenizer* self, Py_ssize_t delta)
     return PySequence_Fast_GET_ITEM(self->text, index);
 }
 
+/*
+    Parse a template or argument at the head of the wikicode string.
+*/
 static int
 Tokenizer_parse_template_or_argument(Tokenizer* self)
 {
 
 }
 
+/*
+    Parse a template at the head of the wikicode string.
+*/
 static int
 Tokenizer_parse_template(Tokenizer* self)
 {
 
 }
 
+/*
+    Parse an argument at the head of the wikicode string.
+*/
 static int
 Tokenizer_parse_argument(Tokenizer* self)
 {
 
 }
 
+/*
+    Verify that there are no unsafe characters in the current stack. The route
+    will be failed if the name contains any element of unsafes in it (not
+    merely at the beginning or end). This is used when parsing a template name
+    or parameter key, which cannot contain newlines.
+*/
 static int
-Tokenizer_verify_safe(Tokenizer* self)
+Tokenizer_verify_safe(Tokenizer* self, Py_UNICODE* unsafes[])
 {
 
 }
 
+/*
+    Handle a template parameter at the head of the string.
+*/
 static int
 Tokenizer_handle_template_param(Tokenizer* self)
 {
 
 }
 
+/*
+    Handle a template parameter's value at the head of the string.
+*/
 static int
 Tokenizer_handle_template_param_value(Tokenizer* self)
 {
 
 }
 
+/*
+    Handle the end of a template at the head of the string.
+*/
 static PyObject*
 Tokenizer_handle_template_end(Tokenizer* self)
 {
 
 }
 
+/*
+    Handle the separator between an argument's name and default.
+*/
 static int
 Tokenizer_handle_argument_separator(Tokenizer* self)
 {
 
 }
 
+/*
+    Handle the end of an argument at the head of the string.
+*/
 static PyObject*
 Tokenizer_handle_argument_end(Tokenizer* self)
 {
 
 }
 
+/*
+    Parse an internal wikilink at the head of the wikicode string.
+*/
 static int
 Tokenizer_parse_wikilink(Tokenizer* self)
 {
 
 }
 
+/*
+    Handle the separator between a wikilink's title and its text.
+*/
 static int
 Tokenizer_handle_wikilink_separator(Tokenizer* self)
 {
 
 }
 
+/*
+    Handle the end of a wikilink at the head of the string.
+*/
 static PyObject*
 Tokenizer_handle_wikilink_end(Tokenizer* self)
 {
 
 }
 
+/*
+    Parse a section heading at the head of the wikicode string.
+*/
 static int
 Tokenizer_parse_heading(Tokenizer* self)
 {
 
 }
 
+/*
+    Handle the end of a section heading at the head of the string.
+*/
 static PyObject*
 Tokenizer_handle_heading_end(Tokenizer* self)
 {
 
 }
 
+/*
+    Actually parse an HTML entity and ensure that it is valid.
+*/
 static int
 Tokenizer_really_parse_entity(Tokenizer* self)
 {
 
 }
 
+/*
+    Parse an HTML entity at the head of the wikicode string.
+*/
 static int
 Tokenizer_parse_entity(Tokenizer* self)
 {
 
 }
 
+/*
+    Parse an HTML comment at the head of the wikicode string.
+*/
 static int
 Tokenizer_parse_comment(Tokenizer* self)
 {
 
 }
 
-
 /*
     Parse the wikicode string, using context for when to stop.
 */

From 7fc45783b78772b5b689f3b724481997e23cd4ca Mon Sep 17 00:00:00 2001
From: Ben Kurtovic <ben.kurtovic@verizon.net>
Date: Sun, 23 Sep 2012 18:30:04 -0400
Subject: [PATCH 07/47] Add a header file; improve context handling.

---
 mwparserfromhell/parser/tokenizer.c | 131 +++---------------------
 mwparserfromhell/parser/tokenizer.h | 199 ++++++++++++++++++++++++++++++++++++
 2 files changed, 213 insertions(+), 117 deletions(-)
 create mode 100644 mwparserfromhell/parser/tokenizer.h

diff --git a/mwparserfromhell/parser/tokenizer.c b/mwparserfromhell/parser/tokenizer.c
index ad013cb..41713e2 100644
--- a/mwparserfromhell/parser/tokenizer.c
+++ b/mwparserfromhell/parser/tokenizer.c
@@ -21,45 +21,7 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 SOFTWARE.
 */
 
-#ifndef PY_SSIZE_T_CLEAN
-#define PY_SSIZE_T_CLEAN
-#endif
-
-#include <Python.h>
-#include <setjmp.h>
-#include <structmember.h>
-
-static PyObject* EMPTY;
-
-#define PU (Py_UNICODE*)
-static const Py_UNICODE* MARKERS[] = {PU"{", PU"}", PU"[", PU"]", PU"<", PU">",
-                                      PU"|", PU"=", PU"&", PU"#", PU"*", PU";",
-                                      PU":", PU"/", PU"-", PU"!", PU"\n", PU""};
-static const int NUM_MARKERS = 17;
-
-#define CONTEXT(name) PyInt_AsSsize_t((PyIntObject*) \
-                                          PyObject_GetAttrString(contexts, name))
-
-static jmp_buf exception_env;
-static const int BAD_ROUTE = 1;
-
-static PyObject* contexts;
-static PyObject* tokens;
-
-static PyMethodDef
-module_methods[] = {
-    {NULL}
-};
-
-typedef struct {
-    PyObject_HEAD
-    PyObject* text;        /* text to tokenize */
-    PyObject* stacks;      /* token stacks */
-    PyObject* topstack;    /* topmost stack */
-    Py_ssize_t head;       /* current position in text */
-    Py_ssize_t length;     /* length of text */
-    Py_ssize_t global;     /* global context */
-} Tokenizer;
+#include "tokenizer.h"
 
 static PyObject*
 Tokenizer_new(PyTypeObject* type, PyObject* args, PyObject* kwds)
@@ -104,11 +66,6 @@ Tokenizer_init(Tokenizer* self, PyObject* args, PyObject* kwds)
     return 0;
 }
 
-#define Tokenizer_STACK(self) PySequence_Fast_GET_ITEM(self->topstack, 0)
-#define Tokenizer_CONTEXT(self) PySequence_Fast_GET_ITEM(self->topstack, 1)
-#define Tokenizer_CONTEXT_VAL(self) PyInt_AsSsize_t((PyIntObject*) Tokenizer_CONTEXT(self))
-#define Tokenizer_TEXTBUFFER(self) PySequence_Fast_GET_ITEM(self->topstack, 2)
-
 static int
 Tokenizer_set_context(Tokenizer* self, Py_ssize_t value)
 {
@@ -539,9 +496,7 @@ Tokenizer_parse_comment(Tokenizer* self)
 static PyObject*
 Tokenizer_parse(Tokenizer* self, Py_ssize_t context)
 {
-    Py_ssize_t fail_contexts = (
-        CONTEXT("TEMPLATE") | CONTEXT("ARGUMENT") | CONTEXT("HEADING") |
-        CONTEXT("COMMENT"));
+    Py_ssize_t fail_contexts = LC_TEMPLATE | LC_ARGUMENT | LC_HEADING | LC_COMMENT;
 
     PyObject *this, *next;
     Py_UNICODE *this_data, *next_data, *next_next_data, *last_data;
@@ -580,7 +535,7 @@ Tokenizer_parse(Tokenizer* self, Py_ssize_t context)
         next = Tokenizer_read(self, 1);
         next_data = PyUnicode_AS_UNICODE(next);
 
-        if (this_context & CONTEXT("COMMENT")) {
+        if (this_context & LC_COMMENT) {
             if (this_data == next_data && next_data == PU "-") {
                 if (PyUnicode_AS_UNICODE(Tokenizer_read(self, 2)) == PU ">") {
                     return Tokenizer_pop(self);
@@ -591,42 +546,40 @@ Tokenizer_parse(Tokenizer* self, Py_ssize_t context)
         else if (this_data == next_data && next_data == PU "{") {
             Tokenizer_parse_template_or_argument(self);
         }
-        else if (this_data == PU "|" && this_context & CONTEXT("TEMPLATE")) {
+        else if (this_data == PU "|" && this_context & LC_TEMPLATE) {
             Tokenizer_handle_template_param(self);
         }
-        else if (this_data == PU "=" && this_context & CONTEXT("TEMPLATE_PARAM_KEY")) {
+        else if (this_data == PU "=" && this_context & LC_TEMPLATE_PARAM_KEY) {
             Tokenizer_handle_template_param_value(self);
         }
-        else if (this_data == next_data && next_data == PU "}" &&
-                 this_context & CONTEXT("TEMPLATE")) {
+        else if (this_data == next_data && next_data == PU "}" && this_context & LC_TEMPLATE) {
             Tokenizer_handle_template_end(self);
         }
-        else if (this_data == PU "|" && this_context & CONTEXT("ARGUMENT_NAME")) {
+        else if (this_data == PU "|" && this_context & LC_ARGUMENT_NAME) {
             Tokenizer_handle_argument_separator(self);
         }
-        else if (this_data == next_data && next_data == PU "}" &&
-                 this_context & CONTEXT("ARGUMENT")) {
+        else if (this_data == next_data && next_data == PU "}" && this_context & LC_ARGUMENT) {
             if (PyUnicode_AS_UNICODE(Tokenizer_read(self, 2)) == PU "}") {
                 return Tokenizer_handle_argument_end(self);
             }
             Tokenizer_write_text(self, this);
         }
         else if (this_data == next_data && next_data == PU "[") {
-            if (!(this_context & CONTEXT("WIKILINK_TITLE"))) {
+            if (!(this_context & LC_WIKILINK_TITLE)) {
                 Tokenizer_parse_wikilink(self);
             }
             else {
                 Tokenizer_write_text(self, this);
             }
         }
-        else if (this_data == PU "|" && this_context & CONTEXT("WIKILINK_TITLE")) {
+        else if (this_data == PU "|" && this_context & LC_WIKILINK_TITLE) {
             Tokenizer_handle_wikilink_separator(self);
         }
         else if (this_data == next_data && next_data == PU "]" &&
-                 this_context & CONTEXT("WIKILINK")) {
+                 this_context & LC_WIKILINK) {
             return Tokenizer_handle_wikilink_end(self);
         }
-        else if (this_data == PU "=" && !(self->global & CONTEXT("GL_HEADING"))) {
+        else if (this_data == PU "=" && !(self->global & GL_HEADING)) {
             last_data = PyUnicode_AS_UNICODE(Tokenizer_read_backwards(self, 1));
             if (last_data == PU "\n" || last_data == PU "") {
                 Tokenizer_parse_heading(self);
@@ -635,10 +588,10 @@ Tokenizer_parse(Tokenizer* self, Py_ssize_t context)
                 Tokenizer_write_text(self, this);
             }
         }
-        else if (this_data == PU "=" && this_context & CONTEXT("HEADING")) {
+        else if (this_data == PU "=" && this_context & LC_HEADING) {
             return Tokenizer_handle_heading_end(self);
         }
-        else if (this_data == PU "\n" && this_context & CONTEXT("HEADING")) {
+        else if (this_data == PU "\n" && this_context & LC_HEADING) {
             Tokenizer_fail_route(self);
         }
         else if (this_data == PU "&") {
@@ -700,61 +653,6 @@ Tokenizer_tokenize(Tokenizer* self, PyObject *args)
     return Tokenizer_parse(self, 0);
 }
 
-static PyMethodDef
-Tokenizer_methods[] = {
-    {"tokenize", (PyCFunction) Tokenizer_tokenize, METH_VARARGS,
-    "Build a list of tokens from a string of wikicode and return it."},
-    {NULL}
-};
-
-static PyMemberDef
-Tokenizer_members[] = {
-    {NULL}
-};
-
-static PyTypeObject
-TokenizerType = {
-    PyObject_HEAD_INIT(NULL)
-    0,                                                      /* ob_size */
-    "_tokenizer.CTokenizer",                                /* tp_name */
-    sizeof(Tokenizer),                                      /* tp_basicsize */
-    0,                                                      /* tp_itemsize */
-    (destructor) Tokenizer_dealloc,                         /* tp_dealloc */
-    0,                                                      /* tp_print */
-    0,                                                      /* tp_getattr */
-    0,                                                      /* tp_setattr */
-    0,                                                      /* tp_compare */
-    0,                                                      /* tp_repr */
-    0,                                                      /* tp_as_number */
-    0,                                                      /* tp_as_sequence */
-    0,                                                      /* tp_as_mapping */
-    0,                                                      /* tp_hash  */
-    0,                                                      /* tp_call */
-    0,                                                      /* tp_str */
-    0,                                                      /* tp_getattro */
-    0,                                                      /* tp_setattro */
-    0,                                                      /* tp_as_buffer */
-    Py_TPFLAGS_DEFAULT,                                     /* tp_flags */
-    "Creates a list of tokens from a string of wikicode.",  /* tp_doc */
-    0,                                                      /* tp_traverse */
-    0,                                                      /* tp_clear */
-    0,                                                      /* tp_richcompare */
-    0,                                                      /* tp_weaklistoffset */
-    0,                                                      /* tp_iter */
-    0,                                                      /* tp_iternext */
-    Tokenizer_methods,                                      /* tp_methods */
-    Tokenizer_members,                                      /* tp_members */
-    0,                                                      /* tp_getset */
-    0,                                                      /* tp_base */
-    0,                                                      /* tp_dict */
-    0,                                                      /* tp_descr_get */
-    0,                                                      /* tp_descr_set */
-    0,                                                      /* tp_dictoffset */
-    (initproc) Tokenizer_init,                              /* tp_init */
-    0,                                                      /* tp_alloc */
-    Tokenizer_new,                                          /* tp_new */
-};
-
 PyMODINIT_FUNC
 init_tokenizer(void)
 {
@@ -775,7 +673,6 @@ init_tokenizer(void)
     PyObject* locals = PyEval_GetLocals();
     PyObject* fromlist = PyList_New(0);
 
-    contexts = PyImport_ImportModuleLevel("contexts", globals, locals, fromlist, 1);
     tokens = PyImport_ImportModuleLevel("tokens", globals, locals, fromlist, 1);
     Py_DECREF(fromlist);
 }
diff --git a/mwparserfromhell/parser/tokenizer.h b/mwparserfromhell/parser/tokenizer.h
new file mode 100644
index 0000000..c504dd8
--- /dev/null
+++ b/mwparserfromhell/parser/tokenizer.h
@@ -0,0 +1,199 @@
+/*
+Tokenizer Header File for MWParserFromHell
+Copyright (C) 2012 Ben Kurtovic <ben.kurtovic@verizon.net>
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of
+this software and associated documentation files (the "Software"), to deal in
+the Software without restriction, including without limitation the rights to
+use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
+of the Software, and to permit persons to whom the Software is furnished to do
+so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+*/
+
+#ifndef PY_SSIZE_T_CLEAN
+#define PY_SSIZE_T_CLEAN
+#endif
+
+#include <Python.h>
+#include <setjmp.h>
+#include <structmember.h>
+
+#define PU (Py_UNICODE*)
+
+static const Py_UNICODE* MARKERS[] = {
+    PU "{", PU "}", PU "[", PU "]", PU "<", PU ">", PU "|", PU "=", PU "&",
+    PU "#", PU "*", PU ";", PU ":", PU "/", PU "-", PU "!", PU "\n", PU ""};
+static const int NUM_MARKERS = 17;
+
+static jmp_buf exception_env;
+static const int BAD_ROUTE = 1;
+
+static PyObject* EMPTY;
+static PyObject* tokens;
+
+
+/* Local contexts: */
+
+static const Py_ssize_t LC_TEMPLATE =             0x0007;
+static const Py_ssize_t LC_TEMPLATE_NAME =        0x0001;
+static const Py_ssize_t LC_TEMPLATE_PARAM_KEY =   0x0002;
+static const Py_ssize_t LC_TEMPLATE_PARAM_VALUE = 0x0004;
+
+static const Py_ssize_t LC_ARGUMENT =             0x0018;
+static const Py_ssize_t LC_ARGUMENT_NAME =        0x0008;
+static const Py_ssize_t LC_ARGUMENT_DEFAULT =     0x0010;
+
+static const Py_ssize_t LC_WIKILINK =             0x0060;
+static const Py_ssize_t LC_WIKILINK_TITLE =       0x0020;
+static const Py_ssize_t LC_WIKILINK_TEXT =        0x0040;
+
+static const Py_ssize_t LC_HEADING =              0x1f80;
+static const Py_ssize_t LC_HEADING_LEVEL_1 =      0x0080;
+static const Py_ssize_t LC_HEADING_LEVEL_2 =      0x0100;
+static const Py_ssize_t LC_HEADING_LEVEL_3 =      0x0200;
+static const Py_ssize_t LC_HEADING_LEVEL_4 =      0x0400;
+static const Py_ssize_t LC_HEADING_LEVEL_5 =      0x0800;
+static const Py_ssize_t LC_HEADING_LEVEL_6 =      0x1000;
+
+static const Py_ssize_t LC_COMMENT =              0x2000;
+
+
+/* Global contexts: */
+
+static const Py_ssize_t GL_HEADING = 0x1;
+
+
+/* Tokenizer object definition: */
+
+typedef struct {
+    PyObject_HEAD
+    PyObject* text;        /* text to tokenize */
+    PyObject* stacks;      /* token stacks */
+    PyObject* topstack;    /* topmost stack */
+    Py_ssize_t head;       /* current position in text */
+    Py_ssize_t length;     /* length of text */
+    Py_ssize_t global;     /* global context */
+} Tokenizer;
+
+
+/* Some macros for accessing Tokenizer data: */
+
+#define Tokenizer_STACK(self) PySequence_Fast_GET_ITEM(self->topstack, 0)
+#define Tokenizer_CONTEXT(self) PySequence_Fast_GET_ITEM(self->topstack, 1)
+#define Tokenizer_CONTEXT_VAL(self) PyInt_AsSsize_t(Tokenizer_CONTEXT(self))
+#define Tokenizer_TEXTBUFFER(self) PySequence_Fast_GET_ITEM(self->topstack, 2)
+
+
+/* Tokenizer function prototypes: */
+
+static PyObject* Tokenizer_new(PyTypeObject* type, PyObject* args, PyObject* kwds);
+static void Tokenizer_dealloc(Tokenizer* self);
+static int Tokenizer_init(Tokenizer* self, PyObject* args, PyObject* kwds);
+static int Tokenizer_set_context(Tokenizer* self, Py_ssize_t value);
+static int Tokenizer_set_textbuffer(Tokenizer* self, PyObject* value);
+static int Tokenizer_push(Tokenizer* self, Py_ssize_t context);
+static int Tokenizer_push_textbuffer(Tokenizer* self);
+static int Tokenizer_delete_top_of_stack(Tokenizer* self);
+static PyObject* Tokenizer_pop(Tokenizer* self);
+static PyObject* Tokenizer_pop_keeping_context(Tokenizer* self);
+static void Tokenizer_fail_route(Tokenizer* self);
+static int Tokenizer_write(Tokenizer* self, PyObject* token);
+static int Tokenizer_write_first(Tokenizer* self, PyObject* token);
+static int Tokenizer_write_text(Tokenizer* self, PyObject* text);
+static int Tokenizer_write_all(Tokenizer* self, PyObject* tokenlist);
+static int Tokenizer_write_text_then_stack(Tokenizer* self, PyObject* text);
+static PyObject* Tokenizer_read(Tokenizer* self, Py_ssize_t delta);
+static PyObject* Tokenizer_read_backwards(Tokenizer* self, Py_ssize_t delta);
+static int Tokenizer_parse_template_or_argument(Tokenizer* self);
+static int Tokenizer_parse_template(Tokenizer* self);
+static int Tokenizer_parse_argument(Tokenizer* self);
+static int Tokenizer_verify_safe(Tokenizer* self, Py_UNICODE* unsafes[]);
+static int Tokenizer_handle_template_param(Tokenizer* self);
+static int Tokenizer_handle_template_param_value(Tokenizer* self);
+static PyObject* Tokenizer_handle_template_end(Tokenizer* self);
+static int Tokenizer_handle_argument_separator(Tokenizer* self);
+static PyObject* Tokenizer_handle_argument_end(Tokenizer* self);
+static int Tokenizer_parse_wikilink(Tokenizer* self);
+static int Tokenizer_handle_wikilink_separator(Tokenizer* self);
+static PyObject* Tokenizer_handle_wikilink_end(Tokenizer* self);
+static int Tokenizer_parse_heading(Tokenizer* self);
+static PyObject* Tokenizer_handle_heading_end(Tokenizer* self);
+static int Tokenizer_really_parse_entity(Tokenizer* self);
+static int Tokenizer_parse_entity(Tokenizer* self);
+static int Tokenizer_parse_comment(Tokenizer* self);
+static PyObject* Tokenizer_parse(Tokenizer* self, Py_ssize_t context);
+static PyObject* Tokenizer_tokenize(Tokenizer* self, PyObject *args);
+
+
+/* More structs for creating the Tokenizer type: */
+
+static PyMethodDef
+Tokenizer_methods[] = {
+    {"tokenize", (PyCFunction) Tokenizer_tokenize, METH_VARARGS,
+    "Build a list of tokens from a string of wikicode and return it."},
+    {NULL}
+};
+
+static PyMemberDef
+Tokenizer_members[] = {
+    {NULL}
+};
+
+static PyMethodDef
+module_methods[] = {
+    {NULL}
+};
+
+static PyTypeObject
+TokenizerType = {
+    PyObject_HEAD_INIT(NULL)
+    0,                                                      /* ob_size */
+    "_tokenizer.CTokenizer",                                /* tp_name */
+    sizeof(Tokenizer),                                      /* tp_basicsize */
+    0,                                                      /* tp_itemsize */
+    (destructor) Tokenizer_dealloc,                         /* tp_dealloc */
+    0,                                                      /* tp_print */
+    0,                                                      /* tp_getattr */
+    0,                                                      /* tp_setattr */
+    0,                                                      /* tp_compare */
+    0,                                                      /* tp_repr */
+    0,                                                      /* tp_as_number */
+    0,                                                      /* tp_as_sequence */
+    0,                                                      /* tp_as_mapping */
+    0,                                                      /* tp_hash  */
+    0,                                                      /* tp_call */
+    0,                                                      /* tp_str */
+    0,                                                      /* tp_getattro */
+    0,                                                      /* tp_setattro */
+    0,                                                      /* tp_as_buffer */
+    Py_TPFLAGS_DEFAULT,                                     /* tp_flags */
+    "Creates a list of tokens from a string of wikicode.",  /* tp_doc */
+    0,                                                      /* tp_traverse */
+    0,                                                      /* tp_clear */
+    0,                                                      /* tp_richcompare */
+    0,                                                      /* tp_weaklistoffset */
+    0,                                                      /* tp_iter */
+    0,                                                      /* tp_iternext */
+    Tokenizer_methods,                                      /* tp_methods */
+    Tokenizer_members,                                      /* tp_members */
+    0,                                                      /* tp_getset */
+    0,                                                      /* tp_base */
+    0,                                                      /* tp_dict */
+    0,                                                      /* tp_descr_get */
+    0,                                                      /* tp_descr_set */
+    0,                                                      /* tp_dictoffset */
+    (initproc) Tokenizer_init,                              /* tp_init */
+    0,                                                      /* tp_alloc */
+    Tokenizer_new,                                          /* tp_new */
+};

From 6edc24037eff257e82cfe3d86d3d2b253d2b5fa5 Mon Sep 17 00:00:00 2001
From: Ben Kurtovic <ben.kurtovic@verizon.net>
Date: Sun, 23 Sep 2012 19:14:23 -0400
Subject: [PATCH 08/47] Implement Tokenizer_parse_template_or_argument().

---
 mwparserfromhell/parser/tokenizer.c  | 118 ++++++++++++++++++++++++++++-------
 mwparserfromhell/parser/tokenizer.h  |   3 +-
 mwparserfromhell/parser/tokenizer.py |   2 +-
 3 files changed, 98 insertions(+), 25 deletions(-)

diff --git a/mwparserfromhell/parser/tokenizer.c b/mwparserfromhell/parser/tokenizer.c
index 41713e2..4877773 100644
--- a/mwparserfromhell/parser/tokenizer.c
+++ b/mwparserfromhell/parser/tokenizer.c
@@ -313,9 +313,8 @@ Tokenizer_read(Tokenizer* self, Py_ssize_t delta)
 {
     Py_ssize_t index = self->head + delta;
 
-    if (index >= self->length) {
+    if (index >= self->length)
         return EMPTY;
-    }
 
     return PySequence_Fast_GET_ITEM(self->text, index);
 }
@@ -326,9 +325,8 @@ Tokenizer_read(Tokenizer* self, Py_ssize_t delta)
 static PyObject*
 Tokenizer_read_backwards(Tokenizer* self, Py_ssize_t delta)
 {
-    if (delta > self->head) {
+    if (delta > self->head)
         return EMPTY;
-    }
 
     Py_ssize_t index = self->head - delta;
     return PySequence_Fast_GET_ITEM(self->text, index);
@@ -340,7 +338,84 @@ Tokenizer_read_backwards(Tokenizer* self, Py_ssize_t delta)
 static int
 Tokenizer_parse_template_or_argument(Tokenizer* self)
 {
+    self->head += 2;
+    unsigned int braces = 2, i;
+
+    while (Tokenizer_READ(self, 0) == PU "{") {
+        self->head++;
+        braces++;
+    }
+    Tokenizer_push(self, 0);
+
+    while (braces) {
+        if (braces == 1) {
+            PyObject* text = PyUnicode_FromString("{");
+
+            if (Tokenizer_write_text_then_stack(self, text)) {
+                Py_XDECREF(text);
+                return -1;
+            }
+
+            Py_XDECREF(text);
+            return 0;
+        }
+
+        if (braces == 2) {
+            if (setjmp(exception_env) == BAD_ROUTE) {
+                PyObject* text = PyUnicode_FromString("{{");
+
+                if (Tokenizer_write_text_then_stack(self, text)) {
+                    Py_XDECREF(text);
+                    return -1;
+                }
+
+                Py_XDECREF(text);
+                return 0;
+            } else {
+                Tokenizer_parse_template(self);
+            }
+            break;
+        }
+
+        if (setjmp(exception_env) == BAD_ROUTE) {
+            if (setjmp(exception_env) == BAD_ROUTE) {
+                char bracestr[braces];
+                for (i = 0; i < braces; i++) {
+                        bracestr[i] = *"{";
+                }
+                PyObject* text = PyUnicode_FromString(bracestr);
+
+                if (Tokenizer_write_text_then_stack(self, text)) {
+                    Py_XDECREF(text);
+                    return -1;
+                }
+
+                Py_XDECREF(text);
+                return 0;
+            }
+            else {
+                Tokenizer_parse_template(self);
+                braces -= 2;
+            }
+        }
+        else {
+            Tokenizer_parse_argument(self);
+            braces -= 3;
+        }
+
+        if (braces) {
+            self->head++;
+        }
+    }
 
+    PyObject* tokenlist = Tokenizer_pop(self);
+    if (Tokenizer_write_all(self, tokenlist)) {
+        Py_DECREF(tokenlist);
+        return -1;
+    }
+
+    Py_DECREF(tokenlist);
+    return 0;
 }
 
 /*
@@ -498,8 +573,8 @@ Tokenizer_parse(Tokenizer* self, Py_ssize_t context)
 {
     Py_ssize_t fail_contexts = LC_TEMPLATE | LC_ARGUMENT | LC_HEADING | LC_COMMENT;
 
-    PyObject *this, *next;
-    Py_UNICODE *this_data, *next_data, *next_next_data, *last_data;
+    PyObject *this;
+    Py_UNICODE *this_data, *next, *next_next, *last;
     Py_ssize_t this_context;
     int is_marker, i;
 
@@ -532,18 +607,17 @@ Tokenizer_parse(Tokenizer* self, Py_ssize_t context)
             return Tokenizer_pop(self);
         }
 
-        next = Tokenizer_read(self, 1);
-        next_data = PyUnicode_AS_UNICODE(next);
+        next = Tokenizer_READ(self, 1);
 
         if (this_context & LC_COMMENT) {
-            if (this_data == next_data && next_data == PU "-") {
-                if (PyUnicode_AS_UNICODE(Tokenizer_read(self, 2)) == PU ">") {
+            if (this_data == next && next == PU "-") {
+                if (Tokenizer_READ(self, 2) == PU ">") {
                     return Tokenizer_pop(self);
                 }
             }
             Tokenizer_write_text(self, this);
         }
-        else if (this_data == next_data && next_data == PU "{") {
+        else if (this_data == next && next == PU "{") {
             Tokenizer_parse_template_or_argument(self);
         }
         else if (this_data == PU "|" && this_context & LC_TEMPLATE) {
@@ -552,19 +626,19 @@ Tokenizer_parse(Tokenizer* self, Py_ssize_t context)
         else if (this_data == PU "=" && this_context & LC_TEMPLATE_PARAM_KEY) {
             Tokenizer_handle_template_param_value(self);
         }
-        else if (this_data == next_data && next_data == PU "}" && this_context & LC_TEMPLATE) {
+        else if (this_data == next && next == PU "}" && this_context & LC_TEMPLATE) {
             Tokenizer_handle_template_end(self);
         }
         else if (this_data == PU "|" && this_context & LC_ARGUMENT_NAME) {
             Tokenizer_handle_argument_separator(self);
         }
-        else if (this_data == next_data && next_data == PU "}" && this_context & LC_ARGUMENT) {
-            if (PyUnicode_AS_UNICODE(Tokenizer_read(self, 2)) == PU "}") {
+        else if (this_data == next && next == PU "}" && this_context & LC_ARGUMENT) {
+            if (Tokenizer_READ(self, 2) == PU "}") {
                 return Tokenizer_handle_argument_end(self);
             }
             Tokenizer_write_text(self, this);
         }
-        else if (this_data == next_data && next_data == PU "[") {
+        else if (this_data == next && next == PU "[") {
             if (!(this_context & LC_WIKILINK_TITLE)) {
                 Tokenizer_parse_wikilink(self);
             }
@@ -575,13 +649,12 @@ Tokenizer_parse(Tokenizer* self, Py_ssize_t context)
         else if (this_data == PU "|" && this_context & LC_WIKILINK_TITLE) {
             Tokenizer_handle_wikilink_separator(self);
         }
-        else if (this_data == next_data && next_data == PU "]" &&
-                 this_context & LC_WIKILINK) {
+        else if (this_data == next && next == PU "]" && this_context & LC_WIKILINK) {
             return Tokenizer_handle_wikilink_end(self);
         }
         else if (this_data == PU "=" && !(self->global & GL_HEADING)) {
-            last_data = PyUnicode_AS_UNICODE(Tokenizer_read_backwards(self, 1));
-            if (last_data == PU "\n" || last_data == PU "") {
+            last = PyUnicode_AS_UNICODE(Tokenizer_read_backwards(self, 1));
+            if (last == PU "\n" || last == PU "") {
                 Tokenizer_parse_heading(self);
             }
             else {
@@ -597,10 +670,9 @@ Tokenizer_parse(Tokenizer* self, Py_ssize_t context)
         else if (this_data == PU "&") {
             Tokenizer_parse_entity(self);
         }
-        else if (this_data == PU "<" && next_data == PU "!") {
-            next_next_data = PyUnicode_AS_UNICODE(Tokenizer_read(self, 2));
-            if (next_next_data == PyUnicode_AS_UNICODE(Tokenizer_read(self, 3)) &&
-                    next_next_data == PU "-") {
+        else if (this_data == PU "<" && next == PU "!") {
+            next_next = Tokenizer_READ(self, 2);
+            if (next_next == Tokenizer_READ(self, 3) && next_next == PU "-") {
                 Tokenizer_parse_comment(self);
             }
             else {
diff --git a/mwparserfromhell/parser/tokenizer.h b/mwparserfromhell/parser/tokenizer.h
index c504dd8..3f7dfdf 100644
--- a/mwparserfromhell/parser/tokenizer.h
+++ b/mwparserfromhell/parser/tokenizer.h
@@ -87,12 +87,13 @@ typedef struct {
 } Tokenizer;
 
 
-/* Some macros for accessing Tokenizer data: */
+/* Macros for accessing Tokenizer data: */
 
 #define Tokenizer_STACK(self) PySequence_Fast_GET_ITEM(self->topstack, 0)
 #define Tokenizer_CONTEXT(self) PySequence_Fast_GET_ITEM(self->topstack, 1)
 #define Tokenizer_CONTEXT_VAL(self) PyInt_AsSsize_t(Tokenizer_CONTEXT(self))
 #define Tokenizer_TEXTBUFFER(self) PySequence_Fast_GET_ITEM(self->topstack, 2)
+#define Tokenizer_READ(self, num) PyUnicode_AS_UNICODE(Tokenizer_read(self, num))
 
 
 /* Tokenizer function prototypes: */
diff --git a/mwparserfromhell/parser/tokenizer.py b/mwparserfromhell/parser/tokenizer.py
index ca645b0..364455d 100644
--- a/mwparserfromhell/parser/tokenizer.py
+++ b/mwparserfromhell/parser/tokenizer.py
@@ -162,8 +162,8 @@ class Tokenizer(object):
         self._head += 2
         braces = 2
         while self._read() == "{":
-            braces += 1
             self._head += 1
+            braces += 1
         self._push()
 
         while braces:

From 0d720a7ef13e7e377dd0d47c88d1e68c717e8b2c Mon Sep 17 00:00:00 2001
From: Ben Kurtovic <ben.kurtovic@verizon.net>
Date: Sun, 23 Sep 2012 20:35:49 -0400
Subject: [PATCH 09/47] Implement Tokenizer_parse_template(); NOARGS and
 NOKWARGS.

---
 mwparserfromhell/parser/tokenizer.c | 50 +++++++++++++++++++++++++++----------
 mwparserfromhell/parser/tokenizer.h |  2 ++
 2 files changed, 39 insertions(+), 13 deletions(-)

diff --git a/mwparserfromhell/parser/tokenizer.c b/mwparserfromhell/parser/tokenizer.c
index 4877773..d9b953b 100644
--- a/mwparserfromhell/parser/tokenizer.c
+++ b/mwparserfromhell/parser/tokenizer.c
@@ -111,29 +111,22 @@ Tokenizer_push_textbuffer(Tokenizer* self)
         PyObject* text = PyUnicode_Join(EMPTY, Tokenizer_TEXTBUFFER(self));
         if (!text) return -1;
 
-        PyObject* klass = PyObject_GetAttrString(tokens, "Text");
-        if (!klass) return -1;
-        PyObject* args = PyTuple_New(0);
-        if (!args) return -1;
+        PyObject* class = PyObject_GetAttrString(tokens, "Text");
+        if (!class) return -1;
         PyObject* kwargs = PyDict_New();
         if (!kwargs) return -1;
         PyDict_SetItemString(kwargs, "text", text);
         Py_DECREF(text);
 
-        PyObject* token = PyInstance_New(klass, args, kwargs);
+        PyObject* token = PyInstance_New(class, NOARGS, kwargs);
+        Py_DECREF(class);
+        Py_DECREF(kwargs);
         if (!token) {
-            Py_DECREF(klass);
-            Py_DECREF(args);
-            Py_DECREF(kwargs);
             return -1;
         }
 
-        Py_DECREF(klass);
-        Py_DECREF(args);
-        Py_DECREF(kwargs);
-
         if (PyList_Append(Tokenizer_STACK(self), token)) {
-            Py_XDECREF(token);
+            Py_DECREF(token);
             return -1;
         }
 
@@ -424,7 +417,36 @@ Tokenizer_parse_template_or_argument(Tokenizer* self)
 static int
 Tokenizer_parse_template(Tokenizer* self)
 {
+    Py_ssize_t reset = self->head;
+    if (setjmp(exception_env) == BAD_ROUTE) {
+        self->head = reset;
+        longjmp(exception_env, BAD_ROUTE);
+    }
+    else {
+        PyObject* template = Tokenizer_parse(self, LC_TEMPLATE_NAME);
+        if (!template) return -1;
+
+        PyObject* class = PyObject_GetAttrString(tokens, "TemplateOpen");
+        if (!class) return -1;
+        PyObject* token = PyInstance_New(class, NOARGS, NOKWARGS);
+        Py_DECREF(class);
+        if (!token) return -1;
 
+        Tokenizer_write_first(self, token);
+        Py_DECREF(token);
+
+        Tokenizer_write_all(self, template);
+        Py_DECREF(template);
+
+        class = PyObject_GetAttrString(tokens, "TemplateClose");
+        if (!class) return -1;
+        token = PyInstance_New(class, NOARGS, NOKWARGS);
+        Py_DECREF(class);
+        if (!token) return -1;
+
+        Tokenizer_write(self, token);
+        Py_DECREF(token);
+    }
 }
 
 /*
@@ -740,6 +762,8 @@ init_tokenizer(void)
     PyModule_AddObject(module, "CTokenizer", (PyObject*) &TokenizerType);
 
     EMPTY = PyUnicode_FromString("");
+    NOARGS = PyTuple_New(0);
+    NOKWARGS = PyDict_New();
 
     PyObject* globals = PyEval_GetGlobals();
     PyObject* locals = PyEval_GetLocals();
diff --git a/mwparserfromhell/parser/tokenizer.h b/mwparserfromhell/parser/tokenizer.h
index 3f7dfdf..d6c97c8 100644
--- a/mwparserfromhell/parser/tokenizer.h
+++ b/mwparserfromhell/parser/tokenizer.h
@@ -40,6 +40,8 @@ static jmp_buf exception_env;
 static const int BAD_ROUTE = 1;
 
 static PyObject* EMPTY;
+static PyObject* NOARGS;
+static PyObject* NOKWARGS;
 static PyObject* tokens;
 
 

From 849016f73488eb4eee51fb8c0b16f49231e2dc3b Mon Sep 17 00:00:00 2001
From: Ben Kurtovic <ben.kurtovic@verizon.net>
Date: Sun, 23 Sep 2012 21:27:08 -0400
Subject: [PATCH 10/47] Implement Tokenizer_verify_safe() and some others.

---
 mwparserfromhell/parser/tokenizer.c | 164 +++++++++++++++++++++++++++++++++---
 mwparserfromhell/parser/tokenizer.h |   2 +-
 2 files changed, 155 insertions(+), 11 deletions(-)

diff --git a/mwparserfromhell/parser/tokenizer.c b/mwparserfromhell/parser/tokenizer.c
index d9b953b..3d3b95f 100644
--- a/mwparserfromhell/parser/tokenizer.c
+++ b/mwparserfromhell/parser/tokenizer.c
@@ -121,9 +121,7 @@ Tokenizer_push_textbuffer(Tokenizer* self)
         PyObject* token = PyInstance_New(class, NOARGS, kwargs);
         Py_DECREF(class);
         Py_DECREF(kwargs);
-        if (!token) {
-            return -1;
-        }
+        if (!token) return -1;
 
         if (PyList_Append(Tokenizer_STACK(self), token)) {
             Py_DECREF(token);
@@ -417,25 +415,34 @@ Tokenizer_parse_template_or_argument(Tokenizer* self)
 static int
 Tokenizer_parse_template(Tokenizer* self)
 {
+    PyObject *template, *class, *token;
     Py_ssize_t reset = self->head;
+
     if (setjmp(exception_env) == BAD_ROUTE) {
         self->head = reset;
         longjmp(exception_env, BAD_ROUTE);
     }
+
     else {
-        PyObject* template = Tokenizer_parse(self, LC_TEMPLATE_NAME);
+        template = Tokenizer_parse(self, LC_TEMPLATE_NAME);
         if (!template) return -1;
 
-        PyObject* class = PyObject_GetAttrString(tokens, "TemplateOpen");
+        class = PyObject_GetAttrString(tokens, "TemplateOpen");
         if (!class) return -1;
-        PyObject* token = PyInstance_New(class, NOARGS, NOKWARGS);
+        token = PyInstance_New(class, NOARGS, NOKWARGS);
         Py_DECREF(class);
         if (!token) return -1;
 
-        Tokenizer_write_first(self, token);
+        if (Tokenizer_write_first(self, token)) {
+            Py_DECREF(token);
+            return -1;
+        }
         Py_DECREF(token);
 
-        Tokenizer_write_all(self, template);
+        if (Tokenizer_write_all(self, template)) {
+            Py_DECREF(template);
+            return -1;
+        }
         Py_DECREF(template);
 
         class = PyObject_GetAttrString(tokens, "TemplateClose");
@@ -444,9 +451,14 @@ Tokenizer_parse_template(Tokenizer* self)
         Py_DECREF(class);
         if (!token) return -1;
 
-        Tokenizer_write(self, token);
+        if (Tokenizer_write(self, token)) {
+            Py_DECREF(token);
+            return -1;
+        }
         Py_DECREF(token);
     }
+
+    return 0;
 }
 
 /*
@@ -455,7 +467,50 @@ Tokenizer_parse_template(Tokenizer* self)
 static int
 Tokenizer_parse_argument(Tokenizer* self)
 {
+    PyObject *argument, *class, *token;
+    Py_ssize_t reset = self->head;
+
+    if (setjmp(exception_env) == BAD_ROUTE) {
+        self->head = reset;
+        longjmp(exception_env, BAD_ROUTE);
+    }
+
+    else {
+        argument = Tokenizer_parse(self, LC_ARGUMENT_NAME);
+        if (!argument) return -1;
+
+        class = PyObject_GetAttrString(tokens, "ArgumentOpen");
+        if (!class) return -1;
+        token = PyInstance_New(class, NOARGS, NOKWARGS);
+        Py_DECREF(class);
+        if (!token) return -1;
+
+        if (Tokenizer_write_first(self, token)) {
+            Py_DECREF(token);
+            return -1;
+        }
+        Py_DECREF(token);
+
+        if (Tokenizer_write_all(self, argument)) {
+            Py_DECREF(argument);
+            return -1;
+        }
+        Py_DECREF(argument);
+
+        class = PyObject_GetAttrString(tokens, "ArgumentClose");
+        if (!class) return -1;
+        token = PyInstance_New(class, NOARGS, NOKWARGS);
+        Py_DECREF(class);
+        if (!token) return -1;
 
+        if (Tokenizer_write(self, token)) {
+            Py_DECREF(token);
+            return -1;
+        }
+        Py_DECREF(token);
+    }
+
+    return 0;
 }
 
 /*
@@ -465,9 +520,98 @@ Tokenizer_parse_argument(Tokenizer* self)
     or parameter key, which cannot contain newlines.
 */
 static int
-Tokenizer_verify_safe(Tokenizer* self, Py_UNICODE* unsafes[])
+Tokenizer_verify_safe(Tokenizer* self, const char* unsafes[])
 {
+    if (Tokenizer_push_textbuffer(self))
+        return -1;
 
+    PyObject* stack = Tokenizer_STACK(self);
+    if (stack) {
+        PyObject* textlist = PyList_New(0);
+        if (!textlist) return -1;
+
+        PyObject* class = PyObject_GetAttrString(tokens, "Text");
+        if (!class) {
+            Py_DECREF(textlist);
+            return -1;
+        }
+
+        int i;
+        Py_ssize_t length = PySequence_Fast_GET_SIZE(stack);
+        PyObject *token, *textdata;
+
+        for (i = 0; i < length; i++) {
+            token = PySequence_Fast_GET_ITEM(stack, i);
+            switch (PyObject_IsInstance(token, class)) {
+                case -1:
+                    Py_DECREF(textlist);
+                    Py_DECREF(class);
+                    return -1;
+                case 0:
+                    break;
+                case 1:
+                    textdata = PyObject_GetAttrString(token, "text");
+                    if (!textdata) {
+                        Py_DECREF(textlist);
+                        Py_DECREF(class);
+                        return -1;
+                    }
+                    if (PyList_Append(textlist, textdata)) {
+                        Py_DECREF(textlist);
+                        Py_DECREF(class);
+                        Py_DECREF(textdata);
+                        return -1;
+                    }
+                    Py_DECREF(textdata);
+            }
+        }
+        Py_DECREF(class);
+
+        PyObject* text = PyUnicode_Join(EMPTY, textlist);
+        if (!text) {
+            Py_DECREF(textlist);
+            return -1;
+        }
+        Py_DECREF(textlist);
+
+        PyObject* stripped = PyObject_CallMethod(text, "strip", NULL);
+        if (!stripped) {
+            Py_DECREF(text);
+            return -1;
+        }
+        Py_DECREF(text);
+
+        const char* unsafe_char;
+        PyObject* unsafe;
+        i = 0;
+        while (1) {
+            unsafe_char = unsafes[i];
+            if (!unsafe_char) break;
+
+            unsafe = PyUnicode_FromString(unsafe_char);
+
+            if (!unsafe) {
+                Py_DECREF(stripped);
+                return -1;
+            }
+
+            switch (PyUnicode_Contains(stripped, unsafe)) {
+                case -1:
+                    Py_DECREF(stripped);
+                    Py_DECREF(unsafe);
+                    return -1;
+                case 0:
+                    break;
+                case 1:
+                    Py_DECREF(stripped);
+                    Py_DECREF(unsafe);
+                    Tokenizer_fail_route(self);
+            }
+            i++;
+        }
+    }
+
+    return 0;
 }
 
 /*
diff --git a/mwparserfromhell/parser/tokenizer.h b/mwparserfromhell/parser/tokenizer.h
index d6c97c8..951e238 100644
--- a/mwparserfromhell/parser/tokenizer.h
+++ b/mwparserfromhell/parser/tokenizer.h
@@ -121,7 +121,7 @@ static PyObject* Tokenizer_read_backwards(Tokenizer* self, Py_ssize_t delta);
 static int Tokenizer_parse_template_or_argument(Tokenizer* self);
 static int Tokenizer_parse_template(Tokenizer* self);
 static int Tokenizer_parse_argument(Tokenizer* self);
-static int Tokenizer_verify_safe(Tokenizer* self, Py_UNICODE* unsafes[]);
+static int Tokenizer_verify_safe(Tokenizer* self, const char* unsafes[]);
 static int Tokenizer_handle_template_param(Tokenizer* self);
 static int Tokenizer_handle_template_param_value(Tokenizer* self);
 static PyObject* Tokenizer_handle_template_end(Tokenizer* self);

From 17af353fb652e01eb61584c0f5c6248edd17e9be Mon Sep 17 00:00:00 2001
From: Ben Kurtovic <ben.kurtovic@verizon.net>
Date: Mon, 24 Sep 2012 21:18:44 -0400
Subject: [PATCH 11/47] Implement Tokenizer_handle_template_param().

---
 mwparserfromhell/parser/tokenizer.c | 62 +++++++++++++++++++++++++++++++------
 1 file changed, 53 insertions(+), 9 deletions(-)

diff --git a/mwparserfromhell/parser/tokenizer.c b/mwparserfromhell/parser/tokenizer.c
index 3d3b95f..3ab2437 100644
--- a/mwparserfromhell/parser/tokenizer.c
+++ b/mwparserfromhell/parser/tokenizer.c
@@ -206,7 +206,8 @@ Tokenizer_pop_keeping_context(Tokenizer* self)
 static void
 Tokenizer_fail_route(Tokenizer* self)
 {
-    Tokenizer_pop(self);
+    PyObject* stack = Tokenizer_pop(self);
+    Py_XDECREF(stack);
     longjmp(exception_env, BAD_ROUTE);
 }
 
@@ -400,6 +401,7 @@ Tokenizer_parse_template_or_argument(Tokenizer* self)
     }
 
     PyObject* tokenlist = Tokenizer_pop(self);
+    if (!tokenlist) return -1;
     if (Tokenizer_write_all(self, tokenlist)) {
         Py_DECREF(tokenlist);
         return -1;
@@ -543,10 +545,6 @@ Tokenizer_verify_safe(Tokenizer* self, const char* unsafes[])
         for (i = 0; i < length; i++) {
             token = PySequence_Fast_GET_ITEM(stack, i);
             switch (PyObject_IsInstance(token, class)) {
-                case -1:
-                    Py_DECREF(textlist);
-                    Py_DECREF(class);
-                    return -1;
                 case 0:
                     break;
                 case 1:
@@ -563,6 +561,11 @@ Tokenizer_verify_safe(Tokenizer* self, const char* unsafes[])
                         return -1;
                     }
                     Py_DECREF(textdata);
+                    break;
+                case -1:
+                    Py_DECREF(textlist);
+                    Py_DECREF(class);
+                    return -1;
             }
         }
         Py_DECREF(class);
@@ -596,16 +599,17 @@ Tokenizer_verify_safe(Tokenizer* self, const char* unsafes[])
             }
 
             switch (PyUnicode_Contains(stripped, unsafe)) {
-                case -1:
-                    Py_DECREF(stripped);
-                    Py_DECREF(unsafe);
-                    return -1;
                 case 0:
                     break;
                 case 1:
                     Py_DECREF(stripped);
                     Py_DECREF(unsafe);
                     Tokenizer_fail_route(self);
+                    break;
+                case -1:
+                    Py_DECREF(stripped);
+                    Py_DECREF(unsafe);
+                    return -1;
             }
             i++;
         }
@@ -620,7 +624,47 @@ Tokenizer_verify_safe(Tokenizer* self, const char* unsafes[])
 static int
 Tokenizer_handle_template_param(Tokenizer* self)
 {
+    Py_ssize_t context = Tokenizer_CONTEXT_VAL(self);
 
+    if (context & LC_TEMPLATE_NAME) {
+        if (Tokenizer_verify_safe(self, {"\n", "{", "}", "[", "]"}))
+            return -1;
+        if (Tokenizer_set_context(self, context ^ LC_TEMPLATE_NAME))
+            return -1;
+    }
+    else if (context & LC_TEMPLATE_PARAM_VALUE) {
+        if (Tokenizer_set_context(self, context ^ LC_TEMPLATE_PARAM_VALUE))
+            return -1;
+    }
+
+    if (context & LC_TEMPLATE_PARAM_KEY) {
+        PyObject* stack = Tokenizer_pop_keeping_context(self);
+        if (!stack) return -1;
+        if (Tokenizer_write_all(stack)) {
+            Py_DECREF(stack);
+            return -1;
+        }
+        Py_DECREF(stack);
+    }
+    else {
+        if (Tokenizer_set_context(self, context | LC_TEMPLATE_PARAM_KEY))
+            return -1;
+    }
+
+    class = PyObject_GetAttrString(tokens, "TemplateParamSeparator");
+    if (!class) return -1;
+    token = PyInstance_New(class, NOARGS, NOKWARGS);
+    Py_DECREF(class);
+    if (!token) return -1;
+
+    if (Tokenizer_write(self, token)) {
+        Py_DECREF(token);
+        return -1;
+    }
+    Py_DECREF(token);
+
+    Tokenizer_push(self, Tokenizer_CONTEXT_VAL(self));
+    return 0;
 }
 
 /*

From 41535992a1a3488724435f4482642c6aa40bca45 Mon Sep 17 00:00:00 2001
From: Ben Kurtovic <ben.kurtovic@verizon.net>
Date: Tue, 25 Sep 2012 17:09:27 -0400
Subject: [PATCH 12/47] Implement Tokenizer_handle_template_param_value().

---
 mwparserfromhell/parser/tokenizer.c  | 45 ++++++++++++++++++++++++++++++++----
 mwparserfromhell/parser/tokenizer.py | 17 ++++++--------
 2 files changed, 48 insertions(+), 14 deletions(-)

diff --git a/mwparserfromhell/parser/tokenizer.c b/mwparserfromhell/parser/tokenizer.c
index 3ab2437..e7699fd 100644
--- a/mwparserfromhell/parser/tokenizer.c
+++ b/mwparserfromhell/parser/tokenizer.c
@@ -627,7 +627,8 @@ Tokenizer_handle_template_param(Tokenizer* self)
     Py_ssize_t context = Tokenizer_CONTEXT_VAL(self);
 
     if (context & LC_TEMPLATE_NAME) {
-        if (Tokenizer_verify_safe(self, {"\n", "{", "}", "[", "]"}))
+        const char* unsafes[] = {"\n", "{", "}", "[", "]"};
+        if (Tokenizer_verify_safe(self, unsafes))
             return -1;
         if (Tokenizer_set_context(self, context ^ LC_TEMPLATE_NAME))
             return -1;
@@ -640,7 +641,7 @@ Tokenizer_handle_template_param(Tokenizer* self)
     if (context & LC_TEMPLATE_PARAM_KEY) {
         PyObject* stack = Tokenizer_pop_keeping_context(self);
         if (!stack) return -1;
-        if (Tokenizer_write_all(stack)) {
+        if (Tokenizer_write_all(self, stack)) {
             Py_DECREF(stack);
             return -1;
         }
@@ -651,9 +652,9 @@ Tokenizer_handle_template_param(Tokenizer* self)
             return -1;
     }
 
-    class = PyObject_GetAttrString(tokens, "TemplateParamSeparator");
+    PyObject* class = PyObject_GetAttrString(tokens, "TemplateParamSeparator");
     if (!class) return -1;
-    token = PyInstance_New(class, NOARGS, NOKWARGS);
+    PyObject* token = PyInstance_New(class, NOARGS, NOKWARGS);
     Py_DECREF(class);
     if (!token) return -1;
 
@@ -673,7 +674,43 @@ Tokenizer_handle_template_param(Tokenizer* self)
 static int
 Tokenizer_handle_template_param_value(Tokenizer* self)
 {
+    if (setjmp(exception_env) == BAD_ROUTE) {
+        PyObject* stack = Tokenizer_pop(self);
+        Py_XDECREF(stack);
+        longjmp(exception_env, BAD_ROUTE);
+    }
+
+    else {
+        const char* unsafes[] = {"\n", "{{", "}}"};
+        if (Tokenizer_verify_safe(self, unsafes))
+            return -1;
+    }
+
+    PyObject* stack = Tokenizer_pop_keeping_context(self);
+    if (!stack) return -1;
+    if (Tokenizer_write_all(self, stack)) {
+        Py_DECREF(stack);
+        return -1;
+    }
+    Py_DECREF(stack);
 
+    Py_ssize_t context = Tokenizer_CONTEXT_VAL(self);
+    context ^= LC_TEMPLATE_PARAM_KEY;
+    context |= LC_TEMPLATE_PARAM_VALUE;
+    if (Tokenizer_set_context(self, context))
+        return -1;
+
+    PyObject* class = PyObject_GetAttrString(tokens, "TemplateParamEquals");
+    if (!class) return -1;
+    PyObject* token = PyInstance_New(class, NOARGS, NOKWARGS);
+    Py_DECREF(class);
+    if (!token) return -1;
+
+    if (Tokenizer_write(self, token)) {
+        Py_DECREF(token);
+        return -1;
+    }
+    Py_DECREF(token);
 }
 
 /*
diff --git a/mwparserfromhell/parser/tokenizer.py b/mwparserfromhell/parser/tokenizer.py
index 364455d..508344e 100644
--- a/mwparserfromhell/parser/tokenizer.py
+++ b/mwparserfromhell/parser/tokenizer.py
@@ -197,10 +197,9 @@ class Tokenizer(object):
         except BadRoute:
             self._head = reset
             raise
-        else:
-            self._write_first(tokens.TemplateOpen())
-            self._write_all(template)
-            self._write(tokens.TemplateClose())
+        self._write_first(tokens.TemplateOpen())
+        self._write_all(template)
+        self._write(tokens.TemplateClose())
 
     def _parse_argument(self):
         """Parse an argument at the head of the wikicode string."""
@@ -210,10 +209,9 @@ class Tokenizer(object):
         except BadRoute:
             self._head = reset
             raise
-        else:
-            self._write_first(tokens.ArgumentOpen())
-            self._write_all(argument)
-            self._write(tokens.ArgumentClose())
+        self._write_first(tokens.ArgumentOpen())
+        self._write_all(argument)
+        self._write(tokens.ArgumentClose())
 
     def _verify_safe(self, unsafes):
         """Verify that there are no unsafe characters in the current stack.
@@ -249,8 +247,7 @@ class Tokenizer(object):
         except BadRoute:
             self._pop()
             raise
-        else:
-            self._write_all(self._pop(keep_context=True))
+        self._write_all(self._pop(keep_context=True))
         self._context ^= contexts.TEMPLATE_PARAM_KEY
         self._context |= contexts.TEMPLATE_PARAM_VALUE
         self._write(tokens.TemplateParamEquals())

From f401ede179b469118ac936a8646e5f5a3be128d4 Mon Sep 17 00:00:00 2001
From: Ben Kurtovic <ben.kurtovic@verizon.net>
Date: Tue, 25 Sep 2012 17:32:43 -0400
Subject: [PATCH 13/47] Implementing more stuff.

---
 mwparserfromhell/parser/tokenizer.c | 84 +++++++++++++++++++++++++++++++++++++
 1 file changed, 84 insertions(+)

diff --git a/mwparserfromhell/parser/tokenizer.c b/mwparserfromhell/parser/tokenizer.c
index e7699fd..b895f6c 100644
--- a/mwparserfromhell/parser/tokenizer.c
+++ b/mwparserfromhell/parser/tokenizer.c
@@ -259,6 +259,39 @@ Tokenizer_write_text(Tokenizer* self, PyObject* text)
 static int
 Tokenizer_write_all(Tokenizer* self, PyObject* tokenlist)
 {
+    if (PySequence_Fast_GET_SIZE(tokenlist) > 0) {
+        PyObject* token = PySequence_Fast_GET_ITEM(tokenlist, 0);
+        PyObject* class = PyObject_GetAttrString(tokens, "Text");
+        if (!class) return -1;
+
+        switch (PyObject_IsInstance(token, class)) {
+            case 0:
+                break;
+            case 1:
+                PyObject* text = PyObject_GetAttrString(token, "text");
+                if (!text) {
+                    Py_DECREF(class);
+                    return -1;
+                }
+                if (PySequence_DelItem(tokenlist, 0)) {
+                    Py_DECREF(text);
+                    Py_DECREF(class);
+                    return -1;
+                }
+                if (Tokenizer_write_text(self, text)) {
+                    Py_DECREF(text);
+                    Py_DECREF(class);
+                    return -1;
+                }
+                Py_DECREF(text);
+                break
+            case -1:
+                Py_DECREF(class);
+                return -1;
+        }
+        Py_DECREF(class);
+    }
+
     if (Tokenizer_push_textbuffer(self))
         return -1;
 
@@ -711,6 +744,7 @@ Tokenizer_handle_template_param_value(Tokenizer* self)
         return -1;
     }
     Py_DECREF(token);
+    return 0;
 }
 
 /*
@@ -719,7 +753,27 @@ Tokenizer_handle_template_param_value(Tokenizer* self)
 static PyObject*
 Tokenizer_handle_template_end(Tokenizer* self)
 {
+    PyObject* stack;
+    Py_ssize_t context = Tokenizer_CONTEXT_VAL(self);
 
+    if (context & LC_TEMPLATE_NAME) {
+        const char* unsafes[] = {"\n", "{", "}", "[", "]"};
+        if (Tokenizer_verify_safe(self, unsafes))
+            return NULL;
+    }
+    else if (context & LC_TEMPLATE_PARAM_KEY) {
+        stack = Tokenizer_pop_keeping_context(self);
+        if (!stack) return NULL;
+        if (Tokenizer_write_all(self, stack)) {
+            Py_DECREF(stack);
+            return NULL;
+        }
+        Py_DECREF(stack);
+    }
+
+    self->head++;
+    stack = Tokenizer_pop(self);
+    return stack;
 }
 
 /*
@@ -728,7 +782,28 @@ Tokenizer_handle_template_end(Tokenizer* self)
 static int
 Tokenizer_handle_argument_separator(Tokenizer* self)
 {
+    const char* unsafes[] = {"\n", "{{", "}}"};
+    if (Tokenizer_verify_safe(self, unsafes))
+        return -1;
+
+    Py_ssize_t context = Tokenizer_CONTEXT_VAL(self);
+    context ^= LC_ARGUMENT_NAME;
+    context |= LC_ARGUMENT_DEFAULT;
+    if (Tokenizer_set_context(self, context))
+        return -1;
+
+    PyObject* class = PyObject_GetAttrString(tokens, "ArgumentSeparator");
+    if (!class) return -1;
+    PyObject* token = PyInstance_New(class, NOARGS, NOKWARGS);
+    Py_DECREF(class);
+    if (!token) return -1;
 
+    if (Tokenizer_write(self, token)) {
+        Py_DECREF(token);
+        return -1;
+    }
+    Py_DECREF(token);
+    return 0;
 }
 
 /*
@@ -737,7 +812,16 @@ Tokenizer_handle_argument_separator(Tokenizer* self)
 static PyObject*
 Tokenizer_handle_argument_end(Tokenizer* self)
 {
+    Py_ssize_t context = Tokenizer_CONTEXT_VAL(self);
+    if (context & LC_ARGUMENT_NAME) {
+        const char* unsafes[] = {"\n", "{{", "}}"};
+        if (Tokenizer_verify_safe(self, unsafes))
+            return NULL;
+    }
 
+    self->head += 2;
+    PyObject* stack = Tokenizer_pop(self);
+    return stack;
 }
 
 /*

From 707ecc383740165096d74c471e5f1b739f752f71 Mon Sep 17 00:00:00 2001
From: Ben Kurtovic <ben.kurtovic@verizon.net>
Date: Tue, 25 Sep 2012 17:51:23 -0400
Subject: [PATCH 14/47] Implement Tokenizer_parse_wikilink() and more.

---
 mwparserfromhell/parser/tokenizer.c | 83 ++++++++++++++++++++++++++++++++++---
 1 file changed, 77 insertions(+), 6 deletions(-)

diff --git a/mwparserfromhell/parser/tokenizer.c b/mwparserfromhell/parser/tokenizer.c
index b895f6c..9068d94 100644
--- a/mwparserfromhell/parser/tokenizer.c
+++ b/mwparserfromhell/parser/tokenizer.c
@@ -264,11 +264,12 @@ Tokenizer_write_all(Tokenizer* self, PyObject* tokenlist)
         PyObject* class = PyObject_GetAttrString(tokens, "Text");
         if (!class) return -1;
 
+        PyObject* text;
         switch (PyObject_IsInstance(token, class)) {
             case 0:
                 break;
             case 1:
-                PyObject* text = PyObject_GetAttrString(token, "text");
+                text = PyObject_GetAttrString(token, "text");
                 if (!text) {
                     Py_DECREF(class);
                     return -1;
@@ -284,7 +285,7 @@ Tokenizer_write_all(Tokenizer* self, PyObject* tokenlist)
                     return -1;
                 }
                 Py_DECREF(text);
-                break
+                break;
             case -1:
                 Py_DECREF(class);
                 return -1;
@@ -463,13 +464,20 @@ Tokenizer_parse_template(Tokenizer* self)
         if (!template) return -1;
 
         class = PyObject_GetAttrString(tokens, "TemplateOpen");
-        if (!class) return -1;
+        if (!class) {
+            Py_DECREF(template);
+            return -1;
+        }
         token = PyInstance_New(class, NOARGS, NOKWARGS);
         Py_DECREF(class);
-        if (!token) return -1;
+        if (!token) {
+            Py_DECREF(template);
+            return -1;
+        }
 
         if (Tokenizer_write_first(self, token)) {
             Py_DECREF(token);
+            Py_DECREF(template);
             return -1;
         }
         Py_DECREF(token);
@@ -515,13 +523,20 @@ Tokenizer_parse_argument(Tokenizer* self)
         if (!argument) return -1;
 
         class = PyObject_GetAttrString(tokens, "ArgumentOpen");
-        if (!class) return -1;
+        if (!class) {
+            Py_DECREF(argument);
+            return -1;
+        }
         token = PyInstance_New(class, NOARGS, NOKWARGS);
         Py_DECREF(class);
-        if (!token) return -1;
+        if (!token) {
+            Py_DECREF(argument);
+            return -1;
+        }
 
         if (Tokenizer_write_first(self, token)) {
             Py_DECREF(token);
+            Py_DECREF(argument);
             return -1;
         }
         Py_DECREF(token);
@@ -830,7 +845,63 @@ Tokenizer_handle_argument_end(Tokenizer* self)
 static int
 Tokenizer_parse_wikilink(Tokenizer* self)
 {
+    self->head += 2;
+    Py_ssize_t reset = self->head - 1;
+
+    if (setjmp(exception_env) == BAD_ROUTE) {
+        self->head = reset;
+        PyObject* text = PyUnicode_FromString("[[");
+        if (!text) return -1;
+        if (Tokenizer_write_text(self, text)) {
+            Py_XDECREF(text);
+            return -1;
+        }
+    }
+
+    else {
+        PyObject *class, *token;
+        PyObject *wikilink = Tokenizer_parse(self, LC_WIKILINK_TITLE);
+        if (!wikilink) return -1;
+
+        class = PyObject_GetAttrString(tokens, "WikilinkOpen");
+        if (!class) {
+            Py_DECREF(wikilink);
+            return -1;
+        }
+        token = PyInstance_New(class, NOARGS, NOKWARGS);
+        Py_DECREF(class);
+        if (!token) {
+            Py_DECREF(wikilink);
+            return -1;
+        }
+
+        if (Tokenizer_write(self, token)) {
+            Py_DECREF(token);
+            Py_DECREF(wikilink);
+            return -1;
+        }
+        Py_DECREF(token);
 
+        if (Tokenizer_write_all(self, wikilink)) {
+            Py_DECREF(wikilink);
+            return -1;
+        }
+        Py_DECREF(wikilink);
+
+        class = PyObject_GetAttrString(tokens, "WikilinkClose");
+        if (!class) return -1;
+        token = PyInstance_New(class, NOARGS, NOKWARGS);
+        Py_DECREF(class);
+        if (!token) return -1;
+
+        if (Tokenizer_write(self, token)) {
+            Py_DECREF(token);
+            return -1;
+        }
+        Py_DECREF(token);
+    }
+
+    return 0;
 }
 
 /*

From 7c29a2a65e253ad5a9473fe7fc65786666889d1a Mon Sep 17 00:00:00 2001
From: Ben Kurtovic <ben.kurtovic@verizon.net>
Date: Tue, 25 Sep 2012 17:54:38 -0400
Subject: [PATCH 15/47] Implement Tokenizer_handle_wikilink_separator()/_end().

---
 mwparserfromhell/parser/tokenizer.c | 32 ++++++++++++++++++++++++++++++--
 1 file changed, 30 insertions(+), 2 deletions(-)

diff --git a/mwparserfromhell/parser/tokenizer.c b/mwparserfromhell/parser/tokenizer.c
index 9068d94..907c55e 100644
--- a/mwparserfromhell/parser/tokenizer.c
+++ b/mwparserfromhell/parser/tokenizer.c
@@ -827,8 +827,7 @@ Tokenizer_handle_argument_separator(Tokenizer* self)
 static PyObject*
 Tokenizer_handle_argument_end(Tokenizer* self)
 {
-    Py_ssize_t context = Tokenizer_CONTEXT_VAL(self);
-    if (context & LC_ARGUMENT_NAME) {
+    if (Tokenizer_CONTEXT_VAL(self) & LC_ARGUMENT_NAME) {
         const char* unsafes[] = {"\n", "{{", "}}"};
         if (Tokenizer_verify_safe(self, unsafes))
             return NULL;
@@ -910,7 +909,28 @@ Tokenizer_parse_wikilink(Tokenizer* self)
 static int
 Tokenizer_handle_wikilink_separator(Tokenizer* self)
 {
+    const char* unsafes[] = {"\n", "{", "}", "[", "]"};
+    if (Tokenizer_verify_safe(self, unsafes))
+        return -1;
+
+    Py_ssize_t context = Tokenizer_CONTEXT_VAL(self);
+    context ^= LC_WIKILINK_TITLE;
+    context |= LC_WIKILINK_TEXT;
+    if (Tokenizer_set_context(self, context))
+        return -1;
+
+    PyObject* class = PyObject_GetAttrString(tokens, "WikilinkSeparator");
+    if (!class) return -1;
+    PyObject* token = PyInstance_New(class, NOARGS, NOKWARGS);
+    Py_DECREF(class);
+    if (!token) return -1;
 
+    if (Tokenizer_write(self, token)) {
+        Py_DECREF(token);
+        return -1;
+    }
+    Py_DECREF(token);
+    return 0;
 }
 
 /*
@@ -919,7 +939,15 @@ Tokenizer_handle_wikilink_separator(Tokenizer* self)
 static PyObject*
 Tokenizer_handle_wikilink_end(Tokenizer* self)
 {
+    if (Tokenizer_CONTEXT_VAL(self) & LC_WIKILINK_TITLE) {
+        const char* unsafes[] = {"\n", "{", "}", "[", "]"};
+        if (Tokenizer_verify_safe(self, unsafes))
+            return NULL;
+    }
 
+    self->head += 1;
+    PyObject* stack = Tokenizer_pop(self);
+    return stack;
 }
 
 /*

From 150f3311290a8569eb960084e070eb23f6e70c3c Mon Sep 17 00:00:00 2001
From: Ben Kurtovic <ben.kurtovic@verizon.net>
Date: Tue, 25 Sep 2012 18:11:29 -0400
Subject: [PATCH 16/47] Implement Tokenizer_parse_entity(),
 Tokenizer_parse_comment().

---
 mwparserfromhell/parser/tokenizer.c | 79 ++++++++++++++++++++++++++++++++++++-
 1 file changed, 77 insertions(+), 2 deletions(-)

diff --git a/mwparserfromhell/parser/tokenizer.c b/mwparserfromhell/parser/tokenizer.c
index 907c55e..d302ea2 100644
--- a/mwparserfromhell/parser/tokenizer.c
+++ b/mwparserfromhell/parser/tokenizer.c
@@ -856,7 +856,6 @@ Tokenizer_parse_wikilink(Tokenizer* self)
             return -1;
         }
     }
-
     else {
         PyObject *class, *token;
         PyObject *wikilink = Tokenizer_parse(self, LC_WIKILINK_TITLE);
@@ -899,7 +898,6 @@ Tokenizer_parse_wikilink(Tokenizer* self)
         }
         Py_DECREF(token);
     }
-
     return 0;
 }
 
@@ -983,7 +981,29 @@ Tokenizer_really_parse_entity(Tokenizer* self)
 static int
 Tokenizer_parse_entity(Tokenizer* self)
 {
+    Py_ssize_t reset = self->head;
+    if (Tokenizer_push(self, 0))
+        return -1;
 
+    if (setjmp(exception_env) == BAD_ROUTE) {
+        self->head = reset;
+        if (Tokenizer_write_text(self, Tokenizer_read(self, 0)))
+            return -1;
+    }
+    else {
+        if (Tokenizer_really_parse_entity(self))
+            return -1;
+
+        PyObject* tokenlist = Tokenizer_pop(self);
+        if (!tokenlist) return -1;
+        if (Tokenizer_write_all(self, tokenlist)) {
+            Py_DECREF(tokenlist);
+            return -1;
+        }
+
+        Py_DECREF(tokenlist);
+    }
+    return 0;
 }
 
 /*
@@ -992,7 +1012,62 @@ Tokenizer_parse_entity(Tokenizer* self)
 static int
 Tokenizer_parse_comment(Tokenizer* self)
 {
+    self->head += 4;
+    Py_ssize_t reset = self->head - 1;
 
+    if (setjmp(exception_env) == BAD_ROUTE) {
+        self->head = reset;
+        PyObject* text = PyUnicode_FromString("<!--");
+        if (!text) return -1;
+        if (Tokenizer_write_text(self, text)) {
+            Py_XDECREF(text);
+            return -1;
+        }
+    }
+    else {
+        PyObject *class, *token;
+        PyObject *comment = Tokenizer_parse(self, LC_WIKILINK_TITLE);
+        if (!comment) return -1;
+
+        class = PyObject_GetAttrString(tokens, "CommentStart");
+        if (!class) {
+            Py_DECREF(comment);
+            return -1;
+        }
+        token = PyInstance_New(class, NOARGS, NOKWARGS);
+        Py_DECREF(class);
+        if (!token) {
+            Py_DECREF(comment);
+            return -1;
+        }
+
+        if (Tokenizer_write(self, token)) {
+            Py_DECREF(token);
+            Py_DECREF(comment);
+            return -1;
+        }
+        Py_DECREF(token);
+
+        if (Tokenizer_write_all(self, comment)) {
+            Py_DECREF(comment);
+            return -1;
+        }
+        Py_DECREF(comment);
+
+        class = PyObject_GetAttrString(tokens, "CommentEnd");
+        if (!class) return -1;
+        token = PyInstance_New(class, NOARGS, NOKWARGS);
+        Py_DECREF(class);
+        if (!token) return -1;
+
+        if (Tokenizer_write(self, token)) {
+            Py_DECREF(token);
+            return -1;
+        }
+        Py_DECREF(token);
+        self->head += 2;
+    }
+    return 0;
 }
 
 /*

From 48188bfa99a01a52dcde7adb97ae03759987e59e Mon Sep 17 00:00:00 2001
From: Ben Kurtovic <ben.kurtovic@verizon.net>
Date: Tue, 25 Sep 2012 19:12:04 -0400
Subject: [PATCH 17/47] Implement Tokenizer_parse_heading(), some other stuff.

---
 mwparserfromhell/parser/tokenizer.c | 122 +++++++++++++++++++++++++++++++++---
 mwparserfromhell/parser/tokenizer.h |  10 ++-
 2 files changed, 124 insertions(+), 8 deletions(-)

diff --git a/mwparserfromhell/parser/tokenizer.c b/mwparserfromhell/parser/tokenizer.c
index d302ea2..a17ec69 100644
--- a/mwparserfromhell/parser/tokenizer.c
+++ b/mwparserfromhell/parser/tokenizer.c
@@ -112,9 +112,16 @@ Tokenizer_push_textbuffer(Tokenizer* self)
         if (!text) return -1;
 
         PyObject* class = PyObject_GetAttrString(tokens, "Text");
-        if (!class) return -1;
+        if (!class) {
+            Py_DECREF(text);
+            return -1;
+        }
         PyObject* kwargs = PyDict_New();
-        if (!kwargs) return -1;
+        if (!kwargs) {
+            Py_DECREF(class);
+            Py_DECREF(text);
+            return -1;
+        }
         PyDict_SetItemString(kwargs, "text", text);
         Py_DECREF(text);
 
@@ -406,9 +413,7 @@ Tokenizer_parse_template_or_argument(Tokenizer* self)
         if (setjmp(exception_env) == BAD_ROUTE) {
             if (setjmp(exception_env) == BAD_ROUTE) {
                 char bracestr[braces];
-                for (i = 0; i < braces; i++) {
-                        bracestr[i] = *"{";
-                }
+                for (i = 0; i < braces; i++) bracestr[i] = *"{";
                 PyObject* text = PyUnicode_FromString(bracestr);
 
                 if (Tokenizer_write_text_then_stack(self, text)) {
@@ -954,13 +959,116 @@ Tokenizer_handle_wikilink_end(Tokenizer* self)
 static int
 Tokenizer_parse_heading(Tokenizer* self)
 {
+    self->global |= GL_HEADING;
+    Py_ssize_t reset = self->head;
+    self->head += 1;
+    Py_ssize_t best = 1, i;
+    PyObject* text;
+
+    while (Tokenizer_READ(self, 0) == PU "=") {
+        best++;
+        self->head++;
+    }
+
+    Py_ssize_t context = LC_HEADING_LEVEL_1 << (best > 5 ? 5 : best - 1);
+
+    if (setjmp(exception_env) == BAD_ROUTE) {
+        self->head = reset + best - 1;
+        char blocks[best];
+        for (i = 0; i < best; i++) blocks[i] = *"{";
+        text = PyUnicode_FromString(blocks);
+        if (!text) return -1;
+
+        if (Tokenizer_write_text_then_stack(self, text)) {
+            Py_DECREF(text);
+            return -1;
+        }
+        Py_DECREF(text);
+        self->global ^= GL_HEADING;
+    }
+    else {
+        HeadingData* heading = (HeadingData*) Tokenizer_parse(self, context);
+        if (!heading) return -1;
+
+        PyObject* level = PyInt_FromSsize_t(heading->level);
+        if (!level) {
+            Py_DECREF(heading->title);
+            return -1;
+        }
+
+        PyObject* class = PyObject_GetAttrString(tokens, "HeadingStart");
+        if (!class) {
+            Py_DECREF(level);
+            Py_DECREF(heading->title);
+            return -1;
+        }
+        PyObject* kwargs = PyDict_New();
+        if (!kwargs) {
+            Py_DECREF(class);
+            Py_DECREF(level);
+            Py_DECREF(heading->title);
+            return -1;
+        }
+        PyDict_SetItemString(kwargs, "level", level);
+        Py_DECREF(level);
+
+        PyObject* token = PyInstance_New(class, NOARGS, kwargs);
+        Py_DECREF(class);
+        Py_DECREF(kwargs);
+        if (!token) return -1;
+
+        if (Tokenizer_write(self, token)) {
+            Py_DECREF(token);
+            Py_DECREF(heading->title);
+            return -1;
+        }
+        Py_DECREF(token);
 
+        if (heading->level < best) {
+            Py_ssize_t diff = best - heading->level;
+            char diffblocks[diff];
+            for (i = 0; i < diff; i++) diffblocks[i] = *"{";
+            PyObject* text = PyUnicode_FromString(diffblocks);
+            if (!text) {
+                Py_DECREF(heading->title);
+                return -1;
+            }
+
+            if (Tokenizer_write_text_then_stack(self, text)) {
+                Py_DECREF(text);
+                Py_DECREF(heading->title);
+                return -1;
+            }
+            Py_DECREF(text);
+        }
+
+        if (Tokenizer_write_all(self, heading->title)) {
+            Py_DECREF(heading->title);
+            return -1;
+        }
+        Py_DECREF(heading->title);
+
+        class = PyObject_GetAttrString(tokens, "HeadingEnd");
+        if (!class) return -1;
+        token = PyInstance_New(class, NOARGS, NOKWARGS);
+        Py_DECREF(class);
+        if (!token) return -1;
+
+        if (Tokenizer_write(self, token)) {
+            Py_DECREF(token);
+            return -1;
+        }
+        Py_DECREF(token);
+
+        self->global ^= GL_HEADING;
+    }
+    return 0;
 }
 
 /*
     Handle the end of a section heading at the head of the string.
 */
-static PyObject*
+static HeadingData*
 Tokenizer_handle_heading_end(Tokenizer* self)
 {
 
@@ -1167,7 +1275,7 @@ Tokenizer_parse(Tokenizer* self, Py_ssize_t context)
             }
         }
         else if (this_data == PU "=" && this_context & LC_HEADING) {
-            return Tokenizer_handle_heading_end(self);
+            return (PyObject*) Tokenizer_handle_heading_end(self);
         }
         else if (this_data == PU "\n" && this_context & LC_HEADING) {
             Tokenizer_fail_route(self);
diff --git a/mwparserfromhell/parser/tokenizer.h b/mwparserfromhell/parser/tokenizer.h
index 951e238..3a87a37 100644
--- a/mwparserfromhell/parser/tokenizer.h
+++ b/mwparserfromhell/parser/tokenizer.h
@@ -76,6 +76,14 @@ static const Py_ssize_t LC_COMMENT =              0x2000;
 static const Py_ssize_t GL_HEADING = 0x1;
 
 
+/* Miscellaneous structs: */
+
+typedef struct {
+    PyObject* title;
+    Py_ssize_t level;
+} HeadingData;
+
+
 /* Tokenizer object definition: */
 
 typedef struct {
@@ -131,7 +139,7 @@ static int Tokenizer_parse_wikilink(Tokenizer* self);
 static int Tokenizer_handle_wikilink_separator(Tokenizer* self);
 static PyObject* Tokenizer_handle_wikilink_end(Tokenizer* self);
 static int Tokenizer_parse_heading(Tokenizer* self);
-static PyObject* Tokenizer_handle_heading_end(Tokenizer* self);
+static HeadingData* Tokenizer_handle_heading_end(Tokenizer* self);
 static int Tokenizer_really_parse_entity(Tokenizer* self);
 static int Tokenizer_parse_entity(Tokenizer* self);
 static int Tokenizer_parse_comment(Tokenizer* self);

From c50de647e25e146c0d065d7c33f1c9a0cd6a0315 Mon Sep 17 00:00:00 2001
From: Ben Kurtovic <ben.kurtovic@verizon.net>
Date: Tue, 25 Sep 2012 20:17:41 -0400
Subject: [PATCH 18/47] Implement Tokenizer_handle_heading_end() and malloc for
 HeadingData.

---
 mwparserfromhell/parser/tokenizer.c | 95 ++++++++++++++++++++++++++++++++++---
 1 file changed, 89 insertions(+), 6 deletions(-)

diff --git a/mwparserfromhell/parser/tokenizer.c b/mwparserfromhell/parser/tokenizer.c
index a17ec69..31bebe8 100644
--- a/mwparserfromhell/parser/tokenizer.c
+++ b/mwparserfromhell/parser/tokenizer.c
@@ -962,8 +962,9 @@ Tokenizer_parse_heading(Tokenizer* self)
     self->global |= GL_HEADING;
     Py_ssize_t reset = self->head;
     self->head += 1;
-    Py_ssize_t best = 1, i;
+    Py_ssize_t best = 1;
     PyObject* text;
+    int i;
 
     while (Tokenizer_READ(self, 0) == PU "=") {
         best++;
@@ -988,11 +989,11 @@ Tokenizer_parse_heading(Tokenizer* self)
     }
     else {
         HeadingData* heading = (HeadingData*) Tokenizer_parse(self, context);
-        if (!heading) return -1;
 
         PyObject* level = PyInt_FromSsize_t(heading->level);
         if (!level) {
             Py_DECREF(heading->title);
+            free(heading);
             return -1;
         }
 
@@ -1000,6 +1001,7 @@ Tokenizer_parse_heading(Tokenizer* self)
         if (!class) {
             Py_DECREF(level);
             Py_DECREF(heading->title);
+            free(heading);
             return -1;
         }
         PyObject* kwargs = PyDict_New();
@@ -1007,6 +1009,7 @@ Tokenizer_parse_heading(Tokenizer* self)
             Py_DECREF(class);
             Py_DECREF(level);
             Py_DECREF(heading->title);
+            free(heading);
             return -1;
         }
         PyDict_SetItemString(kwargs, "level", level);
@@ -1015,11 +1018,16 @@ Tokenizer_parse_heading(Tokenizer* self)
         PyObject* token = PyInstance_New(class, NOARGS, kwargs);
         Py_DECREF(class);
         Py_DECREF(kwargs);
-        if (!token) return -1;
+        if (!token) {
+            Py_DECREF(heading->title);
+            free(heading);
+            return -1;
+        }
 
         if (Tokenizer_write(self, token)) {
             Py_DECREF(token);
             Py_DECREF(heading->title);
+            free(heading);
             return -1;
         }
         Py_DECREF(token);
@@ -1027,16 +1035,18 @@ Tokenizer_parse_heading(Tokenizer* self)
         if (heading->level < best) {
             Py_ssize_t diff = best - heading->level;
             char diffblocks[diff];
-            for (i = 0; i < diff; i++) diffblocks[i] = *"{";
+            for (i = 0; i < diff; i++) diffblocks[i] = *"=";
             PyObject* text = PyUnicode_FromString(diffblocks);
             if (!text) {
                 Py_DECREF(heading->title);
+                free(heading);
                 return -1;
             }
 
             if (Tokenizer_write_text_then_stack(self, text)) {
                 Py_DECREF(text);
                 Py_DECREF(heading->title);
+                free(heading);
                 return -1;
             }
             Py_DECREF(text);
@@ -1044,9 +1054,11 @@ Tokenizer_parse_heading(Tokenizer* self)
 
         if (Tokenizer_write_all(self, heading->title)) {
             Py_DECREF(heading->title);
+            free(heading);
             return -1;
         }
         Py_DECREF(heading->title);
+        free(heading);
 
         class = PyObject_GetAttrString(tokens, "HeadingEnd");
         if (!class) return -1;
@@ -1071,7 +1083,79 @@ Tokenizer_parse_heading(Tokenizer* self)
 static HeadingData*
 Tokenizer_handle_heading_end(Tokenizer* self)
 {
+    Py_ssize_t reset = self->head;
+    self->head += 1;
+    Py_ssize_t best = 1;
+    PyObject* text;
+    int i;
+
+    while (Tokenizer_READ(self, 0) == PU "=") {
+        best++;
+        self->head++;
+    }
+
+    Py_ssize_t current = LC_HEADING_LEVEL_1 << (best > 5 ? 5 : best - 1);       // FIXME
+    Py_ssize_t level = current > best ? (best > 6 ? 6 : best) : (current > 6 ? 6 : current);
+
+    if (setjmp(exception_env) == BAD_ROUTE) {
+        if (level < best) {
+            Py_ssize_t diff = best - level;
+            char diffblocks[diff];
+            for (i = 0; i < diff; i++) diffblocks[i] = *"=";
+            text = PyUnicode_FromString(diffblocks);
+            if (!text) return NULL;
+
+            if (Tokenizer_write_text_then_stack(self, text)) {
+                Py_DECREF(text);
+                return NULL;
+            }
+            Py_DECREF(text);
+        }
+
+        self->head = reset + best - 1;
+    }
+    else {
+        Py_ssize_t context = Tokenizer_CONTEXT_VAL(self);
+        HeadingData* after = (HeadingData*) Tokenizer_parse(self, context);
 
+        char blocks[best];
+        for (i = 0; i < best; i++) blocks[i] = *"=";
+        text = PyUnicode_FromString(blocks);
+        if (!text) {
+            Py_DECREF(after->title);
+            free(after);
+            return NULL;
+        }
+
+        if (Tokenizer_write_text_then_stack(self, text)) {
+            Py_DECREF(text);
+            Py_DECREF(after->title);
+            free(after);
+            return NULL;
+        }
+        Py_DECREF(text);
+
+        if (Tokenizer_write_all(self, after->title)) {
+            Py_DECREF(after->title);
+            free(after);
+            return NULL;
+        }
+        Py_DECREF(after->title);
+        level = after->level;
+        free(after);
+    }
+
+    PyObject* stack = Tokenizer_pop(self);
+    if (!stack) return NULL;
+
+    HeadingData* heading = malloc(sizeof(HeadingData));
+    if (!heading) {
+        PyErr_NoMemory();
+        return NULL;
+    }
+    heading->title = stack;
+    heading->level = level;
+    return heading;
 }
 
 /*
@@ -1184,11 +1268,10 @@ Tokenizer_parse_comment(Tokenizer* self)
 static PyObject*
 Tokenizer_parse(Tokenizer* self, Py_ssize_t context)
 {
-    Py_ssize_t fail_contexts = LC_TEMPLATE | LC_ARGUMENT | LC_HEADING | LC_COMMENT;
-
     PyObject *this;
     Py_UNICODE *this_data, *next, *next_next, *last;
     Py_ssize_t this_context;
+    Py_ssize_t fail_contexts = LC_TEMPLATE | LC_ARGUMENT | LC_HEADING | LC_COMMENT;
     int is_marker, i;
 
     Tokenizer_push(self, context);

From 17a09e395aa9e42017195c5790566f8cba249853 Mon Sep 17 00:00:00 2001
From: Ben Kurtovic <ben.kurtovic@verizon.net>
Date: Sat, 29 Sep 2012 12:13:31 -0400
Subject: [PATCH 19/47] Fix some bugs.

---
 mwparserfromhell/parser/tokenizer.c | 59 +++++++++++++++++------------
 mwparserfromhell/parser/tokenizer.h | 75 +++++++++++++++++++------------------
 2 files changed, 74 insertions(+), 60 deletions(-)

diff --git a/mwparserfromhell/parser/tokenizer.c b/mwparserfromhell/parser/tokenizer.c
index 31bebe8..7ba7472 100644
--- a/mwparserfromhell/parser/tokenizer.c
+++ b/mwparserfromhell/parser/tokenizer.c
@@ -26,25 +26,7 @@ SOFTWARE.
 static PyObject*
 Tokenizer_new(PyTypeObject* type, PyObject* args, PyObject* kwds)
 {
-    Tokenizer *self;
-
-    self = (Tokenizer*) type->tp_alloc(type, 0);
-    if (self != NULL) {
-
-        self->text = Py_None;
-        Py_INCREF(Py_None);
-
-        self->stacks = PyList_New(0);
-        if (!self->stacks) {
-            Py_DECREF(self);
-            return NULL;
-        }
-
-        self->head = 0;
-        self->length = 0;
-        self->global = 0;
-    }
-
+    Tokenizer* self = (Tokenizer*) type->tp_alloc(type, 0);
     return (PyObject*) self;
 }
 
@@ -63,6 +45,22 @@ Tokenizer_init(Tokenizer* self, PyObject* args, PyObject* kwds)
     static char* kwlist[] = {NULL};
     if (!PyArg_ParseTupleAndKeywords(args, kwds, "", kwlist))
         return -1;
+
+    self->text = Py_None;
+    self->topstack = Py_None;
+    Py_INCREF(Py_None);
+    Py_INCREF(Py_None);
+
+    self->stacks = PyList_New(0);
+    if (!self->stacks) {
+        Py_DECREF(self);
+        return -1;
+    }
+
+    self->head = 0;
+    self->length = 0;
+    self->global = 0;
+
     return 0;
 }
 
@@ -89,6 +87,7 @@ static int
 Tokenizer_push(Tokenizer* self, Py_ssize_t context)
 {
     PyObject* top = PyList_New(3);
+    if (!top) return -1;
     PyList_SET_ITEM(top, 0, PyList_New(0));
     PyList_SET_ITEM(top, 1, PyInt_FromSsize_t(context));
     PyList_SET_ITEM(top, 2, PyList_New(0));
@@ -1094,7 +1093,7 @@ Tokenizer_handle_heading_end(Tokenizer* self)
         self->head++;
     }
 
-    Py_ssize_t current = LC_HEADING_LEVEL_1 << (best > 5 ? 5 : best - 1);       // FIXME
+    Py_ssize_t current = log2(Tokenizer_CONTEXT_VAL(self) / LC_HEADING_LEVEL_1) + 1;
     Py_ssize_t level = current > best ? (best > 6 ? 6 : best) : (current > 6 ? 6 : current);
 
     if (setjmp(exception_env) == BAD_ROUTE) {
@@ -1387,7 +1386,7 @@ Tokenizer_parse(Tokenizer* self, Py_ssize_t context)
     Build a list of tokens from a string of wikicode and return it.
 */
 static PyObject*
-Tokenizer_tokenize(Tokenizer* self, PyObject *args)
+Tokenizer_tokenize(Tokenizer* self, PyObject* args)
 {
     PyObject* text;
 
@@ -1439,10 +1438,24 @@ init_tokenizer(void)
     NOARGS = PyTuple_New(0);
     NOKWARGS = PyDict_New();
 
+    char* name = "mwparserfromhell.parser";
     PyObject* globals = PyEval_GetGlobals();
     PyObject* locals = PyEval_GetLocals();
-    PyObject* fromlist = PyList_New(0);
+    PyObject* fromlist = PyList_New(1);
+    if (!fromlist) return;
+    PyObject* submodname = PyBytes_FromString("tokens");
+    if (!submodname) {
+        Py_DECREF(fromlist);
+        return;
+    }
+    PyList_SET_ITEM(fromlist, 0, submodname);
 
-    tokens = PyImport_ImportModuleLevel("tokens", globals, locals, fromlist, 1);
+    PyObject* tokmodule = PyImport_ImportModuleLevel(name, globals, locals, fromlist, 0);
     Py_DECREF(fromlist);
+    if (!tokmodule) {
+        return;
+    }
+
+    tokens = PyObject_GetAttrString(tokmodule, "tokens");
+    Py_DECREF(tokmodule);
 }
diff --git a/mwparserfromhell/parser/tokenizer.h b/mwparserfromhell/parser/tokenizer.h
index 3a87a37..7ba9c40 100644
--- a/mwparserfromhell/parser/tokenizer.h
+++ b/mwparserfromhell/parser/tokenizer.h
@@ -26,6 +26,7 @@ SOFTWARE.
 #endif
 
 #include <Python.h>
+#include <math.h>
 #include <setjmp.h>
 #include <structmember.h>
 
@@ -108,43 +109,43 @@ typedef struct {
 
 /* Tokenizer function prototypes: */
 
-static PyObject* Tokenizer_new(PyTypeObject* type, PyObject* args, PyObject* kwds);
-static void Tokenizer_dealloc(Tokenizer* self);
-static int Tokenizer_init(Tokenizer* self, PyObject* args, PyObject* kwds);
-static int Tokenizer_set_context(Tokenizer* self, Py_ssize_t value);
-static int Tokenizer_set_textbuffer(Tokenizer* self, PyObject* value);
-static int Tokenizer_push(Tokenizer* self, Py_ssize_t context);
-static int Tokenizer_push_textbuffer(Tokenizer* self);
-static int Tokenizer_delete_top_of_stack(Tokenizer* self);
-static PyObject* Tokenizer_pop(Tokenizer* self);
-static PyObject* Tokenizer_pop_keeping_context(Tokenizer* self);
-static void Tokenizer_fail_route(Tokenizer* self);
-static int Tokenizer_write(Tokenizer* self, PyObject* token);
-static int Tokenizer_write_first(Tokenizer* self, PyObject* token);
-static int Tokenizer_write_text(Tokenizer* self, PyObject* text);
-static int Tokenizer_write_all(Tokenizer* self, PyObject* tokenlist);
-static int Tokenizer_write_text_then_stack(Tokenizer* self, PyObject* text);
-static PyObject* Tokenizer_read(Tokenizer* self, Py_ssize_t delta);
-static PyObject* Tokenizer_read_backwards(Tokenizer* self, Py_ssize_t delta);
-static int Tokenizer_parse_template_or_argument(Tokenizer* self);
-static int Tokenizer_parse_template(Tokenizer* self);
-static int Tokenizer_parse_argument(Tokenizer* self);
-static int Tokenizer_verify_safe(Tokenizer* self, const char* unsafes[]);
-static int Tokenizer_handle_template_param(Tokenizer* self);
-static int Tokenizer_handle_template_param_value(Tokenizer* self);
-static PyObject* Tokenizer_handle_template_end(Tokenizer* self);
-static int Tokenizer_handle_argument_separator(Tokenizer* self);
-static PyObject* Tokenizer_handle_argument_end(Tokenizer* self);
-static int Tokenizer_parse_wikilink(Tokenizer* self);
-static int Tokenizer_handle_wikilink_separator(Tokenizer* self);
-static PyObject* Tokenizer_handle_wikilink_end(Tokenizer* self);
-static int Tokenizer_parse_heading(Tokenizer* self);
-static HeadingData* Tokenizer_handle_heading_end(Tokenizer* self);
-static int Tokenizer_really_parse_entity(Tokenizer* self);
-static int Tokenizer_parse_entity(Tokenizer* self);
-static int Tokenizer_parse_comment(Tokenizer* self);
-static PyObject* Tokenizer_parse(Tokenizer* self, Py_ssize_t context);
-static PyObject* Tokenizer_tokenize(Tokenizer* self, PyObject *args);
+static PyObject* Tokenizer_new(PyTypeObject*, PyObject*, PyObject*);
+static void Tokenizer_dealloc(Tokenizer*);
+static int Tokenizer_init(Tokenizer*, PyObject*, PyObject*);
+static int Tokenizer_set_context(Tokenizer*, Py_ssize_t);
+static int Tokenizer_set_textbuffer(Tokenizer*, PyObject*);
+static int Tokenizer_push(Tokenizer*, Py_ssize_t);
+static int Tokenizer_push_textbuffer(Tokenizer*);
+static int Tokenizer_delete_top_of_stack(Tokenizer*);
+static PyObject* Tokenizer_pop(Tokenizer*);
+static PyObject* Tokenizer_pop_keeping_context(Tokenizer*);
+static void Tokenizer_fail_route(Tokenizer*);
+static int Tokenizer_write(Tokenizer*, PyObject*);
+static int Tokenizer_write_first(Tokenizer*, PyObject*);
+static int Tokenizer_write_text(Tokenizer*, PyObject*);
+static int Tokenizer_write_all(Tokenizer*, PyObject*);
+static int Tokenizer_write_text_then_stack(Tokenizer*, PyObject*);
+static PyObject* Tokenizer_read(Tokenizer*, Py_ssize_t);
+static PyObject* Tokenizer_read_backwards(Tokenizer*, Py_ssize_t);
+static int Tokenizer_parse_template_or_argument(Tokenizer*);
+static int Tokenizer_parse_template(Tokenizer*);
+static int Tokenizer_parse_argument(Tokenizer*);
+static int Tokenizer_verify_safe(Tokenizer*, const char* []);
+static int Tokenizer_handle_template_param(Tokenizer*);
+static int Tokenizer_handle_template_param_value(Tokenizer*);
+static PyObject* Tokenizer_handle_template_end(Tokenizer*);
+static int Tokenizer_handle_argument_separator(Tokenizer*);
+static PyObject* Tokenizer_handle_argument_end(Tokenizer*);
+static int Tokenizer_parse_wikilink(Tokenizer*);
+static int Tokenizer_handle_wikilink_separator(Tokenizer*);
+static PyObject* Tokenizer_handle_wikilink_end(Tokenizer*);
+static int Tokenizer_parse_heading(Tokenizer*);
+static HeadingData* Tokenizer_handle_heading_end(Tokenizer*);
+static int Tokenizer_really_parse_entity(Tokenizer*);
+static int Tokenizer_parse_entity(Tokenizer*);
+static int Tokenizer_parse_comment(Tokenizer*);
+static PyObject* Tokenizer_parse(Tokenizer*, Py_ssize_t);
+static PyObject* Tokenizer_tokenize(Tokenizer*, PyObject*);
 
 
 /* More structs for creating the Tokenizer type: */

From 0be18bc5b8db231a33e083f298ac170743af7fe9 Mon Sep 17 00:00:00 2001
From: Ben Kurtovic <ben.kurtovic@verizon.net>
Date: Sat, 29 Sep 2012 12:40:59 -0400
Subject: [PATCH 20/47] Fix Py_UNICODE comparisons.

---
 mwparserfromhell/parser/tokenizer.c | 58 ++++++++++++++++++-------------------
 mwparserfromhell/parser/tokenizer.h | 10 +++----
 2 files changed, 33 insertions(+), 35 deletions(-)

diff --git a/mwparserfromhell/parser/tokenizer.c b/mwparserfromhell/parser/tokenizer.c
index 7ba7472..9f7e37d 100644
--- a/mwparserfromhell/parser/tokenizer.c
+++ b/mwparserfromhell/parser/tokenizer.c
@@ -373,7 +373,7 @@ Tokenizer_parse_template_or_argument(Tokenizer* self)
     self->head += 2;
     unsigned int braces = 2, i;
 
-    while (Tokenizer_READ(self, 0) == PU "{") {
+    while (*Tokenizer_READ(self, 0) == *"{") {
         self->head++;
         braces++;
     }
@@ -965,7 +965,7 @@ Tokenizer_parse_heading(Tokenizer* self)
     PyObject* text;
     int i;
 
-    while (Tokenizer_READ(self, 0) == PU "=") {
+    while (*Tokenizer_READ(self, 0) == *"=") {
         best++;
         self->head++;
     }
@@ -1088,7 +1088,7 @@ Tokenizer_handle_heading_end(Tokenizer* self)
     PyObject* text;
     int i;
 
-    while (Tokenizer_READ(self, 0) == PU "=") {
+    while (*Tokenizer_READ(self, 0) == *"=") {
         best++;
         self->head++;
     }
@@ -1268,7 +1268,7 @@ static PyObject*
 Tokenizer_parse(Tokenizer* self, Py_ssize_t context)
 {
     PyObject *this;
-    Py_UNICODE *this_data, *next, *next_next, *last;
+    Py_UNICODE this_data, next, next_next, last;
     Py_ssize_t this_context;
     Py_ssize_t fail_contexts = LC_TEMPLATE | LC_ARGUMENT | LC_HEADING | LC_COMMENT;
     int is_marker, i;
@@ -1277,11 +1277,11 @@ Tokenizer_parse(Tokenizer* self, Py_ssize_t context)
 
     while (1) {
         this = Tokenizer_read(self, 0);
-        this_data = PyUnicode_AS_UNICODE(this);
+        this_data = *PyUnicode_AS_UNICODE(this);
 
         is_marker = 0;
         for (i = 0; i < NUM_MARKERS; i++) {
-            if (MARKERS[i] == this_data) {
+            if (*MARKERS[i] == this_data) {
                 is_marker = 1;
                 break;
             }
@@ -1295,45 +1295,45 @@ Tokenizer_parse(Tokenizer* self, Py_ssize_t context)
 
         this_context = Tokenizer_CONTEXT_VAL(self);
 
-        if (this == EMPTY) {
+        if (this_data == *"") {
             if (this_context & fail_contexts) {
                 Tokenizer_fail_route(self);
             }
             return Tokenizer_pop(self);
         }
 
-        next = Tokenizer_READ(self, 1);
+        next = *Tokenizer_READ(self, 1);
 
         if (this_context & LC_COMMENT) {
-            if (this_data == next && next == PU "-") {
-                if (Tokenizer_READ(self, 2) == PU ">") {
+            if (this_data == next && next == *"-") {
+                if (*Tokenizer_READ(self, 2) == *">") {
                     return Tokenizer_pop(self);
                 }
             }
             Tokenizer_write_text(self, this);
         }
-        else if (this_data == next && next == PU "{") {
+        else if (this_data == next && next == *"{") {
             Tokenizer_parse_template_or_argument(self);
         }
-        else if (this_data == PU "|" && this_context & LC_TEMPLATE) {
+        else if (this_data == *"|" && this_context & LC_TEMPLATE) {
             Tokenizer_handle_template_param(self);
         }
-        else if (this_data == PU "=" && this_context & LC_TEMPLATE_PARAM_KEY) {
+        else if (this_data == *"=" && this_context & LC_TEMPLATE_PARAM_KEY) {
             Tokenizer_handle_template_param_value(self);
         }
-        else if (this_data == next && next == PU "}" && this_context & LC_TEMPLATE) {
+        else if (this_data == next && next == *"}" && this_context & LC_TEMPLATE) {
             Tokenizer_handle_template_end(self);
         }
-        else if (this_data == PU "|" && this_context & LC_ARGUMENT_NAME) {
+        else if (this_data == *"|" && this_context & LC_ARGUMENT_NAME) {
             Tokenizer_handle_argument_separator(self);
         }
-        else if (this_data == next && next == PU "}" && this_context & LC_ARGUMENT) {
-            if (Tokenizer_READ(self, 2) == PU "}") {
+        else if (this_data == next && next == *"}" && this_context & LC_ARGUMENT) {
+            if (*Tokenizer_READ(self, 2) == *"}") {
                 return Tokenizer_handle_argument_end(self);
             }
             Tokenizer_write_text(self, this);
         }
-        else if (this_data == next && next == PU "[") {
+        else if (this_data == next && next == *"[") {
             if (!(this_context & LC_WIKILINK_TITLE)) {
                 Tokenizer_parse_wikilink(self);
             }
@@ -1341,33 +1341,33 @@ Tokenizer_parse(Tokenizer* self, Py_ssize_t context)
                 Tokenizer_write_text(self, this);
             }
         }
-        else if (this_data == PU "|" && this_context & LC_WIKILINK_TITLE) {
+        else if (this_data == *"|" && this_context & LC_WIKILINK_TITLE) {
             Tokenizer_handle_wikilink_separator(self);
         }
-        else if (this_data == next && next == PU "]" && this_context & LC_WIKILINK) {
+        else if (this_data == next && next == *"]" && this_context & LC_WIKILINK) {
             return Tokenizer_handle_wikilink_end(self);
         }
-        else if (this_data == PU "=" && !(self->global & GL_HEADING)) {
-            last = PyUnicode_AS_UNICODE(Tokenizer_read_backwards(self, 1));
-            if (last == PU "\n" || last == PU "") {
+        else if (this_data == *"=" && !(self->global & GL_HEADING)) {
+            last = *PyUnicode_AS_UNICODE(Tokenizer_read_backwards(self, 1));
+            if (last == *"\n" || last == *"") {
                 Tokenizer_parse_heading(self);
             }
             else {
                 Tokenizer_write_text(self, this);
             }
         }
-        else if (this_data == PU "=" && this_context & LC_HEADING) {
+        else if (this_data == *"=" && this_context & LC_HEADING) {
             return (PyObject*) Tokenizer_handle_heading_end(self);
         }
-        else if (this_data == PU "\n" && this_context & LC_HEADING) {
+        else if (this_data == *"\n" && this_context & LC_HEADING) {
             Tokenizer_fail_route(self);
         }
-        else if (this_data == PU "&") {
+        else if (this_data == *"&") {
             Tokenizer_parse_entity(self);
         }
-        else if (this_data == PU "<" && next == PU "!") {
-            next_next = Tokenizer_READ(self, 2);
-            if (next_next == Tokenizer_READ(self, 3) && next_next == PU "-") {
+        else if (this_data == *"<" && next == *"!") {
+            next_next = *Tokenizer_READ(self, 2);
+            if (next_next == *Tokenizer_READ(self, 3) && next_next == *"-") {
                 Tokenizer_parse_comment(self);
             }
             else {
diff --git a/mwparserfromhell/parser/tokenizer.h b/mwparserfromhell/parser/tokenizer.h
index 7ba9c40..d55e9d1 100644
--- a/mwparserfromhell/parser/tokenizer.h
+++ b/mwparserfromhell/parser/tokenizer.h
@@ -30,12 +30,10 @@ SOFTWARE.
 #include <setjmp.h>
 #include <structmember.h>
 
-#define PU (Py_UNICODE*)
-
-static const Py_UNICODE* MARKERS[] = {
-    PU "{", PU "}", PU "[", PU "]", PU "<", PU ">", PU "|", PU "=", PU "&",
-    PU "#", PU "*", PU ";", PU ":", PU "/", PU "-", PU "!", PU "\n", PU ""};
-static const int NUM_MARKERS = 17;
+static const char* MARKERS[] = {
+    "{",  "}", "[", "]", "<", ">", "|", "=", "&", "#", "*", ";", ":", "/", "-",
+    "!", "\n", ""};
+static const int NUM_MARKERS = 18;
 
 static jmp_buf exception_env;
 static const int BAD_ROUTE = 1;

From 9993f1ba76de47afd0f42fa88c8fd0ab219bd2e5 Mon Sep 17 00:00:00 2001
From: Ben Kurtovic <ben.kurtovic@verizon.net>
Date: Sat, 29 Sep 2012 13:00:12 -0400
Subject: [PATCH 21/47] Fix object calls.

---
 mwparserfromhell/parser/tokenizer.c | 31 +++++++++++++++----------------
 mwparserfromhell/parser/tokenizer.h |  1 -
 2 files changed, 15 insertions(+), 17 deletions(-)

diff --git a/mwparserfromhell/parser/tokenizer.c b/mwparserfromhell/parser/tokenizer.c
index 9f7e37d..1489b8a 100644
--- a/mwparserfromhell/parser/tokenizer.c
+++ b/mwparserfromhell/parser/tokenizer.c
@@ -124,7 +124,7 @@ Tokenizer_push_textbuffer(Tokenizer* self)
         PyDict_SetItemString(kwargs, "text", text);
         Py_DECREF(text);
 
-        PyObject* token = PyInstance_New(class, NOARGS, kwargs);
+        PyObject* token = PyObject_Call(class, NOARGS, kwargs);
         Py_DECREF(class);
         Py_DECREF(kwargs);
         if (!token) return -1;
@@ -472,7 +472,7 @@ Tokenizer_parse_template(Tokenizer* self)
             Py_DECREF(template);
             return -1;
         }
-        token = PyInstance_New(class, NOARGS, NOKWARGS);
+        token = PyObject_CallObject(class, NULL);
         Py_DECREF(class);
         if (!token) {
             Py_DECREF(template);
@@ -494,7 +494,7 @@ Tokenizer_parse_template(Tokenizer* self)
 
         class = PyObject_GetAttrString(tokens, "TemplateClose");
         if (!class) return -1;
-        token = PyInstance_New(class, NOARGS, NOKWARGS);
+        token = PyObject_CallObject(class, NULL);
         Py_DECREF(class);
         if (!token) return -1;
 
@@ -531,7 +531,7 @@ Tokenizer_parse_argument(Tokenizer* self)
             Py_DECREF(argument);
             return -1;
         }
-        token = PyInstance_New(class, NOARGS, NOKWARGS);
+        token = PyObject_CallObject(class, NULL);
         Py_DECREF(class);
         if (!token) {
             Py_DECREF(argument);
@@ -553,7 +553,7 @@ Tokenizer_parse_argument(Tokenizer* self)
 
         class = PyObject_GetAttrString(tokens, "ArgumentClose");
         if (!class) return -1;
-        token = PyInstance_New(class, NOARGS, NOKWARGS);
+        token = PyObject_CallObject(class, NULL);
         Py_DECREF(class);
         if (!token) return -1;
 
@@ -706,7 +706,7 @@ Tokenizer_handle_template_param(Tokenizer* self)
 
     PyObject* class = PyObject_GetAttrString(tokens, "TemplateParamSeparator");
     if (!class) return -1;
-    PyObject* token = PyInstance_New(class, NOARGS, NOKWARGS);
+    PyObject* token = PyObject_CallObject(class, NULL);
     Py_DECREF(class);
     if (!token) return -1;
 
@@ -754,7 +754,7 @@ Tokenizer_handle_template_param_value(Tokenizer* self)
 
     PyObject* class = PyObject_GetAttrString(tokens, "TemplateParamEquals");
     if (!class) return -1;
-    PyObject* token = PyInstance_New(class, NOARGS, NOKWARGS);
+    PyObject* token = PyObject_CallObject(class, NULL);
     Py_DECREF(class);
     if (!token) return -1;
 
@@ -813,7 +813,7 @@ Tokenizer_handle_argument_separator(Tokenizer* self)
 
     PyObject* class = PyObject_GetAttrString(tokens, "ArgumentSeparator");
     if (!class) return -1;
-    PyObject* token = PyInstance_New(class, NOARGS, NOKWARGS);
+    PyObject* token = PyObject_CallObject(class, NULL);
     Py_DECREF(class);
     if (!token) return -1;
 
@@ -870,7 +870,7 @@ Tokenizer_parse_wikilink(Tokenizer* self)
             Py_DECREF(wikilink);
             return -1;
         }
-        token = PyInstance_New(class, NOARGS, NOKWARGS);
+        token = PyObject_CallObject(class, NULL);
         Py_DECREF(class);
         if (!token) {
             Py_DECREF(wikilink);
@@ -892,7 +892,7 @@ Tokenizer_parse_wikilink(Tokenizer* self)
 
         class = PyObject_GetAttrString(tokens, "WikilinkClose");
         if (!class) return -1;
-        token = PyInstance_New(class, NOARGS, NOKWARGS);
+        token = PyObject_CallObject(class, NULL);
         Py_DECREF(class);
         if (!token) return -1;
 
@@ -923,7 +923,7 @@ Tokenizer_handle_wikilink_separator(Tokenizer* self)
 
     PyObject* class = PyObject_GetAttrString(tokens, "WikilinkSeparator");
     if (!class) return -1;
-    PyObject* token = PyInstance_New(class, NOARGS, NOKWARGS);
+    PyObject* token = PyObject_CallObject(class, NULL);
     Py_DECREF(class);
     if (!token) return -1;
 
@@ -1014,7 +1014,7 @@ Tokenizer_parse_heading(Tokenizer* self)
         PyDict_SetItemString(kwargs, "level", level);
         Py_DECREF(level);
 
-        PyObject* token = PyInstance_New(class, NOARGS, kwargs);
+        PyObject* token = PyObject_Call(class, NOARGS, kwargs);
         Py_DECREF(class);
         Py_DECREF(kwargs);
         if (!token) {
@@ -1061,7 +1061,7 @@ Tokenizer_parse_heading(Tokenizer* self)
 
         class = PyObject_GetAttrString(tokens, "HeadingEnd");
         if (!class) return -1;
-        token = PyInstance_New(class, NOARGS, NOKWARGS);
+        token = PyObject_CallObject(class, NULL);
         Py_DECREF(class);
         if (!token) return -1;
 
@@ -1225,7 +1225,7 @@ Tokenizer_parse_comment(Tokenizer* self)
             Py_DECREF(comment);
             return -1;
         }
-        token = PyInstance_New(class, NOARGS, NOKWARGS);
+        token = PyObject_CallObject(class, NULL);
         Py_DECREF(class);
         if (!token) {
             Py_DECREF(comment);
@@ -1247,7 +1247,7 @@ Tokenizer_parse_comment(Tokenizer* self)
 
         class = PyObject_GetAttrString(tokens, "CommentEnd");
         if (!class) return -1;
-        token = PyInstance_New(class, NOARGS, NOKWARGS);
+        token = PyObject_CallObject(class, NULL);
         Py_DECREF(class);
         if (!token) return -1;
 
@@ -1436,7 +1436,6 @@ init_tokenizer(void)
 
     EMPTY = PyUnicode_FromString("");
     NOARGS = PyTuple_New(0);
-    NOKWARGS = PyDict_New();
 
     char* name = "mwparserfromhell.parser";
     PyObject* globals = PyEval_GetGlobals();
diff --git a/mwparserfromhell/parser/tokenizer.h b/mwparserfromhell/parser/tokenizer.h
index d55e9d1..776f38a 100644
--- a/mwparserfromhell/parser/tokenizer.h
+++ b/mwparserfromhell/parser/tokenizer.h
@@ -40,7 +40,6 @@ static const int BAD_ROUTE = 1;
 
 static PyObject* EMPTY;
 static PyObject* NOARGS;
-static PyObject* NOKWARGS;
 static PyObject* tokens;
 
 

From 8e4819ea370fa6a13054fb7ce0562f900184b6fe Mon Sep 17 00:00:00 2001
From: Ben Kurtovic <ben.kurtovic@verizon.net>
Date: Fri, 5 Oct 2012 18:57:02 -0400
Subject: [PATCH 22/47] Fix exception implementation; some other cleanup and
 fixes.

---
 mwparserfromhell/parser/tokenizer.c | 515 ++++++++++++++++++------------------
 mwparserfromhell/parser/tokenizer.h |   7 +-
 2 files changed, 260 insertions(+), 262 deletions(-)

diff --git a/mwparserfromhell/parser/tokenizer.c b/mwparserfromhell/parser/tokenizer.c
index 1489b8a..d604219 100644
--- a/mwparserfromhell/parser/tokenizer.c
+++ b/mwparserfromhell/parser/tokenizer.c
@@ -206,15 +206,15 @@ Tokenizer_pop_keeping_context(Tokenizer* self)
 
 /*
     Fail the current tokenization route. Discards the current
-    stack/context/textbuffer and "raises a BAD_ROUTE exception", which is
-    implemented using longjmp().
+    stack/context/textbuffer and raises a BadRoute exception.
 */
-static void
+static void*
 Tokenizer_fail_route(Tokenizer* self)
 {
     PyObject* stack = Tokenizer_pop(self);
     Py_XDECREF(stack);
-    longjmp(exception_env, BAD_ROUTE);
+    PyErr_SetNone(BadRoute);
+    return NULL;
 }
 
 /*
@@ -393,9 +393,11 @@ Tokenizer_parse_template_or_argument(Tokenizer* self)
         }
 
         if (braces == 2) {
-            if (setjmp(exception_env) == BAD_ROUTE) {
-                PyObject* text = PyUnicode_FromString("{{");
+            if (Tokenizer_parse_template(self)) return -1;
 
+            if (PyErr_Occurred()) {
+                PyErr_Clear();
+                PyObject* text = PyUnicode_FromString("{{");
                 if (Tokenizer_write_text_then_stack(self, text)) {
                     Py_XDECREF(text);
                     return -1;
@@ -403,14 +405,20 @@ Tokenizer_parse_template_or_argument(Tokenizer* self)
 
                 Py_XDECREF(text);
                 return 0;
-            } else {
-                Tokenizer_parse_template(self);
             }
             break;
         }
 
-        if (setjmp(exception_env) == BAD_ROUTE) {
-            if (setjmp(exception_env) == BAD_ROUTE) {
+        if (Tokenizer_parse_argument(self)) return -1;
+        braces -= 3;
+
+        if (PyErr_Occurred()) {
+            PyErr_Clear();
+            if (Tokenizer_parse_template(self)) return -1;
+            braces -= 2;
+
+            if (PyErr_Occurred()) {
+                PyErr_Clear();
                 char bracestr[braces];
                 for (i = 0; i < braces; i++) bracestr[i] = *"{";
                 PyObject* text = PyUnicode_FromString(bracestr);
@@ -423,14 +431,6 @@ Tokenizer_parse_template_or_argument(Tokenizer* self)
                 Py_XDECREF(text);
                 return 0;
             }
-            else {
-                Tokenizer_parse_template(self);
-                braces -= 2;
-            }
-        }
-        else {
-            Tokenizer_parse_argument(self);
-            braces -= 3;
         }
 
         if (braces) {
@@ -458,52 +458,49 @@ Tokenizer_parse_template(Tokenizer* self)
     PyObject *template, *class, *token;
     Py_ssize_t reset = self->head;
 
-    if (setjmp(exception_env) == BAD_ROUTE) {
+    template = Tokenizer_parse(self, LC_TEMPLATE_NAME);
+    if (PyErr_Occurred()) {
         self->head = reset;
-        longjmp(exception_env, BAD_ROUTE);
+        return 0;
     }
+    if (!template) return -1;
 
-    else {
-        template = Tokenizer_parse(self, LC_TEMPLATE_NAME);
-        if (!template) return -1;
-
-        class = PyObject_GetAttrString(tokens, "TemplateOpen");
-        if (!class) {
-            Py_DECREF(template);
-            return -1;
-        }
-        token = PyObject_CallObject(class, NULL);
-        Py_DECREF(class);
-        if (!token) {
-            Py_DECREF(template);
-            return -1;
-        }
+    class = PyObject_GetAttrString(tokens, "TemplateOpen");
+    if (!class) {
+        Py_DECREF(template);
+        return -1;
+    }
+    token = PyObject_CallObject(class, NULL);
+    Py_DECREF(class);
+    if (!token) {
+        Py_DECREF(template);
+        return -1;
+    }
 
-        if (Tokenizer_write_first(self, token)) {
-            Py_DECREF(token);
-            Py_DECREF(template);
-            return -1;
-        }
+    if (Tokenizer_write_first(self, token)) {
         Py_DECREF(token);
+        Py_DECREF(template);
+        return -1;
+    }
+    Py_DECREF(token);
 
-        if (Tokenizer_write_all(self, template)) {
-            Py_DECREF(template);
-            return -1;
-        }
+    if (Tokenizer_write_all(self, template)) {
         Py_DECREF(template);
+        return -1;
+    }
+    Py_DECREF(template);
 
-        class = PyObject_GetAttrString(tokens, "TemplateClose");
-        if (!class) return -1;
-        token = PyObject_CallObject(class, NULL);
-        Py_DECREF(class);
-        if (!token) return -1;
+    class = PyObject_GetAttrString(tokens, "TemplateClose");
+    if (!class) return -1;
+    token = PyObject_CallObject(class, NULL);
+    Py_DECREF(class);
+    if (!token) return -1;
 
-        if (Tokenizer_write(self, token)) {
-            Py_DECREF(token);
-            return -1;
-        }
+    if (Tokenizer_write(self, token)) {
         Py_DECREF(token);
+        return -1;
     }
+    Py_DECREF(token);
 
     return 0;
 }
@@ -517,52 +514,49 @@ Tokenizer_parse_argument(Tokenizer* self)
     PyObject *argument, *class, *token;
     Py_ssize_t reset = self->head;
 
-    if (setjmp(exception_env) == BAD_ROUTE) {
+    argument = Tokenizer_parse(self, LC_ARGUMENT_NAME);
+    if (PyErr_Occurred()) {
         self->head = reset;
-        longjmp(exception_env, BAD_ROUTE);
+        return 0;
     }
+    if (!argument) return -1;
 
-    else {
-        argument = Tokenizer_parse(self, LC_ARGUMENT_NAME);
-        if (!argument) return -1;
-
-        class = PyObject_GetAttrString(tokens, "ArgumentOpen");
-        if (!class) {
-            Py_DECREF(argument);
-            return -1;
-        }
-        token = PyObject_CallObject(class, NULL);
-        Py_DECREF(class);
-        if (!token) {
-            Py_DECREF(argument);
-            return -1;
-        }
+    class = PyObject_GetAttrString(tokens, "ArgumentOpen");
+    if (!class) {
+        Py_DECREF(argument);
+        return -1;
+    }
+    token = PyObject_CallObject(class, NULL);
+    Py_DECREF(class);
+    if (!token) {
+        Py_DECREF(argument);
+        return -1;
+    }
 
-        if (Tokenizer_write_first(self, token)) {
-            Py_DECREF(token);
-            Py_DECREF(argument);
-            return -1;
-        }
+    if (Tokenizer_write_first(self, token)) {
         Py_DECREF(token);
+        Py_DECREF(argument);
+        return -1;
+    }
+    Py_DECREF(token);
 
-        if (Tokenizer_write_all(self, argument)) {
-            Py_DECREF(argument);
-            return -1;
-        }
+    if (Tokenizer_write_all(self, argument)) {
         Py_DECREF(argument);
+        return -1;
+    }
+    Py_DECREF(argument);
 
-        class = PyObject_GetAttrString(tokens, "ArgumentClose");
-        if (!class) return -1;
-        token = PyObject_CallObject(class, NULL);
-        Py_DECREF(class);
-        if (!token) return -1;
+    class = PyObject_GetAttrString(tokens, "ArgumentClose");
+    if (!class) return -1;
+    token = PyObject_CallObject(class, NULL);
+    Py_DECREF(class);
+    if (!token) return -1;
 
-        if (Tokenizer_write(self, token)) {
-            Py_DECREF(token);
-            return -1;
-        }
+    if (Tokenizer_write(self, token)) {
         Py_DECREF(token);
+        return -1;
     }
+    Py_DECREF(token);
 
     return 0;
 }
@@ -679,7 +673,7 @@ Tokenizer_handle_template_param(Tokenizer* self)
     Py_ssize_t context = Tokenizer_CONTEXT_VAL(self);
 
     if (context & LC_TEMPLATE_NAME) {
-        const char* unsafes[] = {"\n", "{", "}", "[", "]"};
+        const char* unsafes[] = {"\n", "{", "}", "[", "]", NULL};
         if (Tokenizer_verify_safe(self, unsafes))
             return -1;
         if (Tokenizer_set_context(self, context ^ LC_TEMPLATE_NAME))
@@ -726,16 +720,14 @@ Tokenizer_handle_template_param(Tokenizer* self)
 static int
 Tokenizer_handle_template_param_value(Tokenizer* self)
 {
-    if (setjmp(exception_env) == BAD_ROUTE) {
+    const char* unsafes[] = {"\n", "{{", "}}", NULL};
+    if (Tokenizer_verify_safe(self, unsafes))
+        return -1;
+
+    if (PyErr_Occurred()) {
         PyObject* stack = Tokenizer_pop(self);
         Py_XDECREF(stack);
-        longjmp(exception_env, BAD_ROUTE);
-    }
-
-    else {
-        const char* unsafes[] = {"\n", "{{", "}}"};
-        if (Tokenizer_verify_safe(self, unsafes))
-            return -1;
+        return 0;
     }
 
     PyObject* stack = Tokenizer_pop_keeping_context(self);
@@ -776,7 +768,7 @@ Tokenizer_handle_template_end(Tokenizer* self)
     Py_ssize_t context = Tokenizer_CONTEXT_VAL(self);
 
     if (context & LC_TEMPLATE_NAME) {
-        const char* unsafes[] = {"\n", "{", "}", "[", "]"};
+        const char* unsafes[] = {"\n", "{", "}", "[", "]", NULL};
         if (Tokenizer_verify_safe(self, unsafes))
             return NULL;
     }
@@ -801,7 +793,7 @@ Tokenizer_handle_template_end(Tokenizer* self)
 static int
 Tokenizer_handle_argument_separator(Tokenizer* self)
 {
-    const char* unsafes[] = {"\n", "{{", "}}"};
+    const char* unsafes[] = {"\n", "{{", "}}", NULL};
     if (Tokenizer_verify_safe(self, unsafes))
         return -1;
 
@@ -832,7 +824,7 @@ static PyObject*
 Tokenizer_handle_argument_end(Tokenizer* self)
 {
     if (Tokenizer_CONTEXT_VAL(self) & LC_ARGUMENT_NAME) {
-        const char* unsafes[] = {"\n", "{{", "}}"};
+        const char* unsafes[] = {"\n", "{{", "}}", NULL};
         if (Tokenizer_verify_safe(self, unsafes))
             return NULL;
     }
@@ -851,7 +843,12 @@ Tokenizer_parse_wikilink(Tokenizer* self)
     self->head += 2;
     Py_ssize_t reset = self->head - 1;
 
-    if (setjmp(exception_env) == BAD_ROUTE) {
+    PyObject *class, *token;
+    PyObject *wikilink = Tokenizer_parse(self, LC_WIKILINK_TITLE);
+    if (!wikilink) return -1;
+
+    if (PyErr_Occurred()) {
+        PyErr_Clear();
         self->head = reset;
         PyObject* text = PyUnicode_FromString("[[");
         if (!text) return -1;
@@ -859,49 +856,45 @@ Tokenizer_parse_wikilink(Tokenizer* self)
             Py_XDECREF(text);
             return -1;
         }
+        return 0;
     }
-    else {
-        PyObject *class, *token;
-        PyObject *wikilink = Tokenizer_parse(self, LC_WIKILINK_TITLE);
-        if (!wikilink) return -1;
 
-        class = PyObject_GetAttrString(tokens, "WikilinkOpen");
-        if (!class) {
-            Py_DECREF(wikilink);
-            return -1;
-        }
-        token = PyObject_CallObject(class, NULL);
-        Py_DECREF(class);
-        if (!token) {
-            Py_DECREF(wikilink);
-            return -1;
-        }
+    class = PyObject_GetAttrString(tokens, "WikilinkOpen");
+    if (!class) {
+        Py_DECREF(wikilink);
+        return -1;
+    }
+    token = PyObject_CallObject(class, NULL);
+    Py_DECREF(class);
+    if (!token) {
+        Py_DECREF(wikilink);
+        return -1;
+    }
 
-        if (Tokenizer_write(self, token)) {
-            Py_DECREF(token);
-            Py_DECREF(wikilink);
-            return -1;
-        }
+    if (Tokenizer_write(self, token)) {
         Py_DECREF(token);
+        Py_DECREF(wikilink);
+        return -1;
+    }
+    Py_DECREF(token);
 
-        if (Tokenizer_write_all(self, wikilink)) {
-            Py_DECREF(wikilink);
-            return -1;
-        }
+    if (Tokenizer_write_all(self, wikilink)) {
         Py_DECREF(wikilink);
+        return -1;
+    }
+    Py_DECREF(wikilink);
 
-        class = PyObject_GetAttrString(tokens, "WikilinkClose");
-        if (!class) return -1;
-        token = PyObject_CallObject(class, NULL);
-        Py_DECREF(class);
-        if (!token) return -1;
+    class = PyObject_GetAttrString(tokens, "WikilinkClose");
+    if (!class) return -1;
+    token = PyObject_CallObject(class, NULL);
+    Py_DECREF(class);
+    if (!token) return -1;
 
-        if (Tokenizer_write(self, token)) {
-            Py_DECREF(token);
-            return -1;
-        }
+    if (Tokenizer_write(self, token)) {
         Py_DECREF(token);
+        return -1;
     }
+    Py_DECREF(token);
     return 0;
 }
 
@@ -911,7 +904,7 @@ Tokenizer_parse_wikilink(Tokenizer* self)
 static int
 Tokenizer_handle_wikilink_separator(Tokenizer* self)
 {
-    const char* unsafes[] = {"\n", "{", "}", "[", "]"};
+    const char* unsafes[] = {"\n", "{", "}", "[", "]", NULL};
     if (Tokenizer_verify_safe(self, unsafes))
         return -1;
 
@@ -942,7 +935,7 @@ static PyObject*
 Tokenizer_handle_wikilink_end(Tokenizer* self)
 {
     if (Tokenizer_CONTEXT_VAL(self) & LC_WIKILINK_TITLE) {
-        const char* unsafes[] = {"\n", "{", "}", "[", "]"};
+        const char* unsafes[] = {"\n", "{", "}", "[", "]", NULL};
         if (Tokenizer_verify_safe(self, unsafes))
             return NULL;
     }
@@ -971,11 +964,13 @@ Tokenizer_parse_heading(Tokenizer* self)
     }
 
     Py_ssize_t context = LC_HEADING_LEVEL_1 << (best > 5 ? 5 : best - 1);
+    HeadingData* heading = (HeadingData*) Tokenizer_parse(self, context);
 
-    if (setjmp(exception_env) == BAD_ROUTE) {
+    if (PyErr_Occurred()) {
+        PyErr_Clear();
         self->head = reset + best - 1;
         char blocks[best];
-        for (i = 0; i < best; i++) blocks[i] = *"{";
+        for (i = 0; i < best; i++) blocks[i] = *"=";
         text = PyUnicode_FromString(blocks);
         if (!text) return -1;
 
@@ -985,94 +980,92 @@ Tokenizer_parse_heading(Tokenizer* self)
         }
         Py_DECREF(text);
         self->global ^= GL_HEADING;
+        return 0;
     }
-    else {
-        HeadingData* heading = (HeadingData*) Tokenizer_parse(self, context);
 
-        PyObject* level = PyInt_FromSsize_t(heading->level);
-        if (!level) {
-            Py_DECREF(heading->title);
-            free(heading);
-            return -1;
-        }
+    PyObject* level = PyInt_FromSsize_t(heading->level);
+    if (!level) {
+        Py_DECREF(heading->title);
+        free(heading);
+        return -1;
+    }
 
-        PyObject* class = PyObject_GetAttrString(tokens, "HeadingStart");
-        if (!class) {
-            Py_DECREF(level);
-            Py_DECREF(heading->title);
-            free(heading);
-            return -1;
-        }
-        PyObject* kwargs = PyDict_New();
-        if (!kwargs) {
-            Py_DECREF(class);
-            Py_DECREF(level);
-            Py_DECREF(heading->title);
-            free(heading);
-            return -1;
-        }
-        PyDict_SetItemString(kwargs, "level", level);
+    PyObject* class = PyObject_GetAttrString(tokens, "HeadingStart");
+    if (!class) {
         Py_DECREF(level);
-
-        PyObject* token = PyObject_Call(class, NOARGS, kwargs);
+        Py_DECREF(heading->title);
+        free(heading);
+        return -1;
+    }
+    PyObject* kwargs = PyDict_New();
+    if (!kwargs) {
         Py_DECREF(class);
-        Py_DECREF(kwargs);
-        if (!token) {
-            Py_DECREF(heading->title);
-            free(heading);
-            return -1;
-        }
+        Py_DECREF(level);
+        Py_DECREF(heading->title);
+        free(heading);
+        return -1;
+    }
+    PyDict_SetItemString(kwargs, "level", level);
+    Py_DECREF(level);
 
-        if (Tokenizer_write(self, token)) {
-            Py_DECREF(token);
+    PyObject* token = PyObject_Call(class, NOARGS, kwargs);
+    Py_DECREF(class);
+    Py_DECREF(kwargs);
+    if (!token) {
+        Py_DECREF(heading->title);
+        free(heading);
+        return -1;
+    }
+
+    if (Tokenizer_write(self, token)) {
+        Py_DECREF(token);
+        Py_DECREF(heading->title);
+        free(heading);
+        return -1;
+    }
+    Py_DECREF(token);
+
+    if (heading->level < best) {
+        Py_ssize_t diff = best - heading->level;
+        char diffblocks[diff];
+        for (i = 0; i < diff; i++) diffblocks[i] = *"=";
+        PyObject* text = PyUnicode_FromString(diffblocks);
+        if (!text) {
             Py_DECREF(heading->title);
             free(heading);
             return -1;
         }
-        Py_DECREF(token);
 
-        if (heading->level < best) {
-            Py_ssize_t diff = best - heading->level;
-            char diffblocks[diff];
-            for (i = 0; i < diff; i++) diffblocks[i] = *"=";
-            PyObject* text = PyUnicode_FromString(diffblocks);
-            if (!text) {
-                Py_DECREF(heading->title);
-                free(heading);
-                return -1;
-            }
-
-            if (Tokenizer_write_text_then_stack(self, text)) {
-                Py_DECREF(text);
-                Py_DECREF(heading->title);
-                free(heading);
-                return -1;
-            }
+        if (Tokenizer_write_text_then_stack(self, text)) {
             Py_DECREF(text);
-        }
-
-        if (Tokenizer_write_all(self, heading->title)) {
             Py_DECREF(heading->title);
             free(heading);
             return -1;
         }
+        Py_DECREF(text);
+    }
+
+    if (Tokenizer_write_all(self, heading->title)) {
         Py_DECREF(heading->title);
         free(heading);
+        return -1;
+    }
+    Py_DECREF(heading->title);
+    free(heading);
 
-        class = PyObject_GetAttrString(tokens, "HeadingEnd");
-        if (!class) return -1;
-        token = PyObject_CallObject(class, NULL);
-        Py_DECREF(class);
-        if (!token) return -1;
+    class = PyObject_GetAttrString(tokens, "HeadingEnd");
+    if (!class) return -1;
+    token = PyObject_CallObject(class, NULL);
+    Py_DECREF(class);
+    if (!token) return -1;
 
-        if (Tokenizer_write(self, token)) {
-            Py_DECREF(token);
-            return -1;
-        }
+    if (Tokenizer_write(self, token)) {
         Py_DECREF(token);
-
-        self->global ^= GL_HEADING;
+        return -1;
     }
+    Py_DECREF(token);
+
+    self->global ^= GL_HEADING;
     return 0;
 }
 
@@ -1096,7 +1089,11 @@ Tokenizer_handle_heading_end(Tokenizer* self)
     Py_ssize_t current = log2(Tokenizer_CONTEXT_VAL(self) / LC_HEADING_LEVEL_1) + 1;
     Py_ssize_t level = current > best ? (best > 6 ? 6 : best) : (current > 6 ? 6 : current);
 
-    if (setjmp(exception_env) == BAD_ROUTE) {
+    Py_ssize_t context = Tokenizer_CONTEXT_VAL(self);
+    HeadingData* after = (HeadingData*) Tokenizer_parse(self, context);
+
+    if (PyErr_Occurred()) {
+        PyErr_Clear();
         if (level < best) {
             Py_ssize_t diff = best - level;
             char diffblocks[diff];
@@ -1113,10 +1110,8 @@ Tokenizer_handle_heading_end(Tokenizer* self)
 
         self->head = reset + best - 1;
     }
-    else {
-        Py_ssize_t context = Tokenizer_CONTEXT_VAL(self);
-        HeadingData* after = (HeadingData*) Tokenizer_parse(self, context);
 
+    else {
         char blocks[best];
         for (i = 0; i < best; i++) blocks[i] = *"=";
         text = PyUnicode_FromString(blocks);
@@ -1176,24 +1171,25 @@ Tokenizer_parse_entity(Tokenizer* self)
     if (Tokenizer_push(self, 0))
         return -1;
 
-    if (setjmp(exception_env) == BAD_ROUTE) {
+    if (Tokenizer_really_parse_entity(self))
+            return -1;
+
+    if (PyErr_Occurred()) {
+        PyErr_Clear();
         self->head = reset;
         if (Tokenizer_write_text(self, Tokenizer_read(self, 0)))
             return -1;
+        return 0;
     }
-    else {
-        if (Tokenizer_really_parse_entity(self))
-            return -1;
-
-        PyObject* tokenlist = Tokenizer_pop(self);
-        if (!tokenlist) return -1;
-        if (Tokenizer_write_all(self, tokenlist)) {
-            Py_DECREF(tokenlist);
-            return -1;
-        }
 
+    PyObject* tokenlist = Tokenizer_pop(self);
+    if (!tokenlist) return -1;
+    if (Tokenizer_write_all(self, tokenlist)) {
         Py_DECREF(tokenlist);
+        return -1;
     }
+
+    Py_DECREF(tokenlist);
     return 0;
 }
 
@@ -1206,7 +1202,12 @@ Tokenizer_parse_comment(Tokenizer* self)
     self->head += 4;
     Py_ssize_t reset = self->head - 1;
 
-    if (setjmp(exception_env) == BAD_ROUTE) {
+    PyObject *class, *token;
+    PyObject *comment = Tokenizer_parse(self, LC_WIKILINK_TITLE);
+    if (!comment) return -1;
+
+    if (PyErr_Occurred()) {
+        PyErr_Clear();
         self->head = reset;
         PyObject* text = PyUnicode_FromString("<!--");
         if (!text) return -1;
@@ -1214,50 +1215,46 @@ Tokenizer_parse_comment(Tokenizer* self)
             Py_XDECREF(text);
             return -1;
         }
+        return 0;
     }
-    else {
-        PyObject *class, *token;
-        PyObject *comment = Tokenizer_parse(self, LC_WIKILINK_TITLE);
-        if (!comment) return -1;
 
-        class = PyObject_GetAttrString(tokens, "CommentStart");
-        if (!class) {
-            Py_DECREF(comment);
-            return -1;
-        }
-        token = PyObject_CallObject(class, NULL);
-        Py_DECREF(class);
-        if (!token) {
-            Py_DECREF(comment);
-            return -1;
-        }
+    class = PyObject_GetAttrString(tokens, "CommentStart");
+    if (!class) {
+        Py_DECREF(comment);
+        return -1;
+    }
+    token = PyObject_CallObject(class, NULL);
+    Py_DECREF(class);
+    if (!token) {
+        Py_DECREF(comment);
+        return -1;
+    }
 
-        if (Tokenizer_write(self, token)) {
-            Py_DECREF(token);
-            Py_DECREF(comment);
-            return -1;
-        }
+    if (Tokenizer_write(self, token)) {
         Py_DECREF(token);
+        Py_DECREF(comment);
+        return -1;
+    }
+    Py_DECREF(token);
 
-        if (Tokenizer_write_all(self, comment)) {
-            Py_DECREF(comment);
-            return -1;
-        }
+    if (Tokenizer_write_all(self, comment)) {
         Py_DECREF(comment);
+        return -1;
+    }
+    Py_DECREF(comment);
 
-        class = PyObject_GetAttrString(tokens, "CommentEnd");
-        if (!class) return -1;
-        token = PyObject_CallObject(class, NULL);
-        Py_DECREF(class);
-        if (!token) return -1;
+    class = PyObject_GetAttrString(tokens, "CommentEnd");
+    if (!class) return -1;
+    token = PyObject_CallObject(class, NULL);
+    Py_DECREF(class);
+    if (!token) return -1;
 
-        if (Tokenizer_write(self, token)) {
-            Py_DECREF(token);
-            return -1;
-        }
+    if (Tokenizer_write(self, token)) {
         Py_DECREF(token);
-        self->head += 2;
+        return -1;
     }
+    Py_DECREF(token);
+    self->head += 2;
     return 0;
 }
 
@@ -1297,7 +1294,7 @@ Tokenizer_parse(Tokenizer* self, Py_ssize_t context)
 
         if (this_data == *"") {
             if (this_context & fail_contexts) {
-                Tokenizer_fail_route(self);
+                return Tokenizer_fail_route(self);
             }
             return Tokenizer_pop(self);
         }
@@ -1322,7 +1319,7 @@ Tokenizer_parse(Tokenizer* self, Py_ssize_t context)
             Tokenizer_handle_template_param_value(self);
         }
         else if (this_data == next && next == *"}" && this_context & LC_TEMPLATE) {
-            Tokenizer_handle_template_end(self);
+            return Tokenizer_handle_template_end(self);
         }
         else if (this_data == *"|" && this_context & LC_ARGUMENT_NAME) {
             Tokenizer_handle_argument_separator(self);
@@ -1360,7 +1357,7 @@ Tokenizer_parse(Tokenizer* self, Py_ssize_t context)
             return (PyObject*) Tokenizer_handle_heading_end(self);
         }
         else if (this_data == *"\n" && this_context & LC_HEADING) {
-            Tokenizer_fail_route(self);
+            return Tokenizer_fail_route(self);
         }
         else if (this_data == *"&") {
             Tokenizer_parse_entity(self);
@@ -1434,6 +1431,10 @@ init_tokenizer(void)
     Py_INCREF(&TokenizerType);
     PyModule_AddObject(module, "CTokenizer", (PyObject*) &TokenizerType);
 
+    BadRoute = PyErr_NewException("_tokenizer.BadRoute", NULL, NULL);
+    Py_INCREF(BadRoute);
+    PyModule_AddObject(module, "BadRoute", BadRoute);
+
     EMPTY = PyUnicode_FromString("");
     NOARGS = PyTuple_New(0);
 
diff --git a/mwparserfromhell/parser/tokenizer.h b/mwparserfromhell/parser/tokenizer.h
index 776f38a..9dc0670 100644
--- a/mwparserfromhell/parser/tokenizer.h
+++ b/mwparserfromhell/parser/tokenizer.h
@@ -27,7 +27,6 @@ SOFTWARE.
 
 #include <Python.h>
 #include <math.h>
-#include <setjmp.h>
 #include <structmember.h>
 
 static const char* MARKERS[] = {
@@ -35,11 +34,9 @@ static const char* MARKERS[] = {
     "!", "\n", ""};
 static const int NUM_MARKERS = 18;
 
-static jmp_buf exception_env;
-static const int BAD_ROUTE = 1;
-
 static PyObject* EMPTY;
 static PyObject* NOARGS;
+static PyObject* BadRoute;
 static PyObject* tokens;
 
 
@@ -116,7 +113,7 @@ static int Tokenizer_push_textbuffer(Tokenizer*);
 static int Tokenizer_delete_top_of_stack(Tokenizer*);
 static PyObject* Tokenizer_pop(Tokenizer*);
 static PyObject* Tokenizer_pop_keeping_context(Tokenizer*);
-static void Tokenizer_fail_route(Tokenizer*);
+static void* Tokenizer_fail_route(Tokenizer*);
 static int Tokenizer_write(Tokenizer*, PyObject*);
 static int Tokenizer_write_first(Tokenizer*, PyObject*);
 static int Tokenizer_write_text(Tokenizer*, PyObject*);

From ff0b4439f8d0b16e4f3a9e3241ef5164b5a17df2 Mon Sep 17 00:00:00 2001
From: Ben Kurtovic <ben.kurtovic@verizon.net>
Date: Sat, 6 Oct 2012 14:29:22 -0400
Subject: [PATCH 23/47] Replace Python exception machinery with a global and
 some macros.

---
 mwparserfromhell/parser/tokenizer.c  | 66 ++++++++++++++++++++----------------
 mwparserfromhell/parser/tokenizer.h  |  6 +++-
 mwparserfromhell/parser/tokenizer.py |  2 ++
 3 files changed, 43 insertions(+), 31 deletions(-)

diff --git a/mwparserfromhell/parser/tokenizer.c b/mwparserfromhell/parser/tokenizer.c
index d604219..73033e4 100644
--- a/mwparserfromhell/parser/tokenizer.c
+++ b/mwparserfromhell/parser/tokenizer.c
@@ -213,7 +213,7 @@ Tokenizer_fail_route(Tokenizer* self)
 {
     PyObject* stack = Tokenizer_pop(self);
     Py_XDECREF(stack);
-    PyErr_SetNone(BadRoute);
+    FAIL_ROUTE();
     return NULL;
 }
 
@@ -382,7 +382,6 @@ Tokenizer_parse_template_or_argument(Tokenizer* self)
     while (braces) {
         if (braces == 1) {
             PyObject* text = PyUnicode_FromString("{");
-
             if (Tokenizer_write_text_then_stack(self, text)) {
                 Py_XDECREF(text);
                 return -1;
@@ -393,10 +392,11 @@ Tokenizer_parse_template_or_argument(Tokenizer* self)
         }
 
         if (braces == 2) {
-            if (Tokenizer_parse_template(self)) return -1;
+            if (Tokenizer_parse_template(self))
+                return -1;
 
-            if (PyErr_Occurred()) {
-                PyErr_Clear();
+            if (BAD_ROUTE) {
+                RESET_ROUTE();
                 PyObject* text = PyUnicode_FromString("{{");
                 if (Tokenizer_write_text_then_stack(self, text)) {
                     Py_XDECREF(text);
@@ -409,16 +409,16 @@ Tokenizer_parse_template_or_argument(Tokenizer* self)
             break;
         }
 
-        if (Tokenizer_parse_argument(self)) return -1;
-        braces -= 3;
+        if (Tokenizer_parse_argument(self))
+            return -1;
 
-        if (PyErr_Occurred()) {
-            PyErr_Clear();
-            if (Tokenizer_parse_template(self)) return -1;
-            braces -= 2;
+        if (BAD_ROUTE) {
+            RESET_ROUTE();
+            if (Tokenizer_parse_template(self))
+                return -1;
 
-            if (PyErr_Occurred()) {
-                PyErr_Clear();
+            if (BAD_ROUTE) {
+                RESET_ROUTE();
                 char bracestr[braces];
                 for (i = 0; i < braces; i++) bracestr[i] = *"{";
                 PyObject* text = PyUnicode_FromString(bracestr);
@@ -431,6 +431,12 @@ Tokenizer_parse_template_or_argument(Tokenizer* self)
                 Py_XDECREF(text);
                 return 0;
             }
+            else {
+                braces -= 2;
+            }
+        }
+        else {
+            braces -= 3;
         }
 
         if (braces) {
@@ -459,7 +465,7 @@ Tokenizer_parse_template(Tokenizer* self)
     Py_ssize_t reset = self->head;
 
     template = Tokenizer_parse(self, LC_TEMPLATE_NAME);
-    if (PyErr_Occurred()) {
+    if (BAD_ROUTE) {
         self->head = reset;
         return 0;
     }
@@ -515,7 +521,7 @@ Tokenizer_parse_argument(Tokenizer* self)
     Py_ssize_t reset = self->head;
 
     argument = Tokenizer_parse(self, LC_ARGUMENT_NAME);
-    if (PyErr_Occurred()) {
+    if (BAD_ROUTE) {
         self->head = reset;
         return 0;
     }
@@ -724,7 +730,7 @@ Tokenizer_handle_template_param_value(Tokenizer* self)
     if (Tokenizer_verify_safe(self, unsafes))
         return -1;
 
-    if (PyErr_Occurred()) {
+    if (BAD_ROUTE) {
         PyObject* stack = Tokenizer_pop(self);
         Py_XDECREF(stack);
         return 0;
@@ -847,8 +853,8 @@ Tokenizer_parse_wikilink(Tokenizer* self)
     PyObject *wikilink = Tokenizer_parse(self, LC_WIKILINK_TITLE);
     if (!wikilink) return -1;
 
-    if (PyErr_Occurred()) {
-        PyErr_Clear();
+    if (BAD_ROUTE) {
+        RESET_ROUTE();
         self->head = reset;
         PyObject* text = PyUnicode_FromString("[[");
         if (!text) return -1;
@@ -966,8 +972,8 @@ Tokenizer_parse_heading(Tokenizer* self)
     Py_ssize_t context = LC_HEADING_LEVEL_1 << (best > 5 ? 5 : best - 1);
     HeadingData* heading = (HeadingData*) Tokenizer_parse(self, context);
 
-    if (PyErr_Occurred()) {
-        PyErr_Clear();
+    if (BAD_ROUTE) {
+        RESET_ROUTE();
         self->head = reset + best - 1;
         char blocks[best];
         for (i = 0; i < best; i++) blocks[i] = *"=";
@@ -1092,8 +1098,8 @@ Tokenizer_handle_heading_end(Tokenizer* self)
     Py_ssize_t context = Tokenizer_CONTEXT_VAL(self);
     HeadingData* after = (HeadingData*) Tokenizer_parse(self, context);
 
-    if (PyErr_Occurred()) {
-        PyErr_Clear();
+    if (BAD_ROUTE) {
+        RESET_ROUTE();
         if (level < best) {
             Py_ssize_t diff = best - level;
             char diffblocks[diff];
@@ -1174,8 +1180,8 @@ Tokenizer_parse_entity(Tokenizer* self)
     if (Tokenizer_really_parse_entity(self))
             return -1;
 
-    if (PyErr_Occurred()) {
-        PyErr_Clear();
+    if (BAD_ROUTE) {
+        RESET_ROUTE();
         self->head = reset;
         if (Tokenizer_write_text(self, Tokenizer_read(self, 0)))
             return -1;
@@ -1206,8 +1212,8 @@ Tokenizer_parse_comment(Tokenizer* self)
     PyObject *comment = Tokenizer_parse(self, LC_WIKILINK_TITLE);
     if (!comment) return -1;
 
-    if (PyErr_Occurred()) {
-        PyErr_Clear();
+    if (BAD_ROUTE) {
+        RESET_ROUTE();
         self->head = reset;
         PyObject* text = PyUnicode_FromString("<!--");
         if (!text) return -1;
@@ -1293,6 +1299,10 @@ Tokenizer_parse(Tokenizer* self, Py_ssize_t context)
         this_context = Tokenizer_CONTEXT_VAL(self);
 
         if (this_data == *"") {
+            if (this_context & LC_TEMPLATE_PARAM_KEY) {
+                PyObject* trash = Tokenizer_pop(self);
+                Py_XDECREF(trash);
+            }
             if (this_context & fail_contexts) {
                 return Tokenizer_fail_route(self);
             }
@@ -1431,10 +1441,6 @@ init_tokenizer(void)
     Py_INCREF(&TokenizerType);
     PyModule_AddObject(module, "CTokenizer", (PyObject*) &TokenizerType);
 
-    BadRoute = PyErr_NewException("_tokenizer.BadRoute", NULL, NULL);
-    Py_INCREF(BadRoute);
-    PyModule_AddObject(module, "BadRoute", BadRoute);
-
     EMPTY = PyUnicode_FromString("");
     NOARGS = PyTuple_New(0);
 
diff --git a/mwparserfromhell/parser/tokenizer.h b/mwparserfromhell/parser/tokenizer.h
index 9dc0670..91da10e 100644
--- a/mwparserfromhell/parser/tokenizer.h
+++ b/mwparserfromhell/parser/tokenizer.h
@@ -34,9 +34,13 @@ static const char* MARKERS[] = {
     "!", "\n", ""};
 static const int NUM_MARKERS = 18;
 
+static int route_state = 0;
+#define BAD_ROUTE     (route_state)
+#define FAIL_ROUTE()  (route_state = 1)
+#define RESET_ROUTE() (route_state = 0)
+
 static PyObject* EMPTY;
 static PyObject* NOARGS;
-static PyObject* BadRoute;
 static PyObject* tokens;
 
 
diff --git a/mwparserfromhell/parser/tokenizer.py b/mwparserfromhell/parser/tokenizer.py
index 508344e..9cd6290 100644
--- a/mwparserfromhell/parser/tokenizer.py
+++ b/mwparserfromhell/parser/tokenizer.py
@@ -432,6 +432,8 @@ class Tokenizer(object):
             if this is self.END:
                 fail = (contexts.TEMPLATE | contexts.ARGUMENT |
                         contexts.HEADING | contexts.COMMENT)
+                if self._context & contexts.TEMPLATE_PARAM_KEY:
+                    self._pop()
                 if self._context & fail:
                     self._fail_route()
                 return self._pop()

From 29ef2a66823fba39f1af8542eebcb397d667fa82 Mon Sep 17 00:00:00 2001
From: Ben Kurtovic <ben.kurtovic@verizon.net>
Date: Sat, 6 Oct 2012 14:34:51 -0400
Subject: [PATCH 24/47] Propogate fix from
 b447e7bfc16322fc296d45be1a7f5e6769a53498

---
 mwparserfromhell/parser/tokenizer.c  | 3 ++-
 mwparserfromhell/parser/tokenizer.py | 3 ++-
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/mwparserfromhell/parser/tokenizer.c b/mwparserfromhell/parser/tokenizer.c
index 73033e4..5117e9c 100644
--- a/mwparserfromhell/parser/tokenizer.c
+++ b/mwparserfromhell/parser/tokenizer.c
@@ -1273,7 +1273,8 @@ Tokenizer_parse(Tokenizer* self, Py_ssize_t context)
     PyObject *this;
     Py_UNICODE this_data, next, next_next, last;
     Py_ssize_t this_context;
-    Py_ssize_t fail_contexts = LC_TEMPLATE | LC_ARGUMENT | LC_HEADING | LC_COMMENT;
+    Py_ssize_t fail_contexts = (
+        LC_TEMPLATE | LC_ARGUMENT | LC_WIKILINK | LC_HEADING | LC_COMMENT);
     int is_marker, i;
 
     Tokenizer_push(self, context);
diff --git a/mwparserfromhell/parser/tokenizer.py b/mwparserfromhell/parser/tokenizer.py
index 9cd6290..5b0e976 100644
--- a/mwparserfromhell/parser/tokenizer.py
+++ b/mwparserfromhell/parser/tokenizer.py
@@ -431,7 +431,8 @@ class Tokenizer(object):
                 continue
             if this is self.END:
                 fail = (contexts.TEMPLATE | contexts.ARGUMENT |
-                        contexts.HEADING | contexts.COMMENT)
+                        contexts.WIKILINK | contexts.HEADING |
+                        contexts.COMMENT)
                 if self._context & contexts.TEMPLATE_PARAM_KEY:
                     self._pop()
                 if self._context & fail:

From 467aef3651a7854329e44111f3ce405232dabc58 Mon Sep 17 00:00:00 2001
From: Ben Kurtovic <ben.kurtovic@verizon.net>
Date: Sat, 6 Oct 2012 14:44:38 -0400
Subject: [PATCH 25/47] FromStringAndSize instead of FromString.

---
 mwparserfromhell/parser/tokenizer.c | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/mwparserfromhell/parser/tokenizer.c b/mwparserfromhell/parser/tokenizer.c
index 5117e9c..a8da2c5 100644
--- a/mwparserfromhell/parser/tokenizer.c
+++ b/mwparserfromhell/parser/tokenizer.c
@@ -421,7 +421,7 @@ Tokenizer_parse_template_or_argument(Tokenizer* self)
                 RESET_ROUTE();
                 char bracestr[braces];
                 for (i = 0; i < braces; i++) bracestr[i] = *"{";
-                PyObject* text = PyUnicode_FromString(bracestr);
+                PyObject* text = PyUnicode_FromStringAndSize(bracestr, braces);
 
                 if (Tokenizer_write_text_then_stack(self, text)) {
                     Py_XDECREF(text);
@@ -977,7 +977,7 @@ Tokenizer_parse_heading(Tokenizer* self)
         self->head = reset + best - 1;
         char blocks[best];
         for (i = 0; i < best; i++) blocks[i] = *"=";
-        text = PyUnicode_FromString(blocks);
+        text = PyUnicode_FromStringAndSize(blocks, best);
         if (!text) return -1;
 
         if (Tokenizer_write_text_then_stack(self, text)) {
@@ -1035,7 +1035,7 @@ Tokenizer_parse_heading(Tokenizer* self)
         Py_ssize_t diff = best - heading->level;
         char diffblocks[diff];
         for (i = 0; i < diff; i++) diffblocks[i] = *"=";
-        PyObject* text = PyUnicode_FromString(diffblocks);
+        PyObject* text = PyUnicode_FromStringAndSize(diffblocks, diff);
         if (!text) {
             Py_DECREF(heading->title);
             free(heading);
@@ -1104,7 +1104,7 @@ Tokenizer_handle_heading_end(Tokenizer* self)
             Py_ssize_t diff = best - level;
             char diffblocks[diff];
             for (i = 0; i < diff; i++) diffblocks[i] = *"=";
-            text = PyUnicode_FromString(diffblocks);
+            text = PyUnicode_FromStringAndSize(diffblocks, diff);
             if (!text) return NULL;
 
             if (Tokenizer_write_text_then_stack(self, text)) {
@@ -1120,7 +1120,7 @@ Tokenizer_handle_heading_end(Tokenizer* self)
     else {
         char blocks[best];
         for (i = 0; i < best; i++) blocks[i] = *"=";
-        text = PyUnicode_FromString(blocks);
+        text = PyUnicode_FromStringAndSize(blocks, best);
         if (!text) {
             Py_DECREF(after->title);
             free(after);

From 230c89711b10477752134bc91f43cb948b943a44 Mon Sep 17 00:00:00 2001
From: Ben Kurtovic <ben.kurtovic@verizon.net>
Date: Sat, 6 Oct 2012 20:47:02 -0400
Subject: [PATCH 26/47] Fix verify_save and propogate errors correctly in the
 parse func.

---
 mwparserfromhell/parser/tokenizer.c | 44 +++++++++++++++++++++----------------
 1 file changed, 25 insertions(+), 19 deletions(-)

diff --git a/mwparserfromhell/parser/tokenizer.c b/mwparserfromhell/parser/tokenizer.c
index a8da2c5..2808157 100644
--- a/mwparserfromhell/parser/tokenizer.c
+++ b/mwparserfromhell/parser/tokenizer.c
@@ -654,10 +654,7 @@ Tokenizer_verify_safe(Tokenizer* self, const char* unsafes[])
                 case 0:
                     break;
                 case 1:
-                    Py_DECREF(stripped);
-                    Py_DECREF(unsafe);
                     Tokenizer_fail_route(self);
-                    break;
                 case -1:
                     Py_DECREF(stripped);
                     Py_DECREF(unsafe);
@@ -682,6 +679,7 @@ Tokenizer_handle_template_param(Tokenizer* self)
         const char* unsafes[] = {"\n", "{", "}", "[", "]", NULL};
         if (Tokenizer_verify_safe(self, unsafes))
             return -1;
+        if (BAD_ROUTE) return -1;
         if (Tokenizer_set_context(self, context ^ LC_TEMPLATE_NAME))
             return -1;
     }
@@ -727,13 +725,12 @@ static int
 Tokenizer_handle_template_param_value(Tokenizer* self)
 {
     const char* unsafes[] = {"\n", "{{", "}}", NULL};
-    if (Tokenizer_verify_safe(self, unsafes))
+    if (Tokenizer_verify_safe(self, unsafes)) {
+        if (BAD_ROUTE) {
+            PyObject* stack = Tokenizer_pop(self);
+            Py_XDECREF(stack);
+        }
         return -1;
-
-    if (BAD_ROUTE) {
-        PyObject* stack = Tokenizer_pop(self);
-        Py_XDECREF(stack);
-        return 0;
     }
 
     PyObject* stack = Tokenizer_pop_keeping_context(self);
@@ -1164,7 +1161,7 @@ Tokenizer_handle_heading_end(Tokenizer* self)
 static int
 Tokenizer_really_parse_entity(Tokenizer* self)
 {
-
+    return 0;
 }
 
 /*
@@ -1321,19 +1318,23 @@ Tokenizer_parse(Tokenizer* self, Py_ssize_t context)
             Tokenizer_write_text(self, this);
         }
         else if (this_data == next && next == *"{") {
-            Tokenizer_parse_template_or_argument(self);
+            if (Tokenizer_parse_template_or_argument(self))
+                return NULL;
         }
         else if (this_data == *"|" && this_context & LC_TEMPLATE) {
-            Tokenizer_handle_template_param(self);
+            if (Tokenizer_handle_template_param(self))
+                return NULL;
         }
         else if (this_data == *"=" && this_context & LC_TEMPLATE_PARAM_KEY) {
-            Tokenizer_handle_template_param_value(self);
+            if (Tokenizer_handle_template_param_value(self))
+                return NULL;
         }
         else if (this_data == next && next == *"}" && this_context & LC_TEMPLATE) {
             return Tokenizer_handle_template_end(self);
         }
         else if (this_data == *"|" && this_context & LC_ARGUMENT_NAME) {
-            Tokenizer_handle_argument_separator(self);
+            if (Tokenizer_handle_argument_separator(self))
+                return NULL;
         }
         else if (this_data == next && next == *"}" && this_context & LC_ARGUMENT) {
             if (*Tokenizer_READ(self, 2) == *"}") {
@@ -1343,14 +1344,16 @@ Tokenizer_parse(Tokenizer* self, Py_ssize_t context)
         }
         else if (this_data == next && next == *"[") {
             if (!(this_context & LC_WIKILINK_TITLE)) {
-                Tokenizer_parse_wikilink(self);
+                if (Tokenizer_parse_wikilink(self))
+                    return NULL;
             }
             else {
                 Tokenizer_write_text(self, this);
             }
         }
         else if (this_data == *"|" && this_context & LC_WIKILINK_TITLE) {
-            Tokenizer_handle_wikilink_separator(self);
+            if (Tokenizer_handle_wikilink_separator(self))
+                return NULL;
         }
         else if (this_data == next && next == *"]" && this_context & LC_WIKILINK) {
             return Tokenizer_handle_wikilink_end(self);
@@ -1358,7 +1361,8 @@ Tokenizer_parse(Tokenizer* self, Py_ssize_t context)
         else if (this_data == *"=" && !(self->global & GL_HEADING)) {
             last = *PyUnicode_AS_UNICODE(Tokenizer_read_backwards(self, 1));
             if (last == *"\n" || last == *"") {
-                Tokenizer_parse_heading(self);
+                if (Tokenizer_parse_heading(self))
+                    return NULL;
             }
             else {
                 Tokenizer_write_text(self, this);
@@ -1371,12 +1375,14 @@ Tokenizer_parse(Tokenizer* self, Py_ssize_t context)
             return Tokenizer_fail_route(self);
         }
         else if (this_data == *"&") {
-            Tokenizer_parse_entity(self);
+            if (Tokenizer_parse_entity(self))
+                return NULL;
         }
         else if (this_data == *"<" && next == *"!") {
             next_next = *Tokenizer_READ(self, 2);
             if (next_next == *Tokenizer_READ(self, 3) && next_next == *"-") {
-                Tokenizer_parse_comment(self);
+                if (Tokenizer_parse_comment(self))
+                    return NULL;
             }
             else {
                 Tokenizer_write_text(self, this);

From 1bfd364a4a46537d37f57182b972393f8c84457e Mon Sep 17 00:00:00 2001
From: Ben Kurtovic <ben.kurtovic@verizon.net>
Date: Sat, 6 Oct 2012 21:24:00 -0400
Subject: [PATCH 27/47] Process contexts in Tokenizer_handle_template_param()
 correctly.

---
 mwparserfromhell/parser/tokenizer.c | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/mwparserfromhell/parser/tokenizer.c b/mwparserfromhell/parser/tokenizer.c
index 2808157..4ac86d3 100644
--- a/mwparserfromhell/parser/tokenizer.c
+++ b/mwparserfromhell/parser/tokenizer.c
@@ -680,11 +680,13 @@ Tokenizer_handle_template_param(Tokenizer* self)
         if (Tokenizer_verify_safe(self, unsafes))
             return -1;
         if (BAD_ROUTE) return -1;
-        if (Tokenizer_set_context(self, context ^ LC_TEMPLATE_NAME))
+        context ^= LC_TEMPLATE_NAME;
+        if (Tokenizer_set_context(self, context))
             return -1;
     }
     else if (context & LC_TEMPLATE_PARAM_VALUE) {
-        if (Tokenizer_set_context(self, context ^ LC_TEMPLATE_PARAM_VALUE))
+        context ^= LC_TEMPLATE_PARAM_VALUE;
+        if (Tokenizer_set_context(self, context))
             return -1;
     }
 
@@ -698,7 +700,8 @@ Tokenizer_handle_template_param(Tokenizer* self)
         Py_DECREF(stack);
     }
     else {
-        if (Tokenizer_set_context(self, context | LC_TEMPLATE_PARAM_KEY))
+        context |= LC_TEMPLATE_PARAM_KEY;
+        if (Tokenizer_set_context(self, context))
             return -1;
     }
 
@@ -714,7 +717,7 @@ Tokenizer_handle_template_param(Tokenizer* self)
     }
     Py_DECREF(token);
 
-    Tokenizer_push(self, Tokenizer_CONTEXT_VAL(self));
+    Tokenizer_push(self, context);
     return 0;
 }
 

From d2868731e85486fe5d2b900c95bf7812793d452f Mon Sep 17 00:00:00 2001
From: Ben Kurtovic <ben.kurtovic@verizon.net>
Date: Sat, 6 Oct 2012 22:56:23 -0400
Subject: [PATCH 28/47] Update some abstract protocols.

---
 mwparserfromhell/parser/tokenizer.c | 24 ++++++++++++------------
 1 file changed, 12 insertions(+), 12 deletions(-)

diff --git a/mwparserfromhell/parser/tokenizer.c b/mwparserfromhell/parser/tokenizer.c
index 4ac86d3..6550aad 100644
--- a/mwparserfromhell/parser/tokenizer.c
+++ b/mwparserfromhell/parser/tokenizer.c
@@ -106,7 +106,7 @@ Tokenizer_push(Tokenizer* self, Py_ssize_t context)
 static int
 Tokenizer_push_textbuffer(Tokenizer* self)
 {
-    if (PySequence_Fast_GET_SIZE(Tokenizer_TEXTBUFFER(self)) > 0) {
+    if (PyList_GET_SIZE(Tokenizer_TEXTBUFFER(self)) > 0) {
         PyObject* text = PyUnicode_Join(EMPTY, Tokenizer_TEXTBUFFER(self));
         if (!text) return -1;
 
@@ -149,9 +149,9 @@ Tokenizer_delete_top_of_stack(Tokenizer* self)
         return -1;
     Py_DECREF(self->topstack);
 
-    Py_ssize_t size = PySequence_Fast_GET_SIZE(self->stacks);
+    Py_ssize_t size = PyList_GET_SIZE(self->stacks);
     if (size > 0) {
-        PyObject* top = PySequence_Fast_GET_ITEM(self->stacks, size - 1);
+        PyObject* top = PyList_GET_ITEM(self->stacks, size - 1);
         self->topstack = top;
         Py_INCREF(top);
     }
@@ -265,8 +265,8 @@ Tokenizer_write_text(Tokenizer* self, PyObject* text)
 static int
 Tokenizer_write_all(Tokenizer* self, PyObject* tokenlist)
 {
-    if (PySequence_Fast_GET_SIZE(tokenlist) > 0) {
-        PyObject* token = PySequence_Fast_GET_ITEM(tokenlist, 0);
+    if (PyList_GET_SIZE(tokenlist) > 0) {
+        PyObject* token = PyList_GET_ITEM(tokenlist, 0);
         PyObject* class = PyObject_GetAttrString(tokens, "Text");
         if (!class) return -1;
 
@@ -303,7 +303,7 @@ Tokenizer_write_all(Tokenizer* self, PyObject* tokenlist)
         return -1;
 
     PyObject* stack = Tokenizer_STACK(self);
-    Py_ssize_t size = PySequence_Fast_GET_SIZE(stack);
+    Py_ssize_t size = PyList_GET_SIZE(stack);
 
     if (PyList_SetSlice(stack, size, size, tokenlist))
         return -1;
@@ -324,7 +324,7 @@ Tokenizer_write_text_then_stack(Tokenizer* self, PyObject* text)
     }
 
     if (stack) {
-        if (PySequence_Fast_GET_SIZE(stack) > 0) {
+        if (PyList_GET_SIZE(stack) > 0) {
             if (Tokenizer_write_all(self, stack)) {
                 Py_DECREF(stack);
                 return -1;
@@ -348,7 +348,7 @@ Tokenizer_read(Tokenizer* self, Py_ssize_t delta)
     if (index >= self->length)
         return EMPTY;
 
-    return PySequence_Fast_GET_ITEM(self->text, index);
+    return PyList_GET_ITEM(self->text, index);
 }
 
 /*
@@ -361,7 +361,7 @@ Tokenizer_read_backwards(Tokenizer* self, Py_ssize_t delta)
         return EMPTY;
 
     Py_ssize_t index = self->head - delta;
-    return PySequence_Fast_GET_ITEM(self->text, index);
+    return PyList_GET_ITEM(self->text, index);
 }
 
 /*
@@ -591,11 +591,11 @@ Tokenizer_verify_safe(Tokenizer* self, const char* unsafes[])
         }
 
         int i;
-        Py_ssize_t length = PySequence_Fast_GET_SIZE(stack);
+        Py_ssize_t length = PyList_GET_SIZE(stack);
         PyObject *token, *textdata;
 
         for (i = 0; i < length; i++) {
-            token = PySequence_Fast_GET_ITEM(stack, i);
+            token = PyList_GET_ITEM(stack, i);
             switch (PyObject_IsInstance(token, class)) {
                 case 0:
                     break;
@@ -1432,7 +1432,7 @@ Tokenizer_tokenize(Tokenizer* self, PyObject* args)
         self->text = PySequence_Fast(text, "expected a sequence");
     }
 
-    self->length = PySequence_Length(self->text);
+    self->length = PyList_GET_SIZE(self->text);
 
     return Tokenizer_parse(self, 0);
 }

From 6d73eeeab1aeaa42c62af99745b9d77d29feb6a6 Mon Sep 17 00:00:00 2001
From: Ben Kurtovic <ben.kurtovic@verizon.net>
Date: Sun, 7 Oct 2012 00:12:29 -0400
Subject: [PATCH 29/47] Replace Python list of stacks with a singly linked
 list.

---
 mwparserfromhell/parser/tokenizer.c | 198 ++++++++++++------------------------
 mwparserfromhell/parser/tokenizer.h |  72 ++++++-------
 2 files changed, 103 insertions(+), 167 deletions(-)

diff --git a/mwparserfromhell/parser/tokenizer.c b/mwparserfromhell/parser/tokenizer.c
index 6550aad..d7c206d 100644
--- a/mwparserfromhell/parser/tokenizer.c
+++ b/mwparserfromhell/parser/tokenizer.c
@@ -34,8 +34,14 @@ static void
 Tokenizer_dealloc(Tokenizer* self)
 {
     Py_XDECREF(self->text);
-    Py_XDECREF(self->stacks);
-    Py_XDECREF(self->topstack);
+    struct Stack *this = self->topstack, *next;
+    while (this) {
+        Py_DECREF(this->stack);
+        Py_DECREF(this->textbuffer);
+        next = this->next;
+        free(this);
+        this = next;
+    }
     self->ob_type->tp_free((PyObject*) self);
 }
 
@@ -47,57 +53,26 @@ Tokenizer_init(Tokenizer* self, PyObject* args, PyObject* kwds)
         return -1;
 
     self->text = Py_None;
-    self->topstack = Py_None;
-    Py_INCREF(Py_None);
     Py_INCREF(Py_None);
-
-    self->stacks = PyList_New(0);
-    if (!self->stacks) {
-        Py_DECREF(self);
-        return -1;
-    }
-
+    self->topstack = NULL;
     self->head = 0;
     self->length = 0;
     self->global = 0;
-
-    return 0;
-}
-
-static int
-Tokenizer_set_context(Tokenizer* self, Py_ssize_t value)
-{
-    if (PyList_SetItem(self->topstack, 1, PyInt_FromSsize_t(value)))
-        return -1;
-    return 0;
-}
-
-static int
-Tokenizer_set_textbuffer(Tokenizer* self, PyObject* value)
-{
-    if (PyList_SetItem(self->topstack, 2, value))
-        return -1;
     return 0;
 }
 
 /*
     Add a new token stack, context, and textbuffer to the list.
 */
-static int
-Tokenizer_push(Tokenizer* self, Py_ssize_t context)
+static void
+Tokenizer_push(Tokenizer* self, int context)
 {
-    PyObject* top = PyList_New(3);
-    if (!top) return -1;
-    PyList_SET_ITEM(top, 0, PyList_New(0));
-    PyList_SET_ITEM(top, 1, PyInt_FromSsize_t(context));
-    PyList_SET_ITEM(top, 2, PyList_New(0));
-
-    Py_XDECREF(self->topstack);
+    struct Stack* top = malloc(sizeof(struct Stack));
+    top->stack = PyList_New(0);
+    top->context = context;
+    top->textbuffer = PyList_New(0);
+    top->next = self->topstack;
     self->topstack = top;
-
-    if (PyList_Append(self->stacks, top))
-        return -1;
-    return 0;
 }
 
 /*
@@ -106,8 +81,8 @@ Tokenizer_push(Tokenizer* self, Py_ssize_t context)
 static int
 Tokenizer_push_textbuffer(Tokenizer* self)
 {
-    if (PyList_GET_SIZE(Tokenizer_TEXTBUFFER(self)) > 0) {
-        PyObject* text = PyUnicode_Join(EMPTY, Tokenizer_TEXTBUFFER(self));
+    if (PyList_GET_SIZE(self->topstack->textbuffer) > 0) {
+        PyObject* text = PyUnicode_Join(EMPTY, self->topstack->textbuffer);
         if (!text) return -1;
 
         PyObject* class = PyObject_GetAttrString(tokens, "Text");
@@ -129,37 +104,28 @@ Tokenizer_push_textbuffer(Tokenizer* self)
         Py_DECREF(kwargs);
         if (!token) return -1;
 
-        if (PyList_Append(Tokenizer_STACK(self), token)) {
+        if (PyList_Append(self->topstack->stack, token)) {
             Py_DECREF(token);
             return -1;
         }
 
         Py_DECREF(token);
 
-        if (Tokenizer_set_textbuffer(self, PyList_New(0)))
+        self->topstack->textbuffer = PyList_New(0);
+        if (!self->topstack->textbuffer)
             return -1;
     }
     return 0;
 }
 
-static int
+static void
 Tokenizer_delete_top_of_stack(Tokenizer* self)
 {
-    if (PySequence_DelItem(self->stacks, -1))
-        return -1;
-    Py_DECREF(self->topstack);
-
-    Py_ssize_t size = PyList_GET_SIZE(self->stacks);
-    if (size > 0) {
-        PyObject* top = PyList_GET_ITEM(self->stacks, size - 1);
-        self->topstack = top;
-        Py_INCREF(top);
-    }
-    else {
-        self->topstack = NULL;
-    }
-
-    return 0;
+    struct Stack* top = self->topstack;
+    Py_DECREF(top->stack);
+    Py_DECREF(top->textbuffer);
+    self->topstack = top->next;
+    free(top);
 }
 
 /*
@@ -171,12 +137,10 @@ Tokenizer_pop(Tokenizer* self)
     if (Tokenizer_push_textbuffer(self))
         return NULL;
 
-    PyObject* stack = Tokenizer_STACK(self);
+    PyObject* stack = self->topstack->stack;
     Py_INCREF(stack);
 
-    if (Tokenizer_delete_top_of_stack(self))
-        return NULL;
-
+    Tokenizer_delete_top_of_stack(self);
     return stack;
 }
 
@@ -190,17 +154,12 @@ Tokenizer_pop_keeping_context(Tokenizer* self)
     if (Tokenizer_push_textbuffer(self))
         return NULL;
 
-    PyObject* stack = Tokenizer_STACK(self);
-    PyObject* context = Tokenizer_CONTEXT(self);
+    PyObject* stack = self->topstack->stack;
     Py_INCREF(stack);
-    Py_INCREF(context);
-
-    if (Tokenizer_delete_top_of_stack(self))
-        return NULL;
-
-    if (PyList_SetItem(self->topstack, 1, context))
-        return NULL;
+    int context = self->topstack->context;
 
+    Tokenizer_delete_top_of_stack(self);
+    self->topstack->context = context;
     return stack;
 }
 
@@ -226,7 +185,7 @@ Tokenizer_write(Tokenizer* self, PyObject* token)
     if (Tokenizer_push_textbuffer(self))
         return -1;
 
-    if (PyList_Append(Tokenizer_STACK(self), token))
+    if (PyList_Append(self->topstack->stack, token))
         return -1;
 
     return 0;
@@ -241,7 +200,7 @@ Tokenizer_write_first(Tokenizer* self, PyObject* token)
     if (Tokenizer_push_textbuffer(self))
         return -1;
 
-    if (PyList_Insert(Tokenizer_STACK(self), 0, token))
+    if (PyList_Insert(self->topstack->stack, 0, token))
         return -1;
 
     return 0;
@@ -253,7 +212,7 @@ Tokenizer_write_first(Tokenizer* self, PyObject* token)
 static int
 Tokenizer_write_text(Tokenizer* self, PyObject* text)
 {
-    if (PyList_Append(Tokenizer_TEXTBUFFER(self), text))
+    if (PyList_Append(self->topstack->textbuffer, text))
         return -1;
 
     return 0;
@@ -302,7 +261,7 @@ Tokenizer_write_all(Tokenizer* self, PyObject* tokenlist)
     if (Tokenizer_push_textbuffer(self))
         return -1;
 
-    PyObject* stack = Tokenizer_STACK(self);
+    PyObject* stack = self->topstack->stack;
     Py_ssize_t size = PyList_GET_SIZE(stack);
 
     if (PyList_SetSlice(stack, size, size, tokenlist))
@@ -579,7 +538,7 @@ Tokenizer_verify_safe(Tokenizer* self, const char* unsafes[])
     if (Tokenizer_push_textbuffer(self))
         return -1;
 
-    PyObject* stack = Tokenizer_STACK(self);
+    PyObject* stack = self->topstack->stack;
     if (stack) {
         PyObject* textlist = PyList_New(0);
         if (!textlist) return -1;
@@ -673,24 +632,18 @@ Tokenizer_verify_safe(Tokenizer* self, const char* unsafes[])
 static int
 Tokenizer_handle_template_param(Tokenizer* self)
 {
-    Py_ssize_t context = Tokenizer_CONTEXT_VAL(self);
-
-    if (context & LC_TEMPLATE_NAME) {
+    if (self->topstack->context & LC_TEMPLATE_NAME) {
         const char* unsafes[] = {"\n", "{", "}", "[", "]", NULL};
         if (Tokenizer_verify_safe(self, unsafes))
             return -1;
         if (BAD_ROUTE) return -1;
-        context ^= LC_TEMPLATE_NAME;
-        if (Tokenizer_set_context(self, context))
-            return -1;
+        self->topstack->context ^= LC_TEMPLATE_NAME;
     }
-    else if (context & LC_TEMPLATE_PARAM_VALUE) {
-        context ^= LC_TEMPLATE_PARAM_VALUE;
-        if (Tokenizer_set_context(self, context))
-            return -1;
+    else if (self->topstack->context & LC_TEMPLATE_PARAM_VALUE) {
+        self->topstack->context ^= LC_TEMPLATE_PARAM_VALUE;
     }
 
-    if (context & LC_TEMPLATE_PARAM_KEY) {
+    if (self->topstack->context & LC_TEMPLATE_PARAM_KEY) {
         PyObject* stack = Tokenizer_pop_keeping_context(self);
         if (!stack) return -1;
         if (Tokenizer_write_all(self, stack)) {
@@ -700,9 +653,7 @@ Tokenizer_handle_template_param(Tokenizer* self)
         Py_DECREF(stack);
     }
     else {
-        context |= LC_TEMPLATE_PARAM_KEY;
-        if (Tokenizer_set_context(self, context))
-            return -1;
+        self->topstack->context |= LC_TEMPLATE_PARAM_KEY;
     }
 
     PyObject* class = PyObject_GetAttrString(tokens, "TemplateParamSeparator");
@@ -717,7 +668,7 @@ Tokenizer_handle_template_param(Tokenizer* self)
     }
     Py_DECREF(token);
 
-    Tokenizer_push(self, context);
+    Tokenizer_push(self, self->topstack->context);
     return 0;
 }
 
@@ -744,11 +695,8 @@ Tokenizer_handle_template_param_value(Tokenizer* self)
     }
     Py_DECREF(stack);
 
-    Py_ssize_t context = Tokenizer_CONTEXT_VAL(self);
-    context ^= LC_TEMPLATE_PARAM_KEY;
-    context |= LC_TEMPLATE_PARAM_VALUE;
-    if (Tokenizer_set_context(self, context))
-        return -1;
+    self->topstack->context ^= LC_TEMPLATE_PARAM_KEY;
+    self->topstack->context |= LC_TEMPLATE_PARAM_VALUE;
 
     PyObject* class = PyObject_GetAttrString(tokens, "TemplateParamEquals");
     if (!class) return -1;
@@ -771,14 +719,12 @@ static PyObject*
 Tokenizer_handle_template_end(Tokenizer* self)
 {
     PyObject* stack;
-    Py_ssize_t context = Tokenizer_CONTEXT_VAL(self);
-
-    if (context & LC_TEMPLATE_NAME) {
+    if (self->topstack->context & LC_TEMPLATE_NAME) {
         const char* unsafes[] = {"\n", "{", "}", "[", "]", NULL};
         if (Tokenizer_verify_safe(self, unsafes))
             return NULL;
     }
-    else if (context & LC_TEMPLATE_PARAM_KEY) {
+    else if (self->topstack->context & LC_TEMPLATE_PARAM_KEY) {
         stack = Tokenizer_pop_keeping_context(self);
         if (!stack) return NULL;
         if (Tokenizer_write_all(self, stack)) {
@@ -803,11 +749,8 @@ Tokenizer_handle_argument_separator(Tokenizer* self)
     if (Tokenizer_verify_safe(self, unsafes))
         return -1;
 
-    Py_ssize_t context = Tokenizer_CONTEXT_VAL(self);
-    context ^= LC_ARGUMENT_NAME;
-    context |= LC_ARGUMENT_DEFAULT;
-    if (Tokenizer_set_context(self, context))
-        return -1;
+    self->topstack->context ^= LC_ARGUMENT_NAME;
+    self->topstack->context |= LC_ARGUMENT_DEFAULT;
 
     PyObject* class = PyObject_GetAttrString(tokens, "ArgumentSeparator");
     if (!class) return -1;
@@ -829,7 +772,7 @@ Tokenizer_handle_argument_separator(Tokenizer* self)
 static PyObject*
 Tokenizer_handle_argument_end(Tokenizer* self)
 {
-    if (Tokenizer_CONTEXT_VAL(self) & LC_ARGUMENT_NAME) {
+    if (self->topstack->context & LC_ARGUMENT_NAME) {
         const char* unsafes[] = {"\n", "{{", "}}", NULL};
         if (Tokenizer_verify_safe(self, unsafes))
             return NULL;
@@ -914,11 +857,8 @@ Tokenizer_handle_wikilink_separator(Tokenizer* self)
     if (Tokenizer_verify_safe(self, unsafes))
         return -1;
 
-    Py_ssize_t context = Tokenizer_CONTEXT_VAL(self);
-    context ^= LC_WIKILINK_TITLE;
-    context |= LC_WIKILINK_TEXT;
-    if (Tokenizer_set_context(self, context))
-        return -1;
+    self->topstack->context ^= LC_WIKILINK_TITLE;
+    self->topstack->context |= LC_WIKILINK_TEXT;
 
     PyObject* class = PyObject_GetAttrString(tokens, "WikilinkSeparator");
     if (!class) return -1;
@@ -940,7 +880,7 @@ Tokenizer_handle_wikilink_separator(Tokenizer* self)
 static PyObject*
 Tokenizer_handle_wikilink_end(Tokenizer* self)
 {
-    if (Tokenizer_CONTEXT_VAL(self) & LC_WIKILINK_TITLE) {
+    if (self->topstack->context & LC_WIKILINK_TITLE) {
         const char* unsafes[] = {"\n", "{", "}", "[", "]", NULL};
         if (Tokenizer_verify_safe(self, unsafes))
             return NULL;
@@ -960,7 +900,7 @@ Tokenizer_parse_heading(Tokenizer* self)
     self->global |= GL_HEADING;
     Py_ssize_t reset = self->head;
     self->head += 1;
-    Py_ssize_t best = 1;
+    int best = 1;
     PyObject* text;
     int i;
 
@@ -969,7 +909,7 @@ Tokenizer_parse_heading(Tokenizer* self)
         self->head++;
     }
 
-    Py_ssize_t context = LC_HEADING_LEVEL_1 << (best > 5 ? 5 : best - 1);
+    int context = LC_HEADING_LEVEL_1 << (best > 5 ? 5 : best - 1);
     HeadingData* heading = (HeadingData*) Tokenizer_parse(self, context);
 
     if (BAD_ROUTE) {
@@ -1032,7 +972,7 @@ Tokenizer_parse_heading(Tokenizer* self)
     Py_DECREF(token);
 
     if (heading->level < best) {
-        Py_ssize_t diff = best - heading->level;
+        int diff = best - heading->level;
         char diffblocks[diff];
         for (i = 0; i < diff; i++) diffblocks[i] = *"=";
         PyObject* text = PyUnicode_FromStringAndSize(diffblocks, diff);
@@ -1092,16 +1032,14 @@ Tokenizer_handle_heading_end(Tokenizer* self)
         self->head++;
     }
 
-    Py_ssize_t current = log2(Tokenizer_CONTEXT_VAL(self) / LC_HEADING_LEVEL_1) + 1;
-    Py_ssize_t level = current > best ? (best > 6 ? 6 : best) : (current > 6 ? 6 : current);
-
-    Py_ssize_t context = Tokenizer_CONTEXT_VAL(self);
-    HeadingData* after = (HeadingData*) Tokenizer_parse(self, context);
+    int current = log2(self->topstack->context / LC_HEADING_LEVEL_1) + 1;
+    int level = current > best ? (best > 6 ? 6 : best) : (current > 6 ? 6 : current);
+    HeadingData* after = (HeadingData*) Tokenizer_parse(self, self->topstack->context);
 
     if (BAD_ROUTE) {
         RESET_ROUTE();
         if (level < best) {
-            Py_ssize_t diff = best - level;
+            int diff = best - level;
             char diffblocks[diff];
             for (i = 0; i < diff; i++) diffblocks[i] = *"=";
             text = PyUnicode_FromStringAndSize(diffblocks, diff);
@@ -1174,8 +1112,7 @@ static int
 Tokenizer_parse_entity(Tokenizer* self)
 {
     Py_ssize_t reset = self->head;
-    if (Tokenizer_push(self, 0))
-        return -1;
+    Tokenizer_push(self, 0);
 
     if (Tokenizer_really_parse_entity(self))
             return -1;
@@ -1268,13 +1205,12 @@ Tokenizer_parse_comment(Tokenizer* self)
     Parse the wikicode string, using context for when to stop.
 */
 static PyObject*
-Tokenizer_parse(Tokenizer* self, Py_ssize_t context)
+Tokenizer_parse(Tokenizer* self, int context)
 {
     PyObject *this;
     Py_UNICODE this_data, next, next_next, last;
-    Py_ssize_t this_context;
-    Py_ssize_t fail_contexts = (
-        LC_TEMPLATE | LC_ARGUMENT | LC_WIKILINK | LC_HEADING | LC_COMMENT);
+    int this_context;
+    int fail_contexts = LC_TEMPLATE | LC_ARGUMENT | LC_WIKILINK | LC_HEADING | LC_COMMENT;
     int is_marker, i;
 
     Tokenizer_push(self, context);
@@ -1297,7 +1233,7 @@ Tokenizer_parse(Tokenizer* self, Py_ssize_t context)
             continue;
         }
 
-        this_context = Tokenizer_CONTEXT_VAL(self);
+        this_context = self->topstack->context;
 
         if (this_data == *"") {
             if (this_context & LC_TEMPLATE_PARAM_KEY) {
diff --git a/mwparserfromhell/parser/tokenizer.h b/mwparserfromhell/parser/tokenizer.h
index 91da10e..1b9b76a 100644
--- a/mwparserfromhell/parser/tokenizer.h
+++ b/mwparserfromhell/parser/tokenizer.h
@@ -46,40 +46,47 @@ static PyObject* tokens;
 
 /* Local contexts: */
 
-static const Py_ssize_t LC_TEMPLATE =             0x0007;
-static const Py_ssize_t LC_TEMPLATE_NAME =        0x0001;
-static const Py_ssize_t LC_TEMPLATE_PARAM_KEY =   0x0002;
-static const Py_ssize_t LC_TEMPLATE_PARAM_VALUE = 0x0004;
+static const int LC_TEMPLATE =             0x0007;
+static const int LC_TEMPLATE_NAME =        0x0001;
+static const int LC_TEMPLATE_PARAM_KEY =   0x0002;
+static const int LC_TEMPLATE_PARAM_VALUE = 0x0004;
 
-static const Py_ssize_t LC_ARGUMENT =             0x0018;
-static const Py_ssize_t LC_ARGUMENT_NAME =        0x0008;
-static const Py_ssize_t LC_ARGUMENT_DEFAULT =     0x0010;
+static const int LC_ARGUMENT =             0x0018;
+static const int LC_ARGUMENT_NAME =        0x0008;
+static const int LC_ARGUMENT_DEFAULT =     0x0010;
 
-static const Py_ssize_t LC_WIKILINK =             0x0060;
-static const Py_ssize_t LC_WIKILINK_TITLE =       0x0020;
-static const Py_ssize_t LC_WIKILINK_TEXT =        0x0040;
+static const int LC_WIKILINK =             0x0060;
+static const int LC_WIKILINK_TITLE =       0x0020;
+static const int LC_WIKILINK_TEXT =        0x0040;
 
-static const Py_ssize_t LC_HEADING =              0x1f80;
-static const Py_ssize_t LC_HEADING_LEVEL_1 =      0x0080;
-static const Py_ssize_t LC_HEADING_LEVEL_2 =      0x0100;
-static const Py_ssize_t LC_HEADING_LEVEL_3 =      0x0200;
-static const Py_ssize_t LC_HEADING_LEVEL_4 =      0x0400;
-static const Py_ssize_t LC_HEADING_LEVEL_5 =      0x0800;
-static const Py_ssize_t LC_HEADING_LEVEL_6 =      0x1000;
+static const int LC_HEADING =              0x1f80;
+static const int LC_HEADING_LEVEL_1 =      0x0080;
+static const int LC_HEADING_LEVEL_2 =      0x0100;
+static const int LC_HEADING_LEVEL_3 =      0x0200;
+static const int LC_HEADING_LEVEL_4 =      0x0400;
+static const int LC_HEADING_LEVEL_5 =      0x0800;
+static const int LC_HEADING_LEVEL_6 =      0x1000;
 
-static const Py_ssize_t LC_COMMENT =              0x2000;
+static const int LC_COMMENT =              0x2000;
 
 
 /* Global contexts: */
 
-static const Py_ssize_t GL_HEADING = 0x1;
+static const int GL_HEADING = 0x1;
 
 
 /* Miscellaneous structs: */
 
+struct Stack {
+    PyObject* stack;
+    int context;
+    PyObject* textbuffer;
+    struct Stack* next;
+};
+
 typedef struct {
     PyObject* title;
-    Py_ssize_t level;
+    int level;
 } HeadingData;
 
 
@@ -87,22 +94,17 @@ typedef struct {
 
 typedef struct {
     PyObject_HEAD
-    PyObject* text;        /* text to tokenize */
-    PyObject* stacks;      /* token stacks */
-    PyObject* topstack;    /* topmost stack */
-    Py_ssize_t head;       /* current position in text */
-    Py_ssize_t length;     /* length of text */
-    Py_ssize_t global;     /* global context */
+    PyObject* text;         /* text to tokenize */
+    struct Stack* topstack; /* topmost stack */
+    Py_ssize_t head;        /* current position in text */
+    Py_ssize_t length;      /* length of text */
+    int global;             /* global context */
 } Tokenizer;
 
 
 /* Macros for accessing Tokenizer data: */
 
-#define Tokenizer_STACK(self) PySequence_Fast_GET_ITEM(self->topstack, 0)
-#define Tokenizer_CONTEXT(self) PySequence_Fast_GET_ITEM(self->topstack, 1)
-#define Tokenizer_CONTEXT_VAL(self) PyInt_AsSsize_t(Tokenizer_CONTEXT(self))
-#define Tokenizer_TEXTBUFFER(self) PySequence_Fast_GET_ITEM(self->topstack, 2)
-#define Tokenizer_READ(self, num) PyUnicode_AS_UNICODE(Tokenizer_read(self, num))
+#define Tokenizer_READ(self, delta) PyUnicode_AS_UNICODE(Tokenizer_read(self, delta))
 
 
 /* Tokenizer function prototypes: */
@@ -110,11 +112,9 @@ typedef struct {
 static PyObject* Tokenizer_new(PyTypeObject*, PyObject*, PyObject*);
 static void Tokenizer_dealloc(Tokenizer*);
 static int Tokenizer_init(Tokenizer*, PyObject*, PyObject*);
-static int Tokenizer_set_context(Tokenizer*, Py_ssize_t);
-static int Tokenizer_set_textbuffer(Tokenizer*, PyObject*);
-static int Tokenizer_push(Tokenizer*, Py_ssize_t);
+static void Tokenizer_push(Tokenizer*, int);
 static int Tokenizer_push_textbuffer(Tokenizer*);
-static int Tokenizer_delete_top_of_stack(Tokenizer*);
+static void Tokenizer_delete_top_of_stack(Tokenizer*);
 static PyObject* Tokenizer_pop(Tokenizer*);
 static PyObject* Tokenizer_pop_keeping_context(Tokenizer*);
 static void* Tokenizer_fail_route(Tokenizer*);
@@ -142,7 +142,7 @@ static HeadingData* Tokenizer_handle_heading_end(Tokenizer*);
 static int Tokenizer_really_parse_entity(Tokenizer*);
 static int Tokenizer_parse_entity(Tokenizer*);
 static int Tokenizer_parse_comment(Tokenizer*);
-static PyObject* Tokenizer_parse(Tokenizer*, Py_ssize_t);
+static PyObject* Tokenizer_parse(Tokenizer*, int);
 static PyObject* Tokenizer_tokenize(Tokenizer*, PyObject*);
 
 

From f0a36f32628d4514fb10d095f8c1734ec8529c3f Mon Sep 17 00:00:00 2001
From: Ben Kurtovic <ben.kurtovic@verizon.net>
Date: Sun, 7 Oct 2012 23:39:18 -0400
Subject: [PATCH 30/47] Store tokens in a global variable instead of retrieving
 anew each time.

---
 mwparserfromhell/parser/tokenizer.c | 161 +++++++++++++-----------------------
 mwparserfromhell/parser/tokenizer.h |  37 +++++++++
 2 files changed, 93 insertions(+), 105 deletions(-)

diff --git a/mwparserfromhell/parser/tokenizer.c b/mwparserfromhell/parser/tokenizer.c
index d7c206d..547207a 100644
--- a/mwparserfromhell/parser/tokenizer.c
+++ b/mwparserfromhell/parser/tokenizer.c
@@ -85,22 +85,15 @@ Tokenizer_push_textbuffer(Tokenizer* self)
         PyObject* text = PyUnicode_Join(EMPTY, self->topstack->textbuffer);
         if (!text) return -1;
 
-        PyObject* class = PyObject_GetAttrString(tokens, "Text");
-        if (!class) {
-            Py_DECREF(text);
-            return -1;
-        }
         PyObject* kwargs = PyDict_New();
         if (!kwargs) {
-            Py_DECREF(class);
             Py_DECREF(text);
             return -1;
         }
         PyDict_SetItemString(kwargs, "text", text);
         Py_DECREF(text);
 
-        PyObject* token = PyObject_Call(class, NOARGS, kwargs);
-        Py_DECREF(class);
+        PyObject* token = PyObject_Call(Text, NOARGS, kwargs);
         Py_DECREF(kwargs);
         if (!token) return -1;
 
@@ -226,36 +219,29 @@ Tokenizer_write_all(Tokenizer* self, PyObject* tokenlist)
 {
     if (PyList_GET_SIZE(tokenlist) > 0) {
         PyObject* token = PyList_GET_ITEM(tokenlist, 0);
-        PyObject* class = PyObject_GetAttrString(tokens, "Text");
-        if (!class) return -1;
 
         PyObject* text;
-        switch (PyObject_IsInstance(token, class)) {
+        switch (PyObject_IsInstance(token, Text)) {
             case 0:
                 break;
             case 1:
                 text = PyObject_GetAttrString(token, "text");
                 if (!text) {
-                    Py_DECREF(class);
                     return -1;
                 }
                 if (PySequence_DelItem(tokenlist, 0)) {
                     Py_DECREF(text);
-                    Py_DECREF(class);
                     return -1;
                 }
                 if (Tokenizer_write_text(self, text)) {
                     Py_DECREF(text);
-                    Py_DECREF(class);
                     return -1;
                 }
                 Py_DECREF(text);
                 break;
             case -1:
-                Py_DECREF(class);
                 return -1;
         }
-        Py_DECREF(class);
     }
 
     if (Tokenizer_push_textbuffer(self))
@@ -420,7 +406,7 @@ Tokenizer_parse_template_or_argument(Tokenizer* self)
 static int
 Tokenizer_parse_template(Tokenizer* self)
 {
-    PyObject *template, *class, *token;
+    PyObject *template, *token;
     Py_ssize_t reset = self->head;
 
     template = Tokenizer_parse(self, LC_TEMPLATE_NAME);
@@ -430,13 +416,7 @@ Tokenizer_parse_template(Tokenizer* self)
     }
     if (!template) return -1;
 
-    class = PyObject_GetAttrString(tokens, "TemplateOpen");
-    if (!class) {
-        Py_DECREF(template);
-        return -1;
-    }
-    token = PyObject_CallObject(class, NULL);
-    Py_DECREF(class);
+    token = PyObject_CallObject(TemplateOpen, NULL);
     if (!token) {
         Py_DECREF(template);
         return -1;
@@ -455,10 +435,7 @@ Tokenizer_parse_template(Tokenizer* self)
     }
     Py_DECREF(template);
 
-    class = PyObject_GetAttrString(tokens, "TemplateClose");
-    if (!class) return -1;
-    token = PyObject_CallObject(class, NULL);
-    Py_DECREF(class);
+    token = PyObject_CallObject(TemplateClose, NULL);
     if (!token) return -1;
 
     if (Tokenizer_write(self, token)) {
@@ -476,7 +453,7 @@ Tokenizer_parse_template(Tokenizer* self)
 static int
 Tokenizer_parse_argument(Tokenizer* self)
 {
-    PyObject *argument, *class, *token;
+    PyObject *argument, *token;
     Py_ssize_t reset = self->head;
 
     argument = Tokenizer_parse(self, LC_ARGUMENT_NAME);
@@ -486,13 +463,7 @@ Tokenizer_parse_argument(Tokenizer* self)
     }
     if (!argument) return -1;
 
-    class = PyObject_GetAttrString(tokens, "ArgumentOpen");
-    if (!class) {
-        Py_DECREF(argument);
-        return -1;
-    }
-    token = PyObject_CallObject(class, NULL);
-    Py_DECREF(class);
+    token = PyObject_CallObject(ArgumentOpen, NULL);
     if (!token) {
         Py_DECREF(argument);
         return -1;
@@ -511,10 +482,7 @@ Tokenizer_parse_argument(Tokenizer* self)
     }
     Py_DECREF(argument);
 
-    class = PyObject_GetAttrString(tokens, "ArgumentClose");
-    if (!class) return -1;
-    token = PyObject_CallObject(class, NULL);
-    Py_DECREF(class);
+    token = PyObject_CallObject(ArgumentClose, NULL);
     if (!token) return -1;
 
     if (Tokenizer_write(self, token)) {
@@ -543,31 +511,23 @@ Tokenizer_verify_safe(Tokenizer* self, const char* unsafes[])
         PyObject* textlist = PyList_New(0);
         if (!textlist) return -1;
 
-        PyObject* class = PyObject_GetAttrString(tokens, "Text");
-        if (!class) {
-            Py_DECREF(textlist);
-            return -1;
-        }
-
         int i;
         Py_ssize_t length = PyList_GET_SIZE(stack);
         PyObject *token, *textdata;
 
         for (i = 0; i < length; i++) {
             token = PyList_GET_ITEM(stack, i);
-            switch (PyObject_IsInstance(token, class)) {
+            switch (PyObject_IsInstance(token, Text)) {
                 case 0:
                     break;
                 case 1:
                     textdata = PyObject_GetAttrString(token, "text");
                     if (!textdata) {
                         Py_DECREF(textlist);
-                        Py_DECREF(class);
                         return -1;
                     }
                     if (PyList_Append(textlist, textdata)) {
                         Py_DECREF(textlist);
-                        Py_DECREF(class);
                         Py_DECREF(textdata);
                         return -1;
                     }
@@ -575,11 +535,9 @@ Tokenizer_verify_safe(Tokenizer* self, const char* unsafes[])
                     break;
                 case -1:
                     Py_DECREF(textlist);
-                    Py_DECREF(class);
                     return -1;
             }
         }
-        Py_DECREF(class);
 
         PyObject* text = PyUnicode_Join(EMPTY, textlist);
         if (!text) {
@@ -656,10 +614,7 @@ Tokenizer_handle_template_param(Tokenizer* self)
         self->topstack->context |= LC_TEMPLATE_PARAM_KEY;
     }
 
-    PyObject* class = PyObject_GetAttrString(tokens, "TemplateParamSeparator");
-    if (!class) return -1;
-    PyObject* token = PyObject_CallObject(class, NULL);
-    Py_DECREF(class);
+    PyObject* token = PyObject_CallObject(TemplateParamSeparator, NULL);
     if (!token) return -1;
 
     if (Tokenizer_write(self, token)) {
@@ -698,10 +653,7 @@ Tokenizer_handle_template_param_value(Tokenizer* self)
     self->topstack->context ^= LC_TEMPLATE_PARAM_KEY;
     self->topstack->context |= LC_TEMPLATE_PARAM_VALUE;
 
-    PyObject* class = PyObject_GetAttrString(tokens, "TemplateParamEquals");
-    if (!class) return -1;
-    PyObject* token = PyObject_CallObject(class, NULL);
-    Py_DECREF(class);
+    PyObject* token = PyObject_CallObject(TemplateParamEquals, NULL);
     if (!token) return -1;
 
     if (Tokenizer_write(self, token)) {
@@ -752,10 +704,7 @@ Tokenizer_handle_argument_separator(Tokenizer* self)
     self->topstack->context ^= LC_ARGUMENT_NAME;
     self->topstack->context |= LC_ARGUMENT_DEFAULT;
 
-    PyObject* class = PyObject_GetAttrString(tokens, "ArgumentSeparator");
-    if (!class) return -1;
-    PyObject* token = PyObject_CallObject(class, NULL);
-    Py_DECREF(class);
+    PyObject* token = PyObject_CallObject(ArgumentSeparator, NULL);
     if (!token) return -1;
 
     if (Tokenizer_write(self, token)) {
@@ -792,7 +741,7 @@ Tokenizer_parse_wikilink(Tokenizer* self)
     self->head += 2;
     Py_ssize_t reset = self->head - 1;
 
-    PyObject *class, *token;
+    PyObject *token;
     PyObject *wikilink = Tokenizer_parse(self, LC_WIKILINK_TITLE);
     if (!wikilink) return -1;
 
@@ -808,13 +757,7 @@ Tokenizer_parse_wikilink(Tokenizer* self)
         return 0;
     }
 
-    class = PyObject_GetAttrString(tokens, "WikilinkOpen");
-    if (!class) {
-        Py_DECREF(wikilink);
-        return -1;
-    }
-    token = PyObject_CallObject(class, NULL);
-    Py_DECREF(class);
+    token = PyObject_CallObject(WikilinkOpen, NULL);
     if (!token) {
         Py_DECREF(wikilink);
         return -1;
@@ -833,10 +776,7 @@ Tokenizer_parse_wikilink(Tokenizer* self)
     }
     Py_DECREF(wikilink);
 
-    class = PyObject_GetAttrString(tokens, "WikilinkClose");
-    if (!class) return -1;
-    token = PyObject_CallObject(class, NULL);
-    Py_DECREF(class);
+    token = PyObject_CallObject(WikilinkClose, NULL);
     if (!token) return -1;
 
     if (Tokenizer_write(self, token)) {
@@ -860,10 +800,7 @@ Tokenizer_handle_wikilink_separator(Tokenizer* self)
     self->topstack->context ^= LC_WIKILINK_TITLE;
     self->topstack->context |= LC_WIKILINK_TEXT;
 
-    PyObject* class = PyObject_GetAttrString(tokens, "WikilinkSeparator");
-    if (!class) return -1;
-    PyObject* token = PyObject_CallObject(class, NULL);
-    Py_DECREF(class);
+    PyObject* token = PyObject_CallObject(WikilinkSeparator, NULL);
     if (!token) return -1;
 
     if (Tokenizer_write(self, token)) {
@@ -936,16 +873,8 @@ Tokenizer_parse_heading(Tokenizer* self)
         return -1;
     }
 
-    PyObject* class = PyObject_GetAttrString(tokens, "HeadingStart");
-    if (!class) {
-        Py_DECREF(level);
-        Py_DECREF(heading->title);
-        free(heading);
-        return -1;
-    }
     PyObject* kwargs = PyDict_New();
     if (!kwargs) {
-        Py_DECREF(class);
         Py_DECREF(level);
         Py_DECREF(heading->title);
         free(heading);
@@ -954,8 +883,7 @@ Tokenizer_parse_heading(Tokenizer* self)
     PyDict_SetItemString(kwargs, "level", level);
     Py_DECREF(level);
 
-    PyObject* token = PyObject_Call(class, NOARGS, kwargs);
-    Py_DECREF(class);
+    PyObject* token = PyObject_Call(HeadingStart, NOARGS, kwargs);
     Py_DECREF(kwargs);
     if (!token) {
         Py_DECREF(heading->title);
@@ -999,10 +927,7 @@ Tokenizer_parse_heading(Tokenizer* self)
     Py_DECREF(heading->title);
     free(heading);
 
-    class = PyObject_GetAttrString(tokens, "HeadingEnd");
-    if (!class) return -1;
-    token = PyObject_CallObject(class, NULL);
-    Py_DECREF(class);
+    token = PyObject_CallObject(HeadingEnd, NULL);
     if (!token) return -1;
 
     if (Tokenizer_write(self, token)) {
@@ -1145,7 +1070,7 @@ Tokenizer_parse_comment(Tokenizer* self)
     self->head += 4;
     Py_ssize_t reset = self->head - 1;
 
-    PyObject *class, *token;
+    PyObject *token;
     PyObject *comment = Tokenizer_parse(self, LC_WIKILINK_TITLE);
     if (!comment) return -1;
 
@@ -1161,13 +1086,7 @@ Tokenizer_parse_comment(Tokenizer* self)
         return 0;
     }
 
-    class = PyObject_GetAttrString(tokens, "CommentStart");
-    if (!class) {
-        Py_DECREF(comment);
-        return -1;
-    }
-    token = PyObject_CallObject(class, NULL);
-    Py_DECREF(class);
+    token = PyObject_CallObject(CommentStart, NULL);
     if (!token) {
         Py_DECREF(comment);
         return -1;
@@ -1186,10 +1105,7 @@ Tokenizer_parse_comment(Tokenizer* self)
     }
     Py_DECREF(comment);
 
-    class = PyObject_GetAttrString(tokens, "CommentEnd");
-    if (!class) return -1;
-    token = PyObject_CallObject(class, NULL);
-    Py_DECREF(class);
+    token = PyObject_CallObject(CommentEnd, NULL);
     if (!token) return -1;
 
     if (Tokenizer_write(self, token)) {
@@ -1410,4 +1326,39 @@ init_tokenizer(void)
 
     tokens = PyObject_GetAttrString(tokmodule, "tokens");
     Py_DECREF(tokmodule);
+
+    Text = PyObject_GetAttrString(tokens, "Text");
+
+    TemplateOpen = PyObject_GetAttrString(tokens, "TemplateOpen");
+    TemplateParamSeparator = PyObject_GetAttrString(tokens, "TemplateParamSeparator");
+    TemplateParamEquals = PyObject_GetAttrString(tokens, "TemplateParamEquals");
+    TemplateClose = PyObject_GetAttrString(tokens, "TemplateClose");
+
+    ArgumentOpen = PyObject_GetAttrString(tokens, "ArgumentOpen");
+    ArgumentSeparator = PyObject_GetAttrString(tokens, "ArgumentSeparator");
+    ArgumentClose = PyObject_GetAttrString(tokens, "ArgumentClose");
+
+    WikilinkOpen = PyObject_GetAttrString(tokens, "WikilinkOpen");
+    WikilinkSeparator = PyObject_GetAttrString(tokens, "WikilinkSeparator");
+    WikilinkClose = PyObject_GetAttrString(tokens, "WikilinkClose");
+
+    HTMLEntityStart = PyObject_GetAttrString(tokens, "HTMLEntityStart");
+    HTMLEntityNumeric = PyObject_GetAttrString(tokens, "HTMLEntityNumeric");
+    HTMLEntityHex = PyObject_GetAttrString(tokens, "HTMLEntityHex");
+    HTMLEntityEnd = PyObject_GetAttrString(tokens, "HTMLEntityEnd");
+
+    HeadingStart = PyObject_GetAttrString(tokens, "HeadingStart");
+    HeadingEnd = PyObject_GetAttrString(tokens, "HeadingEnd");
+
+    CommentStart = PyObject_GetAttrString(tokens, "CommentStart");
+    CommentEnd = PyObject_GetAttrString(tokens, "CommentEnd");
+
+    TagOpenOpen = PyObject_GetAttrString(tokens, "TagOpenOpen");
+    TagAttrStart = PyObject_GetAttrString(tokens, "TagAttrStart");
+    TagAttrEquals = PyObject_GetAttrString(tokens, "TagAttrEquals");
+    TagAttrQuote = PyObject_GetAttrString(tokens, "TagAttrQuote");
+    TagCloseOpen = PyObject_GetAttrString(tokens, "TagCloseOpen");
+    TagCloseSelfclose = PyObject_GetAttrString(tokens, "TagCloseSelfclose");
+    TagOpenClose = PyObject_GetAttrString(tokens, "TagOpenClose");
+    TagCloseClose = PyObject_GetAttrString(tokens, "TagCloseClose");
 }
diff --git a/mwparserfromhell/parser/tokenizer.h b/mwparserfromhell/parser/tokenizer.h
index 1b9b76a..3883d45 100644
--- a/mwparserfromhell/parser/tokenizer.h
+++ b/mwparserfromhell/parser/tokenizer.h
@@ -44,6 +44,43 @@ static PyObject* NOARGS;
 static PyObject* tokens;
 
 
+/* Tokens */
+
+static PyObject* Text;
+
+static PyObject* TemplateOpen;
+static PyObject* TemplateParamSeparator;
+static PyObject* TemplateParamEquals;
+static PyObject* TemplateClose;
+
+static PyObject* ArgumentOpen;
+static PyObject* ArgumentSeparator;
+static PyObject* ArgumentClose;
+
+static PyObject* WikilinkOpen;
+static PyObject* WikilinkSeparator;
+static PyObject* WikilinkClose;
+
+static PyObject* HTMLEntityStart;
+static PyObject* HTMLEntityNumeric;
+static PyObject* HTMLEntityHex;
+static PyObject* HTMLEntityEnd;
+static PyObject* HeadingStart;
+static PyObject* HeadingEnd;
+
+static PyObject* CommentStart;
+static PyObject* CommentEnd;
+
+static PyObject* TagOpenOpen;
+static PyObject* TagAttrStart;
+static PyObject* TagAttrEquals;
+static PyObject* TagAttrQuote;
+static PyObject* TagCloseOpen;
+static PyObject* TagCloseSelfclose;
+static PyObject* TagOpenClose;
+static PyObject* TagCloseClose;
+
+
 /* Local contexts: */
 
 static const int LC_TEMPLATE =             0x0007;

From cdef073a9b9b084ed2b7bdf0dc8827af5a62c300 Mon Sep 17 00:00:00 2001
From: Ben Kurtovic <ben.kurtovic@verizon.net>
Date: Sat, 20 Oct 2012 13:22:34 -0400
Subject: [PATCH 31/47] Replace textbuffer with our own datastructure.

---
 mwparserfromhell/parser/tokenizer.c | 307 ++++++++++++++++++++----------------
 mwparserfromhell/parser/tokenizer.h |  61 ++++---
 2 files changed, 211 insertions(+), 157 deletions(-)

diff --git a/mwparserfromhell/parser/tokenizer.c b/mwparserfromhell/parser/tokenizer.c
index 547207a..2c083c5 100644
--- a/mwparserfromhell/parser/tokenizer.c
+++ b/mwparserfromhell/parser/tokenizer.c
@@ -30,6 +30,25 @@ Tokenizer_new(PyTypeObject* type, PyObject* args, PyObject* kwds)
     return (PyObject*) self;
 }
 
+static struct Textbuffer*
+Textbuffer_new(void)
+{
+    struct Textbuffer* buffer = malloc(sizeof(struct Textbuffer));
+    if (!buffer) {
+        PyErr_NoMemory();
+        return NULL;
+    }
+    buffer->size = 0;
+    buffer->data = malloc(sizeof(Py_UNICODE) * TEXTBUFFER_BLOCKSIZE);
+    if (!buffer->data) {
+        free(buffer);
+        PyErr_NoMemory();
+        return NULL;
+    }
+    buffer->next = NULL;
+    return buffer;
+}
+
 static void
 Tokenizer_dealloc(Tokenizer* self)
 {
@@ -37,7 +56,7 @@ Tokenizer_dealloc(Tokenizer* self)
     struct Stack *this = self->topstack, *next;
     while (this) {
         Py_DECREF(this->stack);
-        Py_DECREF(this->textbuffer);
+        Textbuffer_dealloc(this->textbuffer);
         next = this->next;
         free(this);
         this = next;
@@ -45,6 +64,18 @@ Tokenizer_dealloc(Tokenizer* self)
     self->ob_type->tp_free((PyObject*) self);
 }
 
+static void
+Textbuffer_dealloc(struct Textbuffer* this)
+{
+    struct Textbuffer* next;
+    while (this) {
+        free(this->data);
+        next = this->next;
+        free(this);
+        this = next;
+    }
+}
+
 static int
 Tokenizer_init(Tokenizer* self, PyObject* args, PyObject* kwds)
 {
@@ -64,15 +95,32 @@ Tokenizer_init(Tokenizer* self, PyObject* args, PyObject* kwds)
 /*
     Add a new token stack, context, and textbuffer to the list.
 */
-static void
+static int
 Tokenizer_push(Tokenizer* self, int context)
 {
     struct Stack* top = malloc(sizeof(struct Stack));
+    if (!top) {
+        PyErr_NoMemory();
+        return -1;
+    }
     top->stack = PyList_New(0);
     top->context = context;
-    top->textbuffer = PyList_New(0);
+    top->textbuffer = Textbuffer_new();
+    if (!top->textbuffer) {
+        return -1;
+    }
     top->next = self->topstack;
     self->topstack = top;
+    return 0;
+}
+
+/*
+    Return the contents of the textbuffer as a Python Unicode object.
+*/
+static PyObject*
+Textbuffer_render(struct Textbuffer* self)
+{
+    return PyUnicode_FromUnicode(self->data, self->size);
 }
 
 /*
@@ -81,32 +129,35 @@ Tokenizer_push(Tokenizer* self, int context)
 static int
 Tokenizer_push_textbuffer(Tokenizer* self)
 {
-    if (PyList_GET_SIZE(self->topstack->textbuffer) > 0) {
-        PyObject* text = PyUnicode_Join(EMPTY, self->topstack->textbuffer);
-        if (!text) return -1;
+    struct Textbuffer* buffer = self->topstack->textbuffer;
+    if (buffer->size == 0 && !buffer->next) {
+        return 0;
+    }
+    PyObject* text = Textbuffer_render(buffer);
+    if (!text) return -1;
 
-        PyObject* kwargs = PyDict_New();
-        if (!kwargs) {
-            Py_DECREF(text);
-            return -1;
-        }
-        PyDict_SetItemString(kwargs, "text", text);
+    PyObject* kwargs = PyDict_New();
+    if (!kwargs) {
         Py_DECREF(text);
+        return -1;
+    }
+    PyDict_SetItemString(kwargs, "text", text);
+    Py_DECREF(text);
 
-        PyObject* token = PyObject_Call(Text, NOARGS, kwargs);
-        Py_DECREF(kwargs);
-        if (!token) return -1;
-
-        if (PyList_Append(self->topstack->stack, token)) {
-            Py_DECREF(token);
-            return -1;
-        }
+    PyObject* token = PyObject_Call(Text, NOARGS, kwargs);
+    Py_DECREF(kwargs);
+    if (!token) return -1;
 
+    if (PyList_Append(self->topstack->stack, token)) {
         Py_DECREF(token);
+        return -1;
+    }
 
-        self->topstack->textbuffer = PyList_New(0);
-        if (!self->topstack->textbuffer)
-            return -1;
+    Py_DECREF(token);
+
+    self->topstack->textbuffer = Textbuffer_new();
+    if (!self->topstack->textbuffer) {
+        return -1;
     }
     return 0;
 }
@@ -116,7 +167,7 @@ Tokenizer_delete_top_of_stack(Tokenizer* self)
 {
     struct Stack* top = self->topstack;
     Py_DECREF(top->stack);
-    Py_DECREF(top->textbuffer);
+    Textbuffer_dealloc(top->textbuffer);
     self->topstack = top->next;
     free(top);
 }
@@ -203,11 +254,20 @@ Tokenizer_write_first(Tokenizer* self, PyObject* token)
     Write text to the current textbuffer.
 */
 static int
-Tokenizer_write_text(Tokenizer* self, PyObject* text)
+Tokenizer_write_text(Tokenizer* self, Py_UNICODE text)
 {
-    if (PyList_Append(self->topstack->textbuffer, text))
-        return -1;
-
+    struct Textbuffer* buf = self->topstack->textbuffer;
+    if (buf->size == TEXTBUFFER_BLOCKSIZE) {
+        struct Textbuffer* new = Textbuffer_new();
+        if (!new) {
+            return -1;
+        }
+        new->next = buf;
+        self->topstack->textbuffer = new;
+        buf = new;
+    }
+    buf->data[buf->size] = text;
+    buf->size++;
     return 0;
 }
 
@@ -217,35 +277,48 @@ Tokenizer_write_text(Tokenizer* self, PyObject* text)
 static int
 Tokenizer_write_all(Tokenizer* self, PyObject* tokenlist)
 {
+    int pushed = 0;
     if (PyList_GET_SIZE(tokenlist) > 0) {
         PyObject* token = PyList_GET_ITEM(tokenlist, 0);
-
-        PyObject* text;
         switch (PyObject_IsInstance(token, Text)) {
             case 0:
                 break;
-            case 1:
-                text = PyObject_GetAttrString(token, "text");
-                if (!text) {
-                    return -1;
+            case 1: {
+                pushed = 1;
+                struct Textbuffer* buffer = self->topstack->textbuffer;
+                if (buffer->size == 0 && !buffer->next) {
+                    break;
                 }
-                if (PySequence_DelItem(tokenlist, 0)) {
+                PyObject* left = Textbuffer_render(buffer);
+                if (!left) return -1;
+                PyObject* right = PyObject_GetAttrString(token, "text");
+                if (!right) return -1;
+
+                PyObject* text = PyUnicode_Concat(left, right);
+                Py_DECREF(left);
+                Py_DECREF(right);
+                if (!text) return -1;
+
+                if (PyObject_SetAttrString(token, "text", text)) {
                     Py_DECREF(text);
                     return -1;
                 }
-                if (Tokenizer_write_text(self, text)) {
-                    Py_DECREF(text);
+                Py_DECREF(text);
+
+                self->topstack->textbuffer = Textbuffer_new();
+                if (!self->topstack->textbuffer) {
                     return -1;
                 }
-                Py_DECREF(text);
                 break;
+            }
             case -1:
                 return -1;
         }
     }
-
-    if (Tokenizer_push_textbuffer(self))
-        return -1;
+    if (!pushed) {
+        if (Tokenizer_push_textbuffer(self))
+            return -1;
+    }
 
     PyObject* stack = self->topstack->stack;
     Py_ssize_t size = PyList_GET_SIZE(stack);
@@ -257,15 +330,21 @@ Tokenizer_write_all(Tokenizer* self, PyObject* tokenlist)
 }
 
 /*
-    Pop the current stack, write text, and then write the stack.
+    Pop the current stack, write text, and then write the stack. 'text' is a
+    NULL-terminated array of chars.
 */
 static int
-Tokenizer_write_text_then_stack(Tokenizer* self, PyObject* text)
+Tokenizer_write_text_then_stack(Tokenizer* self, const char* text)
 {
     PyObject* stack = Tokenizer_pop(self);
-    if (Tokenizer_write_text(self, text)) {
-        Py_XDECREF(stack);
-        return -1;
+    int i = 0;
+    while (1) {
+        if (!text[i]) break;
+        if (Tokenizer_write_text(self, (Py_UNICODE) text[i])) {
+            Py_XDECREF(stack);
+            return -1;
+        }
+        i++;
     }
 
     if (stack) {
@@ -322,17 +401,13 @@ Tokenizer_parse_template_or_argument(Tokenizer* self)
         self->head++;
         braces++;
     }
-    Tokenizer_push(self, 0);
+    if (Tokenizer_push(self, 0))
+        return -1;
 
     while (braces) {
         if (braces == 1) {
-            PyObject* text = PyUnicode_FromString("{");
-            if (Tokenizer_write_text_then_stack(self, text)) {
-                Py_XDECREF(text);
+            if (Tokenizer_write_text_then_stack(self, "{"))
                 return -1;
-            }
-
-            Py_XDECREF(text);
             return 0;
         }
 
@@ -342,13 +417,8 @@ Tokenizer_parse_template_or_argument(Tokenizer* self)
 
             if (BAD_ROUTE) {
                 RESET_ROUTE();
-                PyObject* text = PyUnicode_FromString("{{");
-                if (Tokenizer_write_text_then_stack(self, text)) {
-                    Py_XDECREF(text);
+                if (Tokenizer_write_text_then_stack(self, "{{"))
                     return -1;
-                }
-
-                Py_XDECREF(text);
                 return 0;
             }
             break;
@@ -364,10 +434,9 @@ Tokenizer_parse_template_or_argument(Tokenizer* self)
 
             if (BAD_ROUTE) {
                 RESET_ROUTE();
-                char bracestr[braces];
-                for (i = 0; i < braces; i++) bracestr[i] = *"{";
-                PyObject* text = PyUnicode_FromStringAndSize(bracestr, braces);
-
+                char text[braces + 1];
+                for (i = 0; i < braces; i++) text[i] = *"{";
+                text[braces] = *"";
                 if (Tokenizer_write_text_then_stack(self, text)) {
                     Py_XDECREF(text);
                     return -1;
@@ -623,7 +692,8 @@ Tokenizer_handle_template_param(Tokenizer* self)
     }
     Py_DECREF(token);
 
-    Tokenizer_push(self, self->topstack->context);
+    if (Tokenizer_push(self, self->topstack->context))
+        return -1;
     return 0;
 }
 
@@ -748,11 +818,10 @@ Tokenizer_parse_wikilink(Tokenizer* self)
     if (BAD_ROUTE) {
         RESET_ROUTE();
         self->head = reset;
-        PyObject* text = PyUnicode_FromString("[[");
-        if (!text) return -1;
-        if (Tokenizer_write_text(self, text)) {
-            Py_XDECREF(text);
-            return -1;
+        int i;
+        for (i = 0; i < 2; i++) {
+            if (Tokenizer_write_text(self, *"["))
+                return -1;
         }
         return 0;
     }
@@ -837,9 +906,7 @@ Tokenizer_parse_heading(Tokenizer* self)
     self->global |= GL_HEADING;
     Py_ssize_t reset = self->head;
     self->head += 1;
-    int best = 1;
-    PyObject* text;
-    int i;
+    int best = 1, i;
 
     while (*Tokenizer_READ(self, 0) == *"=") {
         best++;
@@ -852,16 +919,11 @@ Tokenizer_parse_heading(Tokenizer* self)
     if (BAD_ROUTE) {
         RESET_ROUTE();
         self->head = reset + best - 1;
-        char blocks[best];
-        for (i = 0; i < best; i++) blocks[i] = *"=";
-        text = PyUnicode_FromStringAndSize(blocks, best);
-        if (!text) return -1;
-
-        if (Tokenizer_write_text_then_stack(self, text)) {
-            Py_DECREF(text);
+        char text[best + 1];
+        for (i = 0; i < best; i++) text[i] = *"=";
+        text[best] = *"";
+        if (Tokenizer_write_text_then_stack(self, text))
             return -1;
-        }
-        Py_DECREF(text);
         self->global ^= GL_HEADING;
         return 0;
     }
@@ -901,22 +963,14 @@ Tokenizer_parse_heading(Tokenizer* self)
 
     if (heading->level < best) {
         int diff = best - heading->level;
-        char diffblocks[diff];
-        for (i = 0; i < diff; i++) diffblocks[i] = *"=";
-        PyObject* text = PyUnicode_FromStringAndSize(diffblocks, diff);
-        if (!text) {
+        char difftext[diff + 1];
+        for (i = 0; i < diff; i++) difftext[i] = *"=";
+        difftext[diff] = *"";
+        if (Tokenizer_write_text_then_stack(self, difftext)) {
             Py_DECREF(heading->title);
             free(heading);
             return -1;
         }
-
-        if (Tokenizer_write_text_then_stack(self, text)) {
-            Py_DECREF(text);
-            Py_DECREF(heading->title);
-            free(heading);
-            return -1;
-        }
-        Py_DECREF(text);
     }
 
     if (Tokenizer_write_all(self, heading->title)) {
@@ -949,7 +1003,6 @@ Tokenizer_handle_heading_end(Tokenizer* self)
     Py_ssize_t reset = self->head;
     self->head += 1;
     Py_ssize_t best = 1;
-    PyObject* text;
     int i;
 
     while (*Tokenizer_READ(self, 0) == *"=") {
@@ -965,39 +1018,24 @@ Tokenizer_handle_heading_end(Tokenizer* self)
         RESET_ROUTE();
         if (level < best) {
             int diff = best - level;
-            char diffblocks[diff];
-            for (i = 0; i < diff; i++) diffblocks[i] = *"=";
-            text = PyUnicode_FromStringAndSize(diffblocks, diff);
-            if (!text) return NULL;
-
-            if (Tokenizer_write_text_then_stack(self, text)) {
-                Py_DECREF(text);
+            char difftext[diff + 1];
+            for (i = 0; i < diff; i++) difftext[i] = *"=";
+            difftext[diff] = *"";
+            if (Tokenizer_write_text_then_stack(self, difftext))
                 return NULL;
-            }
-            Py_DECREF(text);
         }
-
         self->head = reset + best - 1;
     }
 
     else {
-        char blocks[best];
-        for (i = 0; i < best; i++) blocks[i] = *"=";
-        text = PyUnicode_FromStringAndSize(blocks, best);
-        if (!text) {
-            Py_DECREF(after->title);
-            free(after);
-            return NULL;
-        }
-
+        char text[best + 1];
+        for (i = 0; i < best; i++) text[i] = *"=";
+        text[best] = *"";
         if (Tokenizer_write_text_then_stack(self, text)) {
-            Py_DECREF(text);
             Py_DECREF(after->title);
             free(after);
             return NULL;
         }
-        Py_DECREF(text);
-
         if (Tokenizer_write_all(self, after->title)) {
             Py_DECREF(after->title);
             free(after);
@@ -1037,7 +1075,8 @@ static int
 Tokenizer_parse_entity(Tokenizer* self)
 {
     Py_ssize_t reset = self->head;
-    Tokenizer_push(self, 0);
+    if (Tokenizer_push(self, 0))
+        return -1;
 
     if (Tokenizer_really_parse_entity(self))
             return -1;
@@ -1045,7 +1084,7 @@ Tokenizer_parse_entity(Tokenizer* self)
     if (BAD_ROUTE) {
         RESET_ROUTE();
         self->head = reset;
-        if (Tokenizer_write_text(self, Tokenizer_read(self, 0)))
+        if (Tokenizer_write_text(self, *PyUnicode_AS_UNICODE(Tokenizer_read(self, 0))))
             return -1;
         return 0;
     }
@@ -1077,13 +1116,16 @@ Tokenizer_parse_comment(Tokenizer* self)
     if (BAD_ROUTE) {
         RESET_ROUTE();
         self->head = reset;
-        PyObject* text = PyUnicode_FromString("<!--");
-        if (!text) return -1;
-        if (Tokenizer_write_text(self, text)) {
-            Py_XDECREF(text);
-            return -1;
+        const char* text = "<!--";
+        int i = 0;
+        while (1) {
+            if (!text[i]) return 0;
+            if (Tokenizer_write_text(self, (Py_UNICODE) text[i])) {
+                Py_XDECREF(text);
+                return -1;
+            }
+            i++;
         }
-        return 0;
     }
 
     token = PyObject_CallObject(CommentStart, NULL);
@@ -1129,7 +1171,8 @@ Tokenizer_parse(Tokenizer* self, int context)
     int fail_contexts = LC_TEMPLATE | LC_ARGUMENT | LC_WIKILINK | LC_HEADING | LC_COMMENT;
     int is_marker, i;
 
-    Tokenizer_push(self, context);
+    if (Tokenizer_push(self, context))
+        return NULL;
 
     while (1) {
         this = Tokenizer_read(self, 0);
@@ -1144,7 +1187,7 @@ Tokenizer_parse(Tokenizer* self, int context)
         }
 
         if (!is_marker) {
-            Tokenizer_write_text(self, this);
+            Tokenizer_write_text(self, this_data);
             self->head++;
             continue;
         }
@@ -1170,7 +1213,7 @@ Tokenizer_parse(Tokenizer* self, int context)
                     return Tokenizer_pop(self);
                 }
             }
-            Tokenizer_write_text(self, this);
+            Tokenizer_write_text(self, this_data);
         }
         else if (this_data == next && next == *"{") {
             if (Tokenizer_parse_template_or_argument(self))
@@ -1195,7 +1238,7 @@ Tokenizer_parse(Tokenizer* self, int context)
             if (*Tokenizer_READ(self, 2) == *"}") {
                 return Tokenizer_handle_argument_end(self);
             }
-            Tokenizer_write_text(self, this);
+            Tokenizer_write_text(self, this_data);
         }
         else if (this_data == next && next == *"[") {
             if (!(this_context & LC_WIKILINK_TITLE)) {
@@ -1203,7 +1246,7 @@ Tokenizer_parse(Tokenizer* self, int context)
                     return NULL;
             }
             else {
-                Tokenizer_write_text(self, this);
+                Tokenizer_write_text(self, this_data);
             }
         }
         else if (this_data == *"|" && this_context & LC_WIKILINK_TITLE) {
@@ -1220,7 +1263,7 @@ Tokenizer_parse(Tokenizer* self, int context)
                     return NULL;
             }
             else {
-                Tokenizer_write_text(self, this);
+                Tokenizer_write_text(self, this_data);
             }
         }
         else if (this_data == *"=" && this_context & LC_HEADING) {
@@ -1240,11 +1283,11 @@ Tokenizer_parse(Tokenizer* self, int context)
                     return NULL;
             }
             else {
-                Tokenizer_write_text(self, this);
+                Tokenizer_write_text(self, this_data);
             }
         }
         else {
-            Tokenizer_write_text(self, this);
+            Tokenizer_write_text(self, this_data);
         }
 
         self->head++;
diff --git a/mwparserfromhell/parser/tokenizer.h b/mwparserfromhell/parser/tokenizer.h
index 3883d45..468700c 100644
--- a/mwparserfromhell/parser/tokenizer.h
+++ b/mwparserfromhell/parser/tokenizer.h
@@ -32,7 +32,9 @@ SOFTWARE.
 static const char* MARKERS[] = {
     "{",  "}", "[", "]", "<", ">", "|", "=", "&", "#", "*", ";", ":", "/", "-",
     "!", "\n", ""};
-static const int NUM_MARKERS = 18;
+
+#define NUM_MARKERS 18
+#define TEXTBUFFER_BLOCKSIZE 1024
 
 static int route_state = 0;
 #define BAD_ROUTE     (route_state)
@@ -83,41 +85,47 @@ static PyObject* TagCloseClose;
 
 /* Local contexts: */
 
-static const int LC_TEMPLATE =             0x0007;
-static const int LC_TEMPLATE_NAME =        0x0001;
-static const int LC_TEMPLATE_PARAM_KEY =   0x0002;
-static const int LC_TEMPLATE_PARAM_VALUE = 0x0004;
+#define LC_TEMPLATE             0x0007
+#define LC_TEMPLATE_NAME        0x0001
+#define LC_TEMPLATE_PARAM_KEY   0x0002
+#define LC_TEMPLATE_PARAM_VALUE 0x0004
 
-static const int LC_ARGUMENT =             0x0018;
-static const int LC_ARGUMENT_NAME =        0x0008;
-static const int LC_ARGUMENT_DEFAULT =     0x0010;
+#define LC_ARGUMENT             0x0018
+#define LC_ARGUMENT_NAME        0x0008
+#define LC_ARGUMENT_DEFAULT     0x0010
 
-static const int LC_WIKILINK =             0x0060;
-static const int LC_WIKILINK_TITLE =       0x0020;
-static const int LC_WIKILINK_TEXT =        0x0040;
+#define LC_WIKILINK             0x0060
+#define LC_WIKILINK_TITLE       0x0020
+#define LC_WIKILINK_TEXT        0x0040
 
-static const int LC_HEADING =              0x1f80;
-static const int LC_HEADING_LEVEL_1 =      0x0080;
-static const int LC_HEADING_LEVEL_2 =      0x0100;
-static const int LC_HEADING_LEVEL_3 =      0x0200;
-static const int LC_HEADING_LEVEL_4 =      0x0400;
-static const int LC_HEADING_LEVEL_5 =      0x0800;
-static const int LC_HEADING_LEVEL_6 =      0x1000;
+#define LC_HEADING              0x1f80
+#define LC_HEADING_LEVEL_1      0x0080
+#define LC_HEADING_LEVEL_2      0x0100
+#define LC_HEADING_LEVEL_3      0x0200
+#define LC_HEADING_LEVEL_4      0x0400
+#define LC_HEADING_LEVEL_5      0x0800
+#define LC_HEADING_LEVEL_6      0x1000
 
-static const int LC_COMMENT =              0x2000;
+#define LC_COMMENT              0x2000
 
 
 /* Global contexts: */
 
-static const int GL_HEADING = 0x1;
+#define GL_HEADING 0x1
 
 
 /* Miscellaneous structs: */
 
+struct Textbuffer {
+    Py_ssize_t size;
+    Py_UNICODE* data;
+    struct Textbuffer* next;
+};
+
 struct Stack {
     PyObject* stack;
     int context;
-    PyObject* textbuffer;
+    struct Textbuffer* textbuffer;
     struct Stack* next;
 };
 
@@ -144,12 +152,15 @@ typedef struct {
 #define Tokenizer_READ(self, delta) PyUnicode_AS_UNICODE(Tokenizer_read(self, delta))
 
 
-/* Tokenizer function prototypes: */
+/* Function prototypes: */
 
 static PyObject* Tokenizer_new(PyTypeObject*, PyObject*, PyObject*);
+static struct Textbuffer* Textbuffer_new(void);
 static void Tokenizer_dealloc(Tokenizer*);
+static void Textbuffer_dealloc(struct Textbuffer*);
 static int Tokenizer_init(Tokenizer*, PyObject*, PyObject*);
-static void Tokenizer_push(Tokenizer*, int);
+static int Tokenizer_push(Tokenizer*, int);
+static PyObject* Textbuffer_render(struct Textbuffer*);
 static int Tokenizer_push_textbuffer(Tokenizer*);
 static void Tokenizer_delete_top_of_stack(Tokenizer*);
 static PyObject* Tokenizer_pop(Tokenizer*);
@@ -157,9 +168,9 @@ static PyObject* Tokenizer_pop_keeping_context(Tokenizer*);
 static void* Tokenizer_fail_route(Tokenizer*);
 static int Tokenizer_write(Tokenizer*, PyObject*);
 static int Tokenizer_write_first(Tokenizer*, PyObject*);
-static int Tokenizer_write_text(Tokenizer*, PyObject*);
+static int Tokenizer_write_text(Tokenizer*, Py_UNICODE);
 static int Tokenizer_write_all(Tokenizer*, PyObject*);
-static int Tokenizer_write_text_then_stack(Tokenizer*, PyObject*);
+static int Tokenizer_write_text_then_stack(Tokenizer*, const char*);
 static PyObject* Tokenizer_read(Tokenizer*, Py_ssize_t);
 static PyObject* Tokenizer_read_backwards(Tokenizer*, Py_ssize_t);
 static int Tokenizer_parse_template_or_argument(Tokenizer*);

From c5fea8ea34ac6cb9e4c84dc5c1ec2ff5a79c1882 Mon Sep 17 00:00:00 2001
From: Ben Kurtovic <ben.kurtovic@verizon.net>
Date: Sat, 20 Oct 2012 16:05:04 -0400
Subject: [PATCH 32/47] Render the entire textbuffer correctly.

---
 mwparserfromhell/parser/tokenizer.c | 14 +++++++++++++-
 1 file changed, 13 insertions(+), 1 deletion(-)

diff --git a/mwparserfromhell/parser/tokenizer.c b/mwparserfromhell/parser/tokenizer.c
index 2c083c5..1247c3e 100644
--- a/mwparserfromhell/parser/tokenizer.c
+++ b/mwparserfromhell/parser/tokenizer.c
@@ -120,7 +120,17 @@ Tokenizer_push(Tokenizer* self, int context)
 static PyObject*
 Textbuffer_render(struct Textbuffer* self)
 {
-    return PyUnicode_FromUnicode(self->data, self->size);
+    PyObject *result = PyUnicode_FromUnicode(self->data, self->size);
+    PyObject *left, *concat;
+    while (self->next) {
+        self = self->next;
+        left = PyUnicode_FromUnicode(self->data, self->size);
+        concat = PyUnicode_Concat(left, result);
+        Py_DECREF(left);
+        Py_DECREF(result);
+        result = concat;
+    }
+    return result;
 }
 
 /*
@@ -155,6 +165,7 @@ Tokenizer_push_textbuffer(Tokenizer* self)
 
     Py_DECREF(token);
 
+    Textbuffer_dealloc(buffer);
     self->topstack->textbuffer = Textbuffer_new();
     if (!self->topstack->textbuffer) {
         return -1;
@@ -305,6 +316,7 @@ Tokenizer_write_all(Tokenizer* self, PyObject* tokenlist)
                 }
                 Py_DECREF(text);
 
+                Textbuffer_dealloc(buffer);
                 self->topstack->textbuffer = Textbuffer_new();
                 if (!self->topstack->textbuffer) {
                     return -1;

From 76b299e4faaa34dfead07e727978276f3254159b Mon Sep 17 00:00:00 2001
From: Ben Kurtovic <ben.kurtovic@verizon.net>
Date: Sat, 27 Oct 2012 21:47:11 -0400
Subject: [PATCH 33/47] Sync from upstream (6413ae21da); some doc updates.

---
 mwparserfromhell/nodes/template.py |  6 +++---
 mwparserfromhell/wikicode.py       | 16 ++++++++--------
 2 files changed, 11 insertions(+), 11 deletions(-)

diff --git a/mwparserfromhell/nodes/template.py b/mwparserfromhell/nodes/template.py
index c1abc2a..08ab4a5 100644
--- a/mwparserfromhell/nodes/template.py
+++ b/mwparserfromhell/nodes/template.py
@@ -249,11 +249,11 @@ class Template(Node):
         if not force_nonconformity:
             before_n, after_n = self._get_spacing_conventions(use_names=True)
             if before_n and after_n:
-                name = parse_anything([before_n, value, after_n])
+                name = parse_anything([before_n, name, after_n])
             elif before_n:
-                name = parse_anything([before_n, value])
+                name = parse_anything([before_n, name])
             elif after_n:
-                name = parse_anything([value, after_n])
+                name = parse_anything([name, after_n])
 
             before_v, after_v = self._get_spacing_conventions(use_names=False)
             if before_v and after_v:
diff --git a/mwparserfromhell/wikicode.py b/mwparserfromhell/wikicode.py
index e0f5acd..2c532f5 100644
--- a/mwparserfromhell/wikicode.py
+++ b/mwparserfromhell/wikicode.py
@@ -386,12 +386,12 @@ class Wikicode(StringMixIn):
         With *flat* as ``True``, each returned section contains all of its
         subsections within the :py:class:`~.Wikicode`; otherwise, the returned
         sections contain only the section up to the next heading, regardless of
-        its size. If *matches* is given, it should be a regex to matched
+        its size. If *matches* is given, it should be a regex to be matched
         against the titles of section headings; only sections whose headings
-        match the regex will be included. If *levels* is given, it should be a =
-        list of integers; only sections whose heading levels are within the
-        list will be returned. If *include_headings* is ``True``, the section's
-        literal :py:class:`~.Heading` object will be included in returned
+        match the regex will be included. If *levels* is given, it should be a
+        iterable of integers; only sections whose heading levels are within it
+        will be returned. If *include_headings* is ``True``, the section's
+        beginning :py:class:`~.Heading` object will be included in returned
         :py:class:`~.Wikicode` objects; otherwise, this is skipped.
         """
         if matches:
@@ -402,16 +402,16 @@ class Wikicode(StringMixIn):
             headings = [head for head in headings if head.level in levels]
 
         sections = []
-        buffers = [[maxsize, 0]]
+        buffers = [(maxsize, 0)]
         i = 0
         while i < len(self.nodes):
             if self.nodes[i] in headings:
                 this = self.nodes[i].level
                 for (level, start) in buffers:
                     if not flat or this <= level:
-                        buffers.remove([level, start])
+                        buffers.remove((level, start))
                         sections.append(Wikicode(self.nodes[start:i]))
-                buffers.append([this, i])
+                buffers.append((this, i))
                 if not include_headings:
                     i += 1
             i += 1

From b2f933dddc868b729b810ab10cfc5ee59214deb4 Mon Sep 17 00:00:00 2001
From: Ben Kurtovic <ben.kurtovic@verizon.net>
Date: Thu, 1 Nov 2012 12:41:55 -0400
Subject: [PATCH 34/47] Fix memory leak.

---
 mwparserfromhell/parser/tokenizer.c | 39 +++++++++++++++++++------------------
 1 file changed, 20 insertions(+), 19 deletions(-)

diff --git a/mwparserfromhell/parser/tokenizer.c b/mwparserfromhell/parser/tokenizer.c
index 1247c3e..01acd50 100644
--- a/mwparserfromhell/parser/tokenizer.c
+++ b/mwparserfromhell/parser/tokenizer.c
@@ -33,15 +33,15 @@ Tokenizer_new(PyTypeObject* type, PyObject* args, PyObject* kwds)
 static struct Textbuffer*
 Textbuffer_new(void)
 {
-    struct Textbuffer* buffer = malloc(sizeof(struct Textbuffer));
+    struct Textbuffer* buffer = PyObject_Malloc(sizeof(struct Textbuffer));
     if (!buffer) {
         PyErr_NoMemory();
         return NULL;
     }
     buffer->size = 0;
-    buffer->data = malloc(sizeof(Py_UNICODE) * TEXTBUFFER_BLOCKSIZE);
+    buffer->data = PyObject_Malloc(sizeof(Py_UNICODE) * TEXTBUFFER_BLOCKSIZE);
     if (!buffer->data) {
-        free(buffer);
+        PyObject_Free(buffer);
         PyErr_NoMemory();
         return NULL;
     }
@@ -58,7 +58,7 @@ Tokenizer_dealloc(Tokenizer* self)
         Py_DECREF(this->stack);
         Textbuffer_dealloc(this->textbuffer);
         next = this->next;
-        free(this);
+        PyObject_Free(this);
         this = next;
     }
     self->ob_type->tp_free((PyObject*) self);
@@ -69,9 +69,9 @@ Textbuffer_dealloc(struct Textbuffer* this)
 {
     struct Textbuffer* next;
     while (this) {
-        free(this->data);
+        PyObject_Free(this->data);
         next = this->next;
-        free(this);
+        PyObject_Free(this);
         this = next;
     }
 }
@@ -98,7 +98,7 @@ Tokenizer_init(Tokenizer* self, PyObject* args, PyObject* kwds)
 static int
 Tokenizer_push(Tokenizer* self, int context)
 {
-    struct Stack* top = malloc(sizeof(struct Stack));
+    struct Stack* top = PyObject_Malloc(sizeof(struct Stack));
     if (!top) {
         PyErr_NoMemory();
         return -1;
@@ -180,7 +180,7 @@ Tokenizer_delete_top_of_stack(Tokenizer* self)
     Py_DECREF(top->stack);
     Textbuffer_dealloc(top->textbuffer);
     self->topstack = top->next;
-    free(top);
+    PyObject_Free(top);
 }
 
 /*
@@ -607,6 +607,7 @@ Tokenizer_verify_safe(Tokenizer* self, const char* unsafes[])
                         Py_DECREF(textlist);
                         return -1;
                     }
+                    Py_DECREF(textdata);
                     if (PyList_Append(textlist, textdata)) {
                         Py_DECREF(textlist);
                         Py_DECREF(textdata);
@@ -943,7 +944,7 @@ Tokenizer_parse_heading(Tokenizer* self)
     PyObject* level = PyInt_FromSsize_t(heading->level);
     if (!level) {
         Py_DECREF(heading->title);
-        free(heading);
+        PyObject_Free(heading);
         return -1;
     }
 
@@ -951,7 +952,7 @@ Tokenizer_parse_heading(Tokenizer* self)
     if (!kwargs) {
         Py_DECREF(level);
         Py_DECREF(heading->title);
-        free(heading);
+        PyObject_Free(heading);
         return -1;
     }
     PyDict_SetItemString(kwargs, "level", level);
@@ -961,14 +962,14 @@ Tokenizer_parse_heading(Tokenizer* self)
     Py_DECREF(kwargs);
     if (!token) {
         Py_DECREF(heading->title);
-        free(heading);
+        PyObject_Free(heading);
         return -1;
     }
 
     if (Tokenizer_write(self, token)) {
         Py_DECREF(token);
         Py_DECREF(heading->title);
-        free(heading);
+        PyObject_Free(heading);
         return -1;
     }
     Py_DECREF(token);
@@ -980,18 +981,18 @@ Tokenizer_parse_heading(Tokenizer* self)
         difftext[diff] = *"";
         if (Tokenizer_write_text_then_stack(self, difftext)) {
             Py_DECREF(heading->title);
-            free(heading);
+            PyObject_Free(heading);
             return -1;
         }
     }
 
     if (Tokenizer_write_all(self, heading->title)) {
         Py_DECREF(heading->title);
-        free(heading);
+        PyObject_Free(heading);
         return -1;
     }
     Py_DECREF(heading->title);
-    free(heading);
+    PyObject_Free(heading);
 
     token = PyObject_CallObject(HeadingEnd, NULL);
     if (!token) return -1;
@@ -1045,23 +1046,23 @@ Tokenizer_handle_heading_end(Tokenizer* self)
         text[best] = *"";
         if (Tokenizer_write_text_then_stack(self, text)) {
             Py_DECREF(after->title);
-            free(after);
+            PyObject_Free(after);
             return NULL;
         }
         if (Tokenizer_write_all(self, after->title)) {
             Py_DECREF(after->title);
-            free(after);
+            PyObject_Free(after);
             return NULL;
         }
         Py_DECREF(after->title);
         level = after->level;
-        free(after);
+        PyObject_Free(after);
     }
 
     PyObject* stack = Tokenizer_pop(self);
     if (!stack) return NULL;
 
-    HeadingData* heading = malloc(sizeof(HeadingData));
+    HeadingData* heading = PyObject_Malloc(sizeof(HeadingData));
     if (!heading) {
         PyErr_NoMemory();
         return NULL;

From 7de34ed5a6151e7d3d787dbce5bbc83fdba44e64 Mon Sep 17 00:00:00 2001
From: Ben Kurtovic <ben.kurtovic@verizon.net>
Date: Thu, 1 Nov 2012 15:15:56 -0400
Subject: [PATCH 35/47] Rewrite verify_safe completely.

---
 mwparserfromhell/parser/tokenizer.c | 210 +++++++++++++-----------------------
 mwparserfromhell/parser/tokenizer.h |   9 +-
 2 files changed, 79 insertions(+), 140 deletions(-)

diff --git a/mwparserfromhell/parser/tokenizer.c b/mwparserfromhell/parser/tokenizer.c
index 01acd50..82cffaf 100644
--- a/mwparserfromhell/parser/tokenizer.c
+++ b/mwparserfromhell/parser/tokenizer.c
@@ -576,107 +576,12 @@ Tokenizer_parse_argument(Tokenizer* self)
 }
 
 /*
-    Verify that there are no unsafe characters in the current stack. The route
-    will be failed if the name contains any element of unsafes in it (not
-    merely at the beginning or end). This is used when parsing a template name
-    or parameter key, which cannot contain newlines.
-*/
-static int
-Tokenizer_verify_safe(Tokenizer* self, const char* unsafes[])
-{
-    if (Tokenizer_push_textbuffer(self))
-        return -1;
-
-    PyObject* stack = self->topstack->stack;
-    if (stack) {
-        PyObject* textlist = PyList_New(0);
-        if (!textlist) return -1;
-
-        int i;
-        Py_ssize_t length = PyList_GET_SIZE(stack);
-        PyObject *token, *textdata;
-
-        for (i = 0; i < length; i++) {
-            token = PyList_GET_ITEM(stack, i);
-            switch (PyObject_IsInstance(token, Text)) {
-                case 0:
-                    break;
-                case 1:
-                    textdata = PyObject_GetAttrString(token, "text");
-                    if (!textdata) {
-                        Py_DECREF(textlist);
-                        return -1;
-                    }
-                    Py_DECREF(textdata);
-                    if (PyList_Append(textlist, textdata)) {
-                        Py_DECREF(textlist);
-                        Py_DECREF(textdata);
-                        return -1;
-                    }
-                    Py_DECREF(textdata);
-                    break;
-                case -1:
-                    Py_DECREF(textlist);
-                    return -1;
-            }
-        }
-
-        PyObject* text = PyUnicode_Join(EMPTY, textlist);
-        if (!text) {
-            Py_DECREF(textlist);
-            return -1;
-        }
-        Py_DECREF(textlist);
-
-        PyObject* stripped = PyObject_CallMethod(text, "strip", NULL);
-        if (!stripped) {
-            Py_DECREF(text);
-            return -1;
-        }
-        Py_DECREF(text);
-
-        const char* unsafe_char;
-        PyObject* unsafe;
-        i = 0;
-        while (1) {
-            unsafe_char = unsafes[i];
-            if (!unsafe_char) break;
-
-            unsafe = PyUnicode_FromString(unsafe_char);
-
-            if (!unsafe) {
-                Py_DECREF(stripped);
-                return -1;
-            }
-
-            switch (PyUnicode_Contains(stripped, unsafe)) {
-                case 0:
-                    break;
-                case 1:
-                    Tokenizer_fail_route(self);
-                case -1:
-                    Py_DECREF(stripped);
-                    Py_DECREF(unsafe);
-                    return -1;
-            }
-            i++;
-        }
-    }
-
-    return 0;
-}
-
-/*
     Handle a template parameter at the head of the string.
 */
 static int
 Tokenizer_handle_template_param(Tokenizer* self)
 {
     if (self->topstack->context & LC_TEMPLATE_NAME) {
-        const char* unsafes[] = {"\n", "{", "}", "[", "]", NULL};
-        if (Tokenizer_verify_safe(self, unsafes))
-            return -1;
-        if (BAD_ROUTE) return -1;
         self->topstack->context ^= LC_TEMPLATE_NAME;
     }
     else if (self->topstack->context & LC_TEMPLATE_PARAM_VALUE) {
@@ -716,15 +621,6 @@ Tokenizer_handle_template_param(Tokenizer* self)
 static int
 Tokenizer_handle_template_param_value(Tokenizer* self)
 {
-    const char* unsafes[] = {"\n", "{{", "}}", NULL};
-    if (Tokenizer_verify_safe(self, unsafes)) {
-        if (BAD_ROUTE) {
-            PyObject* stack = Tokenizer_pop(self);
-            Py_XDECREF(stack);
-        }
-        return -1;
-    }
-
     PyObject* stack = Tokenizer_pop_keeping_context(self);
     if (!stack) return -1;
     if (Tokenizer_write_all(self, stack)) {
@@ -754,12 +650,7 @@ static PyObject*
 Tokenizer_handle_template_end(Tokenizer* self)
 {
     PyObject* stack;
-    if (self->topstack->context & LC_TEMPLATE_NAME) {
-        const char* unsafes[] = {"\n", "{", "}", "[", "]", NULL};
-        if (Tokenizer_verify_safe(self, unsafes))
-            return NULL;
-    }
-    else if (self->topstack->context & LC_TEMPLATE_PARAM_KEY) {
+    if (self->topstack->context & LC_TEMPLATE_PARAM_KEY) {
         stack = Tokenizer_pop_keeping_context(self);
         if (!stack) return NULL;
         if (Tokenizer_write_all(self, stack)) {
@@ -780,10 +671,6 @@ Tokenizer_handle_template_end(Tokenizer* self)
 static int
 Tokenizer_handle_argument_separator(Tokenizer* self)
 {
-    const char* unsafes[] = {"\n", "{{", "}}", NULL};
-    if (Tokenizer_verify_safe(self, unsafes))
-        return -1;
-
     self->topstack->context ^= LC_ARGUMENT_NAME;
     self->topstack->context |= LC_ARGUMENT_DEFAULT;
 
@@ -804,12 +691,6 @@ Tokenizer_handle_argument_separator(Tokenizer* self)
 static PyObject*
 Tokenizer_handle_argument_end(Tokenizer* self)
 {
-    if (self->topstack->context & LC_ARGUMENT_NAME) {
-        const char* unsafes[] = {"\n", "{{", "}}", NULL};
-        if (Tokenizer_verify_safe(self, unsafes))
-            return NULL;
-    }
-
     self->head += 2;
     PyObject* stack = Tokenizer_pop(self);
     return stack;
@@ -826,7 +707,6 @@ Tokenizer_parse_wikilink(Tokenizer* self)
 
     PyObject *token;
     PyObject *wikilink = Tokenizer_parse(self, LC_WIKILINK_TITLE);
-    if (!wikilink) return -1;
 
     if (BAD_ROUTE) {
         RESET_ROUTE();
@@ -838,6 +718,7 @@ Tokenizer_parse_wikilink(Tokenizer* self)
         }
         return 0;
     }
+    if (!wikilink) return -1;
 
     token = PyObject_CallObject(WikilinkOpen, NULL);
     if (!token) {
@@ -875,10 +756,6 @@ Tokenizer_parse_wikilink(Tokenizer* self)
 static int
 Tokenizer_handle_wikilink_separator(Tokenizer* self)
 {
-    const char* unsafes[] = {"\n", "{", "}", "[", "]", NULL};
-    if (Tokenizer_verify_safe(self, unsafes))
-        return -1;
-
     self->topstack->context ^= LC_WIKILINK_TITLE;
     self->topstack->context |= LC_WIKILINK_TEXT;
 
@@ -899,12 +776,6 @@ Tokenizer_handle_wikilink_separator(Tokenizer* self)
 static PyObject*
 Tokenizer_handle_wikilink_end(Tokenizer* self)
 {
-    if (self->topstack->context & LC_WIKILINK_TITLE) {
-        const char* unsafes[] = {"\n", "{", "}", "[", "]", NULL};
-        if (Tokenizer_verify_safe(self, unsafes))
-            return NULL;
-    }
-
     self->head += 1;
     PyObject* stack = Tokenizer_pop(self);
     return stack;
@@ -1124,7 +995,6 @@ Tokenizer_parse_comment(Tokenizer* self)
 
     PyObject *token;
     PyObject *comment = Tokenizer_parse(self, LC_WIKILINK_TITLE);
-    if (!comment) return -1;
 
     if (BAD_ROUTE) {
         RESET_ROUTE();
@@ -1139,7 +1009,9 @@ Tokenizer_parse_comment(Tokenizer* self)
             }
             i++;
         }
+        return 0;
     }
+    if (!comment) return -1;
 
     token = PyObject_CallObject(CommentStart, NULL);
     if (!token) {
@@ -1173,16 +1045,74 @@ Tokenizer_parse_comment(Tokenizer* self)
 }
 
 /*
+    Make sure we are not trying to write an invalid character.
+*/
+static void
+Tokenizer_verify_safe(Tokenizer* self, int context, Py_UNICODE data)
+{
+    if (context & LC_FAIL_NEXT) {
+        Tokenizer_fail_route(self);
+        return;
+    }
+
+    if (context & (LC_TEMPLATE_NAME | LC_WIKILINK_TITLE)) {
+        if (data == *"{" || data == *"}" || data == *"[" || data == *"]") {
+            self->topstack->context |= LC_FAIL_NEXT;
+            return;
+        }
+    }
+    else if (context & (LC_TEMPLATE_PARAM_KEY | LC_ARGUMENT_NAME)) {
+        if (context & LC_FAIL_ON_LBRACE) {
+            if (data == *"{") {
+                self->topstack->context |= LC_FAIL_NEXT;
+                return;
+            }
+            self->topstack->context ^= LC_FAIL_ON_LBRACE;
+        }
+        else if (context & LC_FAIL_ON_RBRACE) {
+            if (data == *"}") {
+                self->topstack->context |= LC_FAIL_NEXT;
+                return;
+            }
+            self->topstack->context ^= LC_FAIL_ON_RBRACE;
+        }
+        else if (data == *"{") {
+            self->topstack->context |= LC_FAIL_ON_LBRACE;
+        }
+        else if (data == *"}") {
+            self->topstack->context |= LC_FAIL_ON_RBRACE;
+        }
+    }
+
+    if (context & LC_HAS_TEXT) {
+        if (context & LC_FAIL_ON_TEXT) {
+            if (!Py_UNICODE_ISSPACE(data)) {
+                Tokenizer_fail_route(self);
+                return;
+            }
+        }
+        else {
+            if (data == *"\n") {
+                self->topstack->context |= LC_FAIL_ON_TEXT;
+            }
+        }
+    }
+    else if (!Py_UNICODE_ISSPACE(data)) {
+        self->topstack->context |= LC_HAS_TEXT;
+    }
+}
+
+/*
     Parse the wikicode string, using context for when to stop.
 */
 static PyObject*
 Tokenizer_parse(Tokenizer* self, int context)
 {
-    PyObject *this;
+    static int fail_contexts = LC_TEMPLATE | LC_ARGUMENT | LC_WIKILINK | LC_HEADING | LC_COMMENT;
+    static int unsafe_contexts = LC_TEMPLATE_NAME | LC_WIKILINK_TITLE | LC_TEMPLATE_PARAM_KEY | LC_ARGUMENT_NAME;
+    int this_context, is_marker, i;
     Py_UNICODE this_data, next, next_next, last;
-    int this_context;
-    int fail_contexts = LC_TEMPLATE | LC_ARGUMENT | LC_WIKILINK | LC_HEADING | LC_COMMENT;
-    int is_marker, i;
+    PyObject *this;
 
     if (Tokenizer_push(self, context))
         return NULL;
@@ -1190,6 +1120,12 @@ Tokenizer_parse(Tokenizer* self, int context)
     while (1) {
         this = Tokenizer_read(self, 0);
         this_data = *PyUnicode_AS_UNICODE(this);
+        this_context = self->topstack->context;
+
+        if (this_context & unsafe_contexts) {
+            Tokenizer_verify_safe(self, this_context, this_data);
+            if (BAD_ROUTE) return NULL;
+        }
 
         is_marker = 0;
         for (i = 0; i < NUM_MARKERS; i++) {
@@ -1205,8 +1141,6 @@ Tokenizer_parse(Tokenizer* self, int context)
             continue;
         }
 
-        this_context = self->topstack->context;
-
         if (this_data == *"") {
             if (this_context & LC_TEMPLATE_PARAM_KEY) {
                 PyObject* trash = Tokenizer_pop(self);
diff --git a/mwparserfromhell/parser/tokenizer.h b/mwparserfromhell/parser/tokenizer.h
index 468700c..2888f6b 100644
--- a/mwparserfromhell/parser/tokenizer.h
+++ b/mwparserfromhell/parser/tokenizer.h
@@ -98,7 +98,7 @@ static PyObject* TagCloseClose;
 #define LC_WIKILINK_TITLE       0x0020
 #define LC_WIKILINK_TEXT        0x0040
 
-#define LC_HEADING              0x1f80
+#define LC_HEADING              0x1F80
 #define LC_HEADING_LEVEL_1      0x0080
 #define LC_HEADING_LEVEL_2      0x0100
 #define LC_HEADING_LEVEL_3      0x0200
@@ -108,6 +108,11 @@ static PyObject* TagCloseClose;
 
 #define LC_COMMENT              0x2000
 
+#define LC_HAS_TEXT             0x010000
+#define LC_FAIL_ON_TEXT         0x020000
+#define LC_FAIL_NEXT            0x040000
+#define LC_FAIL_ON_LBRACE       0x080000
+#define LC_FAIL_ON_RBRACE       0x100000
 
 /* Global contexts: */
 
@@ -176,7 +181,6 @@ static PyObject* Tokenizer_read_backwards(Tokenizer*, Py_ssize_t);
 static int Tokenizer_parse_template_or_argument(Tokenizer*);
 static int Tokenizer_parse_template(Tokenizer*);
 static int Tokenizer_parse_argument(Tokenizer*);
-static int Tokenizer_verify_safe(Tokenizer*, const char* []);
 static int Tokenizer_handle_template_param(Tokenizer*);
 static int Tokenizer_handle_template_param_value(Tokenizer*);
 static PyObject* Tokenizer_handle_template_end(Tokenizer*);
@@ -190,6 +194,7 @@ static HeadingData* Tokenizer_handle_heading_end(Tokenizer*);
 static int Tokenizer_really_parse_entity(Tokenizer*);
 static int Tokenizer_parse_entity(Tokenizer*);
 static int Tokenizer_parse_comment(Tokenizer*);
+static void Tokenizer_verify_safe(Tokenizer*, int, Py_UNICODE);
 static PyObject* Tokenizer_parse(Tokenizer*, int);
 static PyObject* Tokenizer_tokenize(Tokenizer*, PyObject*);
 

From 087d606097220e0b24909834458921cdf167ebd1 Mon Sep 17 00:00:00 2001
From: Ben Kurtovic <ben.kurtovic@verizon.net>
Date: Thu, 1 Nov 2012 15:35:42 -0400
Subject: [PATCH 36/47] Recover failing contexts when due to a template or link
 opening.

---
 mwparserfromhell/parser/tokenizer.c | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/mwparserfromhell/parser/tokenizer.c b/mwparserfromhell/parser/tokenizer.c
index 82cffaf..c7219ab 100644
--- a/mwparserfromhell/parser/tokenizer.c
+++ b/mwparserfromhell/parser/tokenizer.c
@@ -1124,7 +1124,8 @@ Tokenizer_parse(Tokenizer* self, int context)
 
         if (this_context & unsafe_contexts) {
             Tokenizer_verify_safe(self, this_context, this_data);
-            if (BAD_ROUTE) return NULL;
+            if (BAD_ROUTE)
+                return NULL;
         }
 
         is_marker = 0;
@@ -1165,6 +1166,8 @@ Tokenizer_parse(Tokenizer* self, int context)
         else if (this_data == next && next == *"{") {
             if (Tokenizer_parse_template_or_argument(self))
                 return NULL;
+            if (self->topstack->context & LC_FAIL_NEXT)
+                self->topstack->context ^= LC_FAIL_NEXT;
         }
         else if (this_data == *"|" && this_context & LC_TEMPLATE) {
             if (Tokenizer_handle_template_param(self))
@@ -1191,6 +1194,8 @@ Tokenizer_parse(Tokenizer* self, int context)
             if (!(this_context & LC_WIKILINK_TITLE)) {
                 if (Tokenizer_parse_wikilink(self))
                     return NULL;
+                if (self->topstack->context & LC_FAIL_NEXT)
+                    self->topstack->context ^= LC_FAIL_NEXT;
             }
             else {
                 Tokenizer_write_text(self, this_data);

From 16024494cf5134587001f7d7160cae8bad2966a4 Mon Sep 17 00:00:00 2001
From: Ben Kurtovic <ben.kurtovic@verizon.net>
Date: Sat, 17 Nov 2012 15:42:16 -0500
Subject: [PATCH 37/47] Mostly implement Tokenizer_really_parse_entity().

---
 mwparserfromhell/parser/tokenizer.c | 223 ++++++++++++++++++++++++++++++------
 mwparserfromhell/parser/tokenizer.h |  50 ++++----
 2 files changed, 211 insertions(+), 62 deletions(-)

diff --git a/mwparserfromhell/parser/tokenizer.c b/mwparserfromhell/parser/tokenizer.c
index c7219ab..708cd8b 100644
--- a/mwparserfromhell/parser/tokenizer.c
+++ b/mwparserfromhell/parser/tokenizer.c
@@ -409,7 +409,7 @@ Tokenizer_parse_template_or_argument(Tokenizer* self)
     self->head += 2;
     unsigned int braces = 2, i;
 
-    while (*Tokenizer_READ(self, 0) == *"{") {
+    while (Tokenizer_READ(self, 0) == *"{") {
         self->head++;
         braces++;
     }
@@ -792,7 +792,7 @@ Tokenizer_parse_heading(Tokenizer* self)
     self->head += 1;
     int best = 1, i;
 
-    while (*Tokenizer_READ(self, 0) == *"=") {
+    while (Tokenizer_READ(self, 0) == *"=") {
         best++;
         self->head++;
     }
@@ -889,7 +889,7 @@ Tokenizer_handle_heading_end(Tokenizer* self)
     Py_ssize_t best = 1;
     int i;
 
-    while (*Tokenizer_READ(self, 0) == *"=") {
+    while (Tokenizer_READ(self, 0) == *"=") {
         best++;
         self->head++;
     }
@@ -949,7 +949,155 @@ Tokenizer_handle_heading_end(Tokenizer* self)
 static int
 Tokenizer_really_parse_entity(Tokenizer* self)
 {
-    return 0;
+    PyObject *token, *kwargs, *textobj;
+    Py_UNICODE this;
+    int numeric, hexadecimal, i, j, test;
+    char *valid, *def;
+    char text[];
+
+    token = PyObject_CallObject(HTMLEntityStart, NULL);
+    if (!token) return -1;
+    if (Tokenizer_write(self, token)) {
+        Py_DECREF(token);
+        return -1;
+    }
+    Py_DECREF(token);
+
+    self->head++;
+
+    this = Tokenizer_READ(self, 0);
+    if (this == *"") {
+        Tokenizer_fail_route();
+        return 0;
+    }
+    if (this == *"#") {
+        numeric = 1;
+        token = PyObject_CallObject(HTMLEntityNumeric, NULL);
+        if (!token) return -1;
+        if (Tokenizer_write(self, token)) {
+            Py_DECREF(token);
+            return -1;
+        }
+        Py_DECREF(token);
+
+        self->head++;
+        this = Tokenizer_READ(self, 0);
+        if (this == *"") {
+            Tokenizer_fail_route();
+            return 0;
+        }
+        if (this == *"x" || this == *"X") {
+            hexadecimal = 1;
+            kwargs = PyDict_New();
+            if (!kwargs) return -1;
+            PyDict_SetItemString(kwargs, "char", Tokenizer_read(self, 0));
+            PyObject* token = PyObject_Call(HTMLEntityHex, NOARGS, kwargs);
+            Py_DECREF(kwargs);
+            if (!token) return -1;
+            if (Tokenizer_write(self, token)) {
+                Py_DECREF(token);
+                return -1;
+            }
+            Py_DECREF(token);
+            self->head++;
+        }
+        else {
+            hexadecimal = 0;
+        }
+    }
+    else {
+        numeric = hexadecimal = 0;
+    }
+
+    if (hexadecimal)
+        valid = "0123456789abcdefABCDEF";
+    else if (numeric)
+        valid = "0123456789";
+    else
+        valid = "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ";
+
+    text = PyObject_Malloc(8 * sizeof(char));  // Max theoretical size
+    if (!text) {
+        PyErr_NoMemory();
+        return -1;
+    }
+
+    #define FAIL_ROUTE_AND_EXIT() { \
+        Tokenizer_fail_route();     \
+        PyObject_Free(text);        \
+        return 0;                   \
+    }
+
+    i = 0;
+    while (1) {
+        this = Tokenizer_READ(self, 0);
+        if (this == *";") {
+            if (i == 0)
+                FAIL_ROUTE_AND_EXIT()
+            break;
+        }
+        if (i == 0 && this == *"0") {
+            self->head++;
+            continue;
+        }
+        if (i >= 8)
+            FAIL_ROUTE_AND_EXIT()
+        for (j = 0; j < NUM_MARKERS; j++) {
+            if (this == *MARKERS[j])
+                FAIL_ROUTE_AND_EXIT()
+        }
+        text[i] = this;
+        self->head++;
+        i++;
+    }
+
+    if (numeric) {
+        sscanf(text, (hexadecimal ? "%x" : "%d"), &test);
+        if (test < 1 || test > 0x10FFFF)
+            FAIL_ROUTE_AND_EXIT()
+    }
+    else {
+        i = 0;
+        while (1) {
+            def = entitydefs[i];
+            if (!def)  // We've reached the end of the def list without finding it
+                FAIL_ROUTE_AND_EXIT()
+            if (strcmp(text, def) == 0)
+                break;
+            i++;
+        }
+    }
+
+    textobj = PyUnicode_FromString(text);
+    if (!textobj) {
+        PyObject_Free(text);
+        return -1;
+    }
+    PyObject_Free(text);
+
+    kwargs = PyDict_New();
+    if (!kwargs) {
+        Py_DECREF(textobj);
+        return -1;
+    }
+    PyDict_SetItemString(kwargs, "text", textobj);
+    Py_DECREF(textobj);
+    PyObject* token = PyObject_Call(Text, NOARGS, kwargs);
+    Py_DECREF(kwargs);
+    if (!token) return -1;
+    if (Tokenizer_write(self, token)) {
+        Py_DECREF(token);
+        return -1;
+    }
+    Py_DECREF(token);
+
+    token = PyObject_CallObject(HTMLEntityEnd, NULL);
+    if (!token) return -1;
+    if (Tokenizer_write(self, token)) {
+        Py_DECREF(token);
+        return -1;
+    }
+    Py_DECREF(token);
 }
 
 /*
@@ -968,7 +1116,7 @@ Tokenizer_parse_entity(Tokenizer* self)
     if (BAD_ROUTE) {
         RESET_ROUTE();
         self->head = reset;
-        if (Tokenizer_write_text(self, *PyUnicode_AS_UNICODE(Tokenizer_read(self, 0))))
+        if (Tokenizer_write_text(self, *"&"))
             return -1;
         return 0;
     }
@@ -1111,38 +1259,37 @@ Tokenizer_parse(Tokenizer* self, int context)
     static int fail_contexts = LC_TEMPLATE | LC_ARGUMENT | LC_WIKILINK | LC_HEADING | LC_COMMENT;
     static int unsafe_contexts = LC_TEMPLATE_NAME | LC_WIKILINK_TITLE | LC_TEMPLATE_PARAM_KEY | LC_ARGUMENT_NAME;
     int this_context, is_marker, i;
-    Py_UNICODE this_data, next, next_next, last;
+    Py_UNICODE this, next, next_next, last;
     PyObject *this;
 
     if (Tokenizer_push(self, context))
         return NULL;
 
     while (1) {
-        this = Tokenizer_read(self, 0);
-        this_data = *PyUnicode_AS_UNICODE(this);
+        this = Tokenizer_READ(self, 0);
         this_context = self->topstack->context;
 
         if (this_context & unsafe_contexts) {
-            Tokenizer_verify_safe(self, this_context, this_data);
+            Tokenizer_verify_safe(self, this_context, this);
             if (BAD_ROUTE)
                 return NULL;
         }
 
         is_marker = 0;
         for (i = 0; i < NUM_MARKERS; i++) {
-            if (*MARKERS[i] == this_data) {
+            if (*MARKERS[i] == this) {
                 is_marker = 1;
                 break;
             }
         }
 
         if (!is_marker) {
-            Tokenizer_write_text(self, this_data);
+            Tokenizer_write_text(self, this);
             self->head++;
             continue;
         }
 
-        if (this_data == *"") {
+        if (this == *"") {
             if (this_context & LC_TEMPLATE_PARAM_KEY) {
                 PyObject* trash = Tokenizer_pop(self);
                 Py_XDECREF(trash);
@@ -1153,44 +1300,44 @@ Tokenizer_parse(Tokenizer* self, int context)
             return Tokenizer_pop(self);
         }
 
-        next = *Tokenizer_READ(self, 1);
+        next = Tokenizer_READ(self, 1);
 
         if (this_context & LC_COMMENT) {
-            if (this_data == next && next == *"-") {
-                if (*Tokenizer_READ(self, 2) == *">") {
+            if (this == next && next == *"-") {
+                if (Tokenizer_READ(self, 2) == *">") {
                     return Tokenizer_pop(self);
                 }
             }
-            Tokenizer_write_text(self, this_data);
+            Tokenizer_write_text(self, this);
         }
-        else if (this_data == next && next == *"{") {
+        else if (this == next && next == *"{") {
             if (Tokenizer_parse_template_or_argument(self))
                 return NULL;
             if (self->topstack->context & LC_FAIL_NEXT)
                 self->topstack->context ^= LC_FAIL_NEXT;
         }
-        else if (this_data == *"|" && this_context & LC_TEMPLATE) {
+        else if (this == *"|" && this_context & LC_TEMPLATE) {
             if (Tokenizer_handle_template_param(self))
                 return NULL;
         }
-        else if (this_data == *"=" && this_context & LC_TEMPLATE_PARAM_KEY) {
+        else if (this == *"=" && this_context & LC_TEMPLATE_PARAM_KEY) {
             if (Tokenizer_handle_template_param_value(self))
                 return NULL;
         }
-        else if (this_data == next && next == *"}" && this_context & LC_TEMPLATE) {
+        else if (this == next && next == *"}" && this_context & LC_TEMPLATE) {
             return Tokenizer_handle_template_end(self);
         }
-        else if (this_data == *"|" && this_context & LC_ARGUMENT_NAME) {
+        else if (this == *"|" && this_context & LC_ARGUMENT_NAME) {
             if (Tokenizer_handle_argument_separator(self))
                 return NULL;
         }
-        else if (this_data == next && next == *"}" && this_context & LC_ARGUMENT) {
-            if (*Tokenizer_READ(self, 2) == *"}") {
+        else if (this == next && next == *"}" && this_context & LC_ARGUMENT) {
+            if (Tokenizer_READ(self, 2) == *"}") {
                 return Tokenizer_handle_argument_end(self);
             }
-            Tokenizer_write_text(self, this_data);
+            Tokenizer_write_text(self, this);
         }
-        else if (this_data == next && next == *"[") {
+        else if (this == next && next == *"[") {
             if (!(this_context & LC_WIKILINK_TITLE)) {
                 if (Tokenizer_parse_wikilink(self))
                     return NULL;
@@ -1198,48 +1345,48 @@ Tokenizer_parse(Tokenizer* self, int context)
                     self->topstack->context ^= LC_FAIL_NEXT;
             }
             else {
-                Tokenizer_write_text(self, this_data);
+                Tokenizer_write_text(self, this);
             }
         }
-        else if (this_data == *"|" && this_context & LC_WIKILINK_TITLE) {
+        else if (this == *"|" && this_context & LC_WIKILINK_TITLE) {
             if (Tokenizer_handle_wikilink_separator(self))
                 return NULL;
         }
-        else if (this_data == next && next == *"]" && this_context & LC_WIKILINK) {
+        else if (this == next && next == *"]" && this_context & LC_WIKILINK) {
             return Tokenizer_handle_wikilink_end(self);
         }
-        else if (this_data == *"=" && !(self->global & GL_HEADING)) {
+        else if (this == *"=" && !(self->global & GL_HEADING)) {
             last = *PyUnicode_AS_UNICODE(Tokenizer_read_backwards(self, 1));
             if (last == *"\n" || last == *"") {
                 if (Tokenizer_parse_heading(self))
                     return NULL;
             }
             else {
-                Tokenizer_write_text(self, this_data);
+                Tokenizer_write_text(self, this);
             }
         }
-        else if (this_data == *"=" && this_context & LC_HEADING) {
+        else if (this == *"=" && this_context & LC_HEADING) {
             return (PyObject*) Tokenizer_handle_heading_end(self);
         }
-        else if (this_data == *"\n" && this_context & LC_HEADING) {
+        else if (this == *"\n" && this_context & LC_HEADING) {
             return Tokenizer_fail_route(self);
         }
-        else if (this_data == *"&") {
+        else if (this == *"&") {
             if (Tokenizer_parse_entity(self))
                 return NULL;
         }
-        else if (this_data == *"<" && next == *"!") {
-            next_next = *Tokenizer_READ(self, 2);
-            if (next_next == *Tokenizer_READ(self, 3) && next_next == *"-") {
+        else if (this == *"<" && next == *"!") {
+            next_next = Tokenizer_READ(self, 2);
+            if (next_next == Tokenizer_READ(self, 3) && next_next == *"-") {
                 if (Tokenizer_parse_comment(self))
                     return NULL;
             }
             else {
-                Tokenizer_write_text(self, this_data);
+                Tokenizer_write_text(self, this);
             }
         }
         else {
-            Tokenizer_write_text(self, this_data);
+            Tokenizer_write_text(self, this);
         }
 
         self->head++;
diff --git a/mwparserfromhell/parser/tokenizer.h b/mwparserfromhell/parser/tokenizer.h
index 2888f6b..9e94dbc 100644
--- a/mwparserfromhell/parser/tokenizer.h
+++ b/mwparserfromhell/parser/tokenizer.h
@@ -41,6 +41,8 @@ static int route_state = 0;
 #define FAIL_ROUTE()  (route_state = 1)
 #define RESET_ROUTE() (route_state = 0)
 
+static char* entitydefs[];
+
 static PyObject* EMPTY;
 static PyObject* NOARGS;
 static PyObject* tokens;
@@ -85,34 +87,34 @@ static PyObject* TagCloseClose;
 
 /* Local contexts: */
 
-#define LC_TEMPLATE             0x0007
-#define LC_TEMPLATE_NAME        0x0001
-#define LC_TEMPLATE_PARAM_KEY   0x0002
-#define LC_TEMPLATE_PARAM_VALUE 0x0004
+#define LC_TEMPLATE             0x00007
+#define LC_TEMPLATE_NAME        0x00001
+#define LC_TEMPLATE_PARAM_KEY   0x00002
+#define LC_TEMPLATE_PARAM_VALUE 0x00004
 
-#define LC_ARGUMENT             0x0018
-#define LC_ARGUMENT_NAME        0x0008
-#define LC_ARGUMENT_DEFAULT     0x0010
+#define LC_ARGUMENT             0x00018
+#define LC_ARGUMENT_NAME        0x00008
+#define LC_ARGUMENT_DEFAULT     0x00010
 
-#define LC_WIKILINK             0x0060
-#define LC_WIKILINK_TITLE       0x0020
-#define LC_WIKILINK_TEXT        0x0040
+#define LC_WIKILINK             0x00060
+#define LC_WIKILINK_TITLE       0x00020
+#define LC_WIKILINK_TEXT        0x00040
 
-#define LC_HEADING              0x1F80
-#define LC_HEADING_LEVEL_1      0x0080
-#define LC_HEADING_LEVEL_2      0x0100
-#define LC_HEADING_LEVEL_3      0x0200
-#define LC_HEADING_LEVEL_4      0x0400
-#define LC_HEADING_LEVEL_5      0x0800
-#define LC_HEADING_LEVEL_6      0x1000
+#define LC_HEADING              0x01F80
+#define LC_HEADING_LEVEL_1      0x00080
+#define LC_HEADING_LEVEL_2      0x00100
+#define LC_HEADING_LEVEL_3      0x00200
+#define LC_HEADING_LEVEL_4      0x00400
+#define LC_HEADING_LEVEL_5      0x00800
+#define LC_HEADING_LEVEL_6      0x01000
 
-#define LC_COMMENT              0x2000
+#define LC_COMMENT              0x02000
 
-#define LC_HAS_TEXT             0x010000
-#define LC_FAIL_ON_TEXT         0x020000
-#define LC_FAIL_NEXT            0x040000
-#define LC_FAIL_ON_LBRACE       0x080000
-#define LC_FAIL_ON_RBRACE       0x100000
+#define LC_HAS_TEXT             0x04000
+#define LC_FAIL_ON_TEXT         0x08000
+#define LC_FAIL_NEXT            0x10000
+#define LC_FAIL_ON_LBRACE       0x20000
+#define LC_FAIL_ON_RBRACE       0x40000
 
 /* Global contexts: */
 
@@ -154,7 +156,7 @@ typedef struct {
 
 /* Macros for accessing Tokenizer data: */
 
-#define Tokenizer_READ(self, delta) PyUnicode_AS_UNICODE(Tokenizer_read(self, delta))
+#define Tokenizer_READ(self, delta) (*PyUnicode_AS_UNICODE(Tokenizer_read(self, delta)))
 
 
 /* Function prototypes: */

From 06b20dd8c0e947c8b48dbb59b62bd72afc1a81d2 Mon Sep 17 00:00:00 2001
From: Ben Kurtovic <ben.kurtovic@verizon.net>
Date: Sat, 17 Nov 2012 16:15:23 -0500
Subject: [PATCH 38/47] More updates.

---
 mwparserfromhell/parser/tokenizer.c | 82 ++++++++++++++++++++++---------------
 mwparserfromhell/parser/tokenizer.h |  6 ++-
 2 files changed, 54 insertions(+), 34 deletions(-)

diff --git a/mwparserfromhell/parser/tokenizer.c b/mwparserfromhell/parser/tokenizer.c
index 708cd8b..0935770 100644
--- a/mwparserfromhell/parser/tokenizer.c
+++ b/mwparserfromhell/parser/tokenizer.c
@@ -33,15 +33,15 @@ Tokenizer_new(PyTypeObject* type, PyObject* args, PyObject* kwds)
 static struct Textbuffer*
 Textbuffer_new(void)
 {
-    struct Textbuffer* buffer = PyObject_Malloc(sizeof(struct Textbuffer));
+    struct Textbuffer* buffer = malloc(sizeof(struct Textbuffer));
     if (!buffer) {
         PyErr_NoMemory();
         return NULL;
     }
     buffer->size = 0;
-    buffer->data = PyObject_Malloc(sizeof(Py_UNICODE) * TEXTBUFFER_BLOCKSIZE);
+    buffer->data = malloc(sizeof(Py_UNICODE) * TEXTBUFFER_BLOCKSIZE);
     if (!buffer->data) {
-        PyObject_Free(buffer);
+        free(buffer);
         PyErr_NoMemory();
         return NULL;
     }
@@ -58,7 +58,7 @@ Tokenizer_dealloc(Tokenizer* self)
         Py_DECREF(this->stack);
         Textbuffer_dealloc(this->textbuffer);
         next = this->next;
-        PyObject_Free(this);
+        free(this);
         this = next;
     }
     self->ob_type->tp_free((PyObject*) self);
@@ -69,9 +69,9 @@ Textbuffer_dealloc(struct Textbuffer* this)
 {
     struct Textbuffer* next;
     while (this) {
-        PyObject_Free(this->data);
+        free(this->data);
         next = this->next;
-        PyObject_Free(this);
+        free(this);
         this = next;
     }
 }
@@ -98,7 +98,7 @@ Tokenizer_init(Tokenizer* self, PyObject* args, PyObject* kwds)
 static int
 Tokenizer_push(Tokenizer* self, int context)
 {
-    struct Stack* top = PyObject_Malloc(sizeof(struct Stack));
+    struct Stack* top = malloc(sizeof(struct Stack));
     if (!top) {
         PyErr_NoMemory();
         return -1;
@@ -180,7 +180,7 @@ Tokenizer_delete_top_of_stack(Tokenizer* self)
     Py_DECREF(top->stack);
     Textbuffer_dealloc(top->textbuffer);
     self->topstack = top->next;
-    PyObject_Free(top);
+    free(top);
 }
 
 /*
@@ -815,7 +815,7 @@ Tokenizer_parse_heading(Tokenizer* self)
     PyObject* level = PyInt_FromSsize_t(heading->level);
     if (!level) {
         Py_DECREF(heading->title);
-        PyObject_Free(heading);
+        free(heading);
         return -1;
     }
 
@@ -823,7 +823,7 @@ Tokenizer_parse_heading(Tokenizer* self)
     if (!kwargs) {
         Py_DECREF(level);
         Py_DECREF(heading->title);
-        PyObject_Free(heading);
+        free(heading);
         return -1;
     }
     PyDict_SetItemString(kwargs, "level", level);
@@ -833,14 +833,14 @@ Tokenizer_parse_heading(Tokenizer* self)
     Py_DECREF(kwargs);
     if (!token) {
         Py_DECREF(heading->title);
-        PyObject_Free(heading);
+        free(heading);
         return -1;
     }
 
     if (Tokenizer_write(self, token)) {
         Py_DECREF(token);
         Py_DECREF(heading->title);
-        PyObject_Free(heading);
+        free(heading);
         return -1;
     }
     Py_DECREF(token);
@@ -852,18 +852,18 @@ Tokenizer_parse_heading(Tokenizer* self)
         difftext[diff] = *"";
         if (Tokenizer_write_text_then_stack(self, difftext)) {
             Py_DECREF(heading->title);
-            PyObject_Free(heading);
+            free(heading);
             return -1;
         }
     }
 
     if (Tokenizer_write_all(self, heading->title)) {
         Py_DECREF(heading->title);
-        PyObject_Free(heading);
+        free(heading);
         return -1;
     }
     Py_DECREF(heading->title);
-    PyObject_Free(heading);
+    free(heading);
 
     token = PyObject_CallObject(HeadingEnd, NULL);
     if (!token) return -1;
@@ -917,23 +917,23 @@ Tokenizer_handle_heading_end(Tokenizer* self)
         text[best] = *"";
         if (Tokenizer_write_text_then_stack(self, text)) {
             Py_DECREF(after->title);
-            PyObject_Free(after);
+            free(after);
             return NULL;
         }
         if (Tokenizer_write_all(self, after->title)) {
             Py_DECREF(after->title);
-            PyObject_Free(after);
+            free(after);
             return NULL;
         }
         Py_DECREF(after->title);
         level = after->level;
-        PyObject_Free(after);
+        free(after);
     }
 
     PyObject* stack = Tokenizer_pop(self);
     if (!stack) return NULL;
 
-    HeadingData* heading = PyObject_Malloc(sizeof(HeadingData));
+    HeadingData* heading = malloc(sizeof(HeadingData));
     if (!heading) {
         PyErr_NoMemory();
         return NULL;
@@ -952,8 +952,7 @@ Tokenizer_really_parse_entity(Tokenizer* self)
     PyObject *token, *kwargs, *textobj;
     Py_UNICODE this;
     int numeric, hexadecimal, i, j, test;
-    char *valid, *def;
-    char text[];
+    char *valid, *text, *def;
 
     token = PyObject_CallObject(HTMLEntityStart, NULL);
     if (!token) return -1;
@@ -967,7 +966,7 @@ Tokenizer_really_parse_entity(Tokenizer* self)
 
     this = Tokenizer_READ(self, 0);
     if (this == *"") {
-        Tokenizer_fail_route();
+        Tokenizer_fail_route(self);
         return 0;
     }
     if (this == *"#") {
@@ -983,7 +982,7 @@ Tokenizer_really_parse_entity(Tokenizer* self)
         self->head++;
         this = Tokenizer_READ(self, 0);
         if (this == *"") {
-            Tokenizer_fail_route();
+            Tokenizer_fail_route(self);
             return 0;
         }
         if (this == *"x" || this == *"X") {
@@ -1016,15 +1015,15 @@ Tokenizer_really_parse_entity(Tokenizer* self)
     else
         valid = "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ";
 
-    text = PyObject_Malloc(8 * sizeof(char));  // Max theoretical size
+    text = malloc(MAX_ENTITY_SIZE * sizeof(char));
     if (!text) {
         PyErr_NoMemory();
         return -1;
     }
 
     #define FAIL_ROUTE_AND_EXIT() { \
-        Tokenizer_fail_route();     \
-        PyObject_Free(text);        \
+        Tokenizer_fail_route(self); \
+        free(text);                 \
         return 0;                   \
     }
 
@@ -1070,10 +1069,10 @@ Tokenizer_really_parse_entity(Tokenizer* self)
 
     textobj = PyUnicode_FromString(text);
     if (!textobj) {
-        PyObject_Free(text);
+        free(text);
         return -1;
     }
-    PyObject_Free(text);
+    free(text);
 
     kwargs = PyDict_New();
     if (!kwargs) {
@@ -1082,7 +1081,7 @@ Tokenizer_really_parse_entity(Tokenizer* self)
     }
     PyDict_SetItemString(kwargs, "text", textobj);
     Py_DECREF(textobj);
-    PyObject* token = PyObject_Call(Text, NOARGS, kwargs);
+    token = PyObject_Call(Text, NOARGS, kwargs);
     Py_DECREF(kwargs);
     if (!token) return -1;
     if (Tokenizer_write(self, token)) {
@@ -1098,6 +1097,7 @@ Tokenizer_really_parse_entity(Tokenizer* self)
         return -1;
     }
     Py_DECREF(token);
+    return 0;
 }
 
 /*
@@ -1260,7 +1260,6 @@ Tokenizer_parse(Tokenizer* self, int context)
     static int unsafe_contexts = LC_TEMPLATE_NAME | LC_WIKILINK_TITLE | LC_TEMPLATE_PARAM_KEY | LC_ARGUMENT_NAME;
     int this_context, is_marker, i;
     Py_UNICODE this, next, next_next, last;
-    PyObject *this;
 
     if (Tokenizer_push(self, context))
         return NULL;
@@ -1445,6 +1444,25 @@ init_tokenizer(void)
     Py_INCREF(&TokenizerType);
     PyModule_AddObject(module, "CTokenizer", (PyObject*) &TokenizerType);
 
+    PyObject* htmlentitydefs = PyImport_ImportModule("htmlentitydefs");
+    if (!htmlentitydefs) return;
+
+    PyObject* defmap = PyObject_GetAttrString(htmlentitydefs, "entitydefs");
+    if (!defmap) return;
+    Py_DECREF(htmlentitydefs);
+
+    unsigned numdefs = (unsigned) PyDict_Size(defmap);
+    entitydefs = malloc(numdefs * sizeof(char));
+    PyObject* deflist = PyDict_Keys(defmap);
+    if (!deflist) return;
+    Py_DECREF(defmap);
+
+    unsigned i;
+    for (i = 0; i < numdefs; i++) {
+        entitydefs[i] = PyString_AsString(PyList_GET_ITEM(deflist, i));
+    }
+    Py_DECREF(deflist);
+
     EMPTY = PyUnicode_FromString("");
     NOARGS = PyTuple_New(0);
 
@@ -1462,9 +1480,7 @@ init_tokenizer(void)
 
     PyObject* tokmodule = PyImport_ImportModuleLevel(name, globals, locals, fromlist, 0);
     Py_DECREF(fromlist);
-    if (!tokmodule) {
-        return;
-    }
+    if (!tokmodule) return;
 
     tokens = PyObject_GetAttrString(tokmodule, "tokens");
     Py_DECREF(tokmodule);
diff --git a/mwparserfromhell/parser/tokenizer.h b/mwparserfromhell/parser/tokenizer.h
index 9e94dbc..67c39cd 100644
--- a/mwparserfromhell/parser/tokenizer.h
+++ b/mwparserfromhell/parser/tokenizer.h
@@ -29,19 +29,23 @@ SOFTWARE.
 #include <math.h>
 #include <structmember.h>
 
+#define malloc PyObject_Malloc
+#define free   PyObject_Free
+
 static const char* MARKERS[] = {
     "{",  "}", "[", "]", "<", ">", "|", "=", "&", "#", "*", ";", ":", "/", "-",
     "!", "\n", ""};
 
 #define NUM_MARKERS 18
 #define TEXTBUFFER_BLOCKSIZE 1024
+#define MAX_ENTITY_SIZE 8
 
 static int route_state = 0;
 #define BAD_ROUTE     (route_state)
 #define FAIL_ROUTE()  (route_state = 1)
 #define RESET_ROUTE() (route_state = 0)
 
-static char* entitydefs[];
+static char** entitydefs;
 
 static PyObject* EMPTY;
 static PyObject* NOARGS;

From 19e7c3b6acd5ca2ab6b40413bab5f6779ab2e8d9 Mon Sep 17 00:00:00 2001
From: Ben Kurtovic <ben.kurtovic@verizon.net>
Date: Sat, 17 Nov 2012 16:47:09 -0500
Subject: [PATCH 39/47] Fix type.

---
 mwparserfromhell/parser/tokenizer.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/mwparserfromhell/parser/tokenizer.c b/mwparserfromhell/parser/tokenizer.c
index 0935770..18972aa 100644
--- a/mwparserfromhell/parser/tokenizer.c
+++ b/mwparserfromhell/parser/tokenizer.c
@@ -1452,7 +1452,7 @@ init_tokenizer(void)
     Py_DECREF(htmlentitydefs);
 
     unsigned numdefs = (unsigned) PyDict_Size(defmap);
-    entitydefs = malloc(numdefs * sizeof(char));
+    entitydefs = malloc(numdefs * sizeof(char*));
     PyObject* deflist = PyDict_Keys(defmap);
     if (!deflist) return;
     Py_DECREF(defmap);

From a1e6dfcef67a056ccb930e111a025db46093c398 Mon Sep 17 00:00:00 2001
From: Ben Kurtovic <ben.kurtovic@verizon.net>
Date: Sat, 17 Nov 2012 16:52:01 -0500
Subject: [PATCH 40/47] Ensure that there is null terminator at the end of
 entitydefs.

---
 mwparserfromhell/parser/tokenizer.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/mwparserfromhell/parser/tokenizer.c b/mwparserfromhell/parser/tokenizer.c
index 18972aa..a8c69e7 100644
--- a/mwparserfromhell/parser/tokenizer.c
+++ b/mwparserfromhell/parser/tokenizer.c
@@ -1452,7 +1452,7 @@ init_tokenizer(void)
     Py_DECREF(htmlentitydefs);
 
     unsigned numdefs = (unsigned) PyDict_Size(defmap);
-    entitydefs = malloc(numdefs * sizeof(char*));
+    entitydefs = calloc(numdefs + 1, sizeof(char*));
     PyObject* deflist = PyDict_Keys(defmap);
     if (!deflist) return;
     Py_DECREF(defmap);

From 0e78571672ea99727a854985e63e4ba50ee1f6f8 Mon Sep 17 00:00:00 2001
From: Ben Kurtovic <ben.kurtovic@verizon.net>
Date: Sat, 17 Nov 2012 16:57:02 -0500
Subject: [PATCH 41/47] Forgot to check numeric/hexadecimal entities for string
 chars.

---
 mwparserfromhell/parser/tokenizer.c | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/mwparserfromhell/parser/tokenizer.c b/mwparserfromhell/parser/tokenizer.c
index a8c69e7..a3e2a95 100644
--- a/mwparserfromhell/parser/tokenizer.c
+++ b/mwparserfromhell/parser/tokenizer.c
@@ -1045,6 +1045,14 @@ Tokenizer_really_parse_entity(Tokenizer* self)
             if (this == *MARKERS[j])
                 FAIL_ROUTE_AND_EXIT()
         }
+        j = 0;
+        while (1) {
+            if (!valid[j])
+                FAIL_ROUTE_AND_EXIT()
+            if (this == valid[j])
+                break;
+            j++;
+        }
         text[i] = this;
         self->head++;
         i++;

From 06f02b9753c5e614cb4f74218163ea1c5f4fc398 Mon Sep 17 00:00:00 2001
From: Ben Kurtovic <ben.kurtovic@verizon.net>
Date: Sat, 17 Nov 2012 17:12:02 -0500
Subject: [PATCH 42/47] Clean up whitespace / newlines.

---
 mwparserfromhell/parser/tokenizer.c | 231 +++++++++++++++++-------------------
 1 file changed, 108 insertions(+), 123 deletions(-)

diff --git a/mwparserfromhell/parser/tokenizer.c b/mwparserfromhell/parser/tokenizer.c
index a3e2a95..40f91a1 100644
--- a/mwparserfromhell/parser/tokenizer.c
+++ b/mwparserfromhell/parser/tokenizer.c
@@ -106,9 +106,8 @@ Tokenizer_push(Tokenizer* self, int context)
     top->stack = PyList_New(0);
     top->context = context;
     top->textbuffer = Textbuffer_new();
-    if (!top->textbuffer) {
+    if (!top->textbuffer)
         return -1;
-    }
     top->next = self->topstack;
     self->topstack = top;
     return 0;
@@ -140,11 +139,11 @@ static int
 Tokenizer_push_textbuffer(Tokenizer* self)
 {
     struct Textbuffer* buffer = self->topstack->textbuffer;
-    if (buffer->size == 0 && !buffer->next) {
+    if (buffer->size == 0 && !buffer->next)
         return 0;
-    }
     PyObject* text = Textbuffer_render(buffer);
-    if (!text) return -1;
+    if (!text)
+        return -1;
 
     PyObject* kwargs = PyDict_New();
     if (!kwargs) {
@@ -156,20 +155,19 @@ Tokenizer_push_textbuffer(Tokenizer* self)
 
     PyObject* token = PyObject_Call(Text, NOARGS, kwargs);
     Py_DECREF(kwargs);
-    if (!token) return -1;
+    if (!token)
+        return -1;
 
     if (PyList_Append(self->topstack->stack, token)) {
         Py_DECREF(token);
         return -1;
     }
-
     Py_DECREF(token);
 
     Textbuffer_dealloc(buffer);
     self->topstack->textbuffer = Textbuffer_new();
-    if (!self->topstack->textbuffer) {
+    if (!self->topstack->textbuffer)
         return -1;
-    }
     return 0;
 }
 
@@ -239,10 +237,8 @@ Tokenizer_write(Tokenizer* self, PyObject* token)
 {
     if (Tokenizer_push_textbuffer(self))
         return -1;
-
     if (PyList_Append(self->topstack->stack, token))
         return -1;
-
     return 0;
 }
 
@@ -254,10 +250,8 @@ Tokenizer_write_first(Tokenizer* self, PyObject* token)
 {
     if (Tokenizer_push_textbuffer(self))
         return -1;
-
     if (PyList_Insert(self->topstack->stack, 0, token))
         return -1;
-
     return 0;
 }
 
@@ -270,9 +264,8 @@ Tokenizer_write_text(Tokenizer* self, Py_UNICODE text)
     struct Textbuffer* buf = self->topstack->textbuffer;
     if (buf->size == TEXTBUFFER_BLOCKSIZE) {
         struct Textbuffer* new = Textbuffer_new();
-        if (!new) {
+        if (!new)
             return -1;
-        }
         new->next = buf;
         self->topstack->textbuffer = new;
         buf = new;
@@ -297,18 +290,20 @@ Tokenizer_write_all(Tokenizer* self, PyObject* tokenlist)
             case 1: {
                 pushed = 1;
                 struct Textbuffer* buffer = self->topstack->textbuffer;
-                if (buffer->size == 0 && !buffer->next) {
+                if (buffer->size == 0 && !buffer->next)
                     break;
-                }
                 PyObject* left = Textbuffer_render(buffer);
-                if (!left) return -1;
+                if (!left)
+                    return -1;
                 PyObject* right = PyObject_GetAttrString(token, "text");
-                if (!right) return -1;
+                if (!right)
+                    return -1;
 
                 PyObject* text = PyUnicode_Concat(left, right);
                 Py_DECREF(left);
                 Py_DECREF(right);
-                if (!text) return -1;
+                if (!text)
+                    return -1;
 
                 if (PyObject_SetAttrString(token, "text", text)) {
                     Py_DECREF(text);
@@ -318,9 +313,8 @@ Tokenizer_write_all(Tokenizer* self, PyObject* tokenlist)
 
                 Textbuffer_dealloc(buffer);
                 self->topstack->textbuffer = Textbuffer_new();
-                if (!self->topstack->textbuffer) {
+                if (!self->topstack->textbuffer)
                     return -1;
-                }
                 break;
             }
             case -1:
@@ -334,10 +328,8 @@ Tokenizer_write_all(Tokenizer* self, PyObject* tokenlist)
 
     PyObject* stack = self->topstack->stack;
     Py_ssize_t size = PyList_GET_SIZE(stack);
-
     if (PyList_SetSlice(stack, size, size, tokenlist))
         return -1;
-
     return 0;
 }
 
@@ -351,7 +343,8 @@ Tokenizer_write_text_then_stack(Tokenizer* self, const char* text)
     PyObject* stack = Tokenizer_pop(self);
     int i = 0;
     while (1) {
-        if (!text[i]) break;
+        if (!text[i])
+            break;
         if (Tokenizer_write_text(self, (Py_UNICODE) text[i])) {
             Py_XDECREF(stack);
             return -1;
@@ -380,10 +373,8 @@ static PyObject*
 Tokenizer_read(Tokenizer* self, Py_ssize_t delta)
 {
     Py_ssize_t index = self->head + delta;
-
     if (index >= self->length)
         return EMPTY;
-
     return PyList_GET_ITEM(self->text, index);
 }
 
@@ -395,7 +386,6 @@ Tokenizer_read_backwards(Tokenizer* self, Py_ssize_t delta)
 {
     if (delta > self->head)
         return EMPTY;
-
     Py_ssize_t index = self->head - delta;
     return PyList_GET_ITEM(self->text, index);
 }
@@ -457,21 +447,19 @@ Tokenizer_parse_template_or_argument(Tokenizer* self)
                 Py_XDECREF(text);
                 return 0;
             }
-            else {
+            else
                 braces -= 2;
-            }
         }
-        else {
+        else
             braces -= 3;
-        }
 
-        if (braces) {
+        if (braces)
             self->head++;
-        }
     }
 
     PyObject* tokenlist = Tokenizer_pop(self);
-    if (!tokenlist) return -1;
+    if (!tokenlist)
+        return -1;
     if (Tokenizer_write_all(self, tokenlist)) {
         Py_DECREF(tokenlist);
         return -1;
@@ -495,7 +483,8 @@ Tokenizer_parse_template(Tokenizer* self)
         self->head = reset;
         return 0;
     }
-    if (!template) return -1;
+    if (!template)
+        return -1;
 
     token = PyObject_CallObject(TemplateOpen, NULL);
     if (!token) {
@@ -517,14 +506,14 @@ Tokenizer_parse_template(Tokenizer* self)
     Py_DECREF(template);
 
     token = PyObject_CallObject(TemplateClose, NULL);
-    if (!token) return -1;
+    if (!token)
+        return -1;
 
     if (Tokenizer_write(self, token)) {
         Py_DECREF(token);
         return -1;
     }
     Py_DECREF(token);
-
     return 0;
 }
 
@@ -542,7 +531,8 @@ Tokenizer_parse_argument(Tokenizer* self)
         self->head = reset;
         return 0;
     }
-    if (!argument) return -1;
+    if (!argument)
+        return -1;
 
     token = PyObject_CallObject(ArgumentOpen, NULL);
     if (!token) {
@@ -564,14 +554,14 @@ Tokenizer_parse_argument(Tokenizer* self)
     Py_DECREF(argument);
 
     token = PyObject_CallObject(ArgumentClose, NULL);
-    if (!token) return -1;
+    if (!token)
+        return -1;
 
     if (Tokenizer_write(self, token)) {
         Py_DECREF(token);
         return -1;
     }
     Py_DECREF(token);
-
     return 0;
 }
 
@@ -581,28 +571,27 @@ Tokenizer_parse_argument(Tokenizer* self)
 static int
 Tokenizer_handle_template_param(Tokenizer* self)
 {
-    if (self->topstack->context & LC_TEMPLATE_NAME) {
+    if (self->topstack->context & LC_TEMPLATE_NAME)
         self->topstack->context ^= LC_TEMPLATE_NAME;
-    }
-    else if (self->topstack->context & LC_TEMPLATE_PARAM_VALUE) {
+    else if (self->topstack->context & LC_TEMPLATE_PARAM_VALUE)
         self->topstack->context ^= LC_TEMPLATE_PARAM_VALUE;
-    }
 
     if (self->topstack->context & LC_TEMPLATE_PARAM_KEY) {
         PyObject* stack = Tokenizer_pop_keeping_context(self);
-        if (!stack) return -1;
+        if (!stack)
+            return -1;
         if (Tokenizer_write_all(self, stack)) {
             Py_DECREF(stack);
             return -1;
         }
         Py_DECREF(stack);
     }
-    else {
+    else
         self->topstack->context |= LC_TEMPLATE_PARAM_KEY;
-    }
 
     PyObject* token = PyObject_CallObject(TemplateParamSeparator, NULL);
-    if (!token) return -1;
+    if (!token)
+        return -1;
 
     if (Tokenizer_write(self, token)) {
         Py_DECREF(token);
@@ -622,7 +611,8 @@ static int
 Tokenizer_handle_template_param_value(Tokenizer* self)
 {
     PyObject* stack = Tokenizer_pop_keeping_context(self);
-    if (!stack) return -1;
+    if (!stack)
+        return -1;
     if (Tokenizer_write_all(self, stack)) {
         Py_DECREF(stack);
         return -1;
@@ -633,8 +623,8 @@ Tokenizer_handle_template_param_value(Tokenizer* self)
     self->topstack->context |= LC_TEMPLATE_PARAM_VALUE;
 
     PyObject* token = PyObject_CallObject(TemplateParamEquals, NULL);
-    if (!token) return -1;
-
+    if (!token)
+        return -1;
     if (Tokenizer_write(self, token)) {
         Py_DECREF(token);
         return -1;
@@ -652,14 +642,14 @@ Tokenizer_handle_template_end(Tokenizer* self)
     PyObject* stack;
     if (self->topstack->context & LC_TEMPLATE_PARAM_KEY) {
         stack = Tokenizer_pop_keeping_context(self);
-        if (!stack) return NULL;
+        if (!stack)
+            return NULL;
         if (Tokenizer_write_all(self, stack)) {
             Py_DECREF(stack);
             return NULL;
         }
         Py_DECREF(stack);
     }
-
     self->head++;
     stack = Tokenizer_pop(self);
     return stack;
@@ -675,8 +665,8 @@ Tokenizer_handle_argument_separator(Tokenizer* self)
     self->topstack->context |= LC_ARGUMENT_DEFAULT;
 
     PyObject* token = PyObject_CallObject(ArgumentSeparator, NULL);
-    if (!token) return -1;
-
+    if (!token)
+        return -1;
     if (Tokenizer_write(self, token)) {
         Py_DECREF(token);
         return -1;
@@ -702,11 +692,12 @@ Tokenizer_handle_argument_end(Tokenizer* self)
 static int
 Tokenizer_parse_wikilink(Tokenizer* self)
 {
-    self->head += 2;
-    Py_ssize_t reset = self->head - 1;
+    Py_ssize_t reset;
+    PyObject *token, *wikilink;
 
-    PyObject *token;
-    PyObject *wikilink = Tokenizer_parse(self, LC_WIKILINK_TITLE);
+    self->head += 2;
+    reset = self->head - 1;
+    wikilink = Tokenizer_parse(self, LC_WIKILINK_TITLE);
 
     if (BAD_ROUTE) {
         RESET_ROUTE();
@@ -718,7 +709,8 @@ Tokenizer_parse_wikilink(Tokenizer* self)
         }
         return 0;
     }
-    if (!wikilink) return -1;
+    if (!wikilink)
+        return -1;
 
     token = PyObject_CallObject(WikilinkOpen, NULL);
     if (!token) {
@@ -740,8 +732,8 @@ Tokenizer_parse_wikilink(Tokenizer* self)
     Py_DECREF(wikilink);
 
     token = PyObject_CallObject(WikilinkClose, NULL);
-    if (!token) return -1;
-
+    if (!token)
+        return -1;
     if (Tokenizer_write(self, token)) {
         Py_DECREF(token);
         return -1;
@@ -760,8 +752,8 @@ Tokenizer_handle_wikilink_separator(Tokenizer* self)
     self->topstack->context |= LC_WIKILINK_TEXT;
 
     PyObject* token = PyObject_CallObject(WikilinkSeparator, NULL);
-    if (!token) return -1;
-
+    if (!token)
+        return -1;
     if (Tokenizer_write(self, token)) {
         Py_DECREF(token);
         return -1;
@@ -866,14 +858,13 @@ Tokenizer_parse_heading(Tokenizer* self)
     free(heading);
 
     token = PyObject_CallObject(HeadingEnd, NULL);
-    if (!token) return -1;
-
+    if (!token)
+        return -1;
     if (Tokenizer_write(self, token)) {
         Py_DECREF(token);
         return -1;
     }
     Py_DECREF(token);
-
     self->global ^= GL_HEADING;
     return 0;
 }
@@ -931,7 +922,8 @@ Tokenizer_handle_heading_end(Tokenizer* self)
     }
 
     PyObject* stack = Tokenizer_pop(self);
-    if (!stack) return NULL;
+    if (!stack)
+        return NULL;
 
     HeadingData* heading = malloc(sizeof(HeadingData));
     if (!heading) {
@@ -955,7 +947,8 @@ Tokenizer_really_parse_entity(Tokenizer* self)
     char *valid, *text, *def;
 
     token = PyObject_CallObject(HTMLEntityStart, NULL);
-    if (!token) return -1;
+    if (!token)
+        return -1;
     if (Tokenizer_write(self, token)) {
         Py_DECREF(token);
         return -1;
@@ -972,7 +965,8 @@ Tokenizer_really_parse_entity(Tokenizer* self)
     if (this == *"#") {
         numeric = 1;
         token = PyObject_CallObject(HTMLEntityNumeric, NULL);
-        if (!token) return -1;
+        if (!token)
+            return -1;
         if (Tokenizer_write(self, token)) {
             Py_DECREF(token);
             return -1;
@@ -988,11 +982,13 @@ Tokenizer_really_parse_entity(Tokenizer* self)
         if (this == *"x" || this == *"X") {
             hexadecimal = 1;
             kwargs = PyDict_New();
-            if (!kwargs) return -1;
+            if (!kwargs)
+                return -1;
             PyDict_SetItemString(kwargs, "char", Tokenizer_read(self, 0));
             PyObject* token = PyObject_Call(HTMLEntityHex, NOARGS, kwargs);
             Py_DECREF(kwargs);
-            if (!token) return -1;
+            if (!token)
+                return -1;
             if (Tokenizer_write(self, token)) {
                 Py_DECREF(token);
                 return -1;
@@ -1000,13 +996,11 @@ Tokenizer_really_parse_entity(Tokenizer* self)
             Py_DECREF(token);
             self->head++;
         }
-        else {
+        else
             hexadecimal = 0;
-        }
     }
-    else {
+    else
         numeric = hexadecimal = 0;
-    }
 
     if (hexadecimal)
         valid = "0123456789abcdefABCDEF";
@@ -1091,7 +1085,8 @@ Tokenizer_really_parse_entity(Tokenizer* self)
     Py_DECREF(textobj);
     token = PyObject_Call(Text, NOARGS, kwargs);
     Py_DECREF(kwargs);
-    if (!token) return -1;
+    if (!token)
+        return -1;
     if (Tokenizer_write(self, token)) {
         Py_DECREF(token);
         return -1;
@@ -1099,7 +1094,8 @@ Tokenizer_really_parse_entity(Tokenizer* self)
     Py_DECREF(token);
 
     token = PyObject_CallObject(HTMLEntityEnd, NULL);
-    if (!token) return -1;
+    if (!token)
+        return -1;
     if (Tokenizer_write(self, token)) {
         Py_DECREF(token);
         return -1;
@@ -1117,9 +1113,8 @@ Tokenizer_parse_entity(Tokenizer* self)
     Py_ssize_t reset = self->head;
     if (Tokenizer_push(self, 0))
         return -1;
-
     if (Tokenizer_really_parse_entity(self))
-            return -1;
+        return -1;
 
     if (BAD_ROUTE) {
         RESET_ROUTE();
@@ -1130,12 +1125,12 @@ Tokenizer_parse_entity(Tokenizer* self)
     }
 
     PyObject* tokenlist = Tokenizer_pop(self);
-    if (!tokenlist) return -1;
+    if (!tokenlist)
+        return -1;
     if (Tokenizer_write_all(self, tokenlist)) {
         Py_DECREF(tokenlist);
         return -1;
     }
-
     Py_DECREF(tokenlist);
     return 0;
 }
@@ -1158,7 +1153,8 @@ Tokenizer_parse_comment(Tokenizer* self)
         const char* text = "<!--";
         int i = 0;
         while (1) {
-            if (!text[i]) return 0;
+            if (!text[i])
+                return 0;
             if (Tokenizer_write_text(self, (Py_UNICODE) text[i])) {
                 Py_XDECREF(text);
                 return -1;
@@ -1167,7 +1163,8 @@ Tokenizer_parse_comment(Tokenizer* self)
         }
         return 0;
     }
-    if (!comment) return -1;
+    if (!comment)
+        return -1;
 
     token = PyObject_CallObject(CommentStart, NULL);
     if (!token) {
@@ -1181,7 +1178,6 @@ Tokenizer_parse_comment(Tokenizer* self)
         return -1;
     }
     Py_DECREF(token);
-
     if (Tokenizer_write_all(self, comment)) {
         Py_DECREF(comment);
         return -1;
@@ -1189,8 +1185,8 @@ Tokenizer_parse_comment(Tokenizer* self)
     Py_DECREF(comment);
 
     token = PyObject_CallObject(CommentEnd, NULL);
-    if (!token) return -1;
-
+    if (!token)
+        return -1;
     if (Tokenizer_write(self, token)) {
         Py_DECREF(token);
         return -1;
@@ -1232,12 +1228,10 @@ Tokenizer_verify_safe(Tokenizer* self, int context, Py_UNICODE data)
             }
             self->topstack->context ^= LC_FAIL_ON_RBRACE;
         }
-        else if (data == *"{") {
+        else if (data == *"{")
             self->topstack->context |= LC_FAIL_ON_LBRACE;
-        }
-        else if (data == *"}") {
+        else if (data == *"}")
             self->topstack->context |= LC_FAIL_ON_RBRACE;
-        }
     }
 
     if (context & LC_HAS_TEXT) {
@@ -1248,14 +1242,12 @@ Tokenizer_verify_safe(Tokenizer* self, int context, Py_UNICODE data)
             }
         }
         else {
-            if (data == *"\n") {
+            if (data == *"\n")
                 self->topstack->context |= LC_FAIL_ON_TEXT;
-            }
         }
     }
-    else if (!Py_UNICODE_ISSPACE(data)) {
+    else if (!Py_UNICODE_ISSPACE(data))
         self->topstack->context |= LC_HAS_TEXT;
-    }
 }
 
 /*
@@ -1301,9 +1293,8 @@ Tokenizer_parse(Tokenizer* self, int context)
                 PyObject* trash = Tokenizer_pop(self);
                 Py_XDECREF(trash);
             }
-            if (this_context & fail_contexts) {
+            if (this_context & fail_contexts)
                 return Tokenizer_fail_route(self);
-            }
             return Tokenizer_pop(self);
         }
 
@@ -1311,9 +1302,8 @@ Tokenizer_parse(Tokenizer* self, int context)
 
         if (this_context & LC_COMMENT) {
             if (this == next && next == *"-") {
-                if (Tokenizer_READ(self, 2) == *">") {
+                if (Tokenizer_READ(self, 2) == *">")
                     return Tokenizer_pop(self);
-                }
             }
             Tokenizer_write_text(self, this);
         }
@@ -1331,9 +1321,8 @@ Tokenizer_parse(Tokenizer* self, int context)
             if (Tokenizer_handle_template_param_value(self))
                 return NULL;
         }
-        else if (this == next && next == *"}" && this_context & LC_TEMPLATE) {
+        else if (this == next && next == *"}" && this_context & LC_TEMPLATE)
             return Tokenizer_handle_template_end(self);
-        }
         else if (this == *"|" && this_context & LC_ARGUMENT_NAME) {
             if (Tokenizer_handle_argument_separator(self))
                 return NULL;
@@ -1359,25 +1348,21 @@ Tokenizer_parse(Tokenizer* self, int context)
             if (Tokenizer_handle_wikilink_separator(self))
                 return NULL;
         }
-        else if (this == next && next == *"]" && this_context & LC_WIKILINK) {
+        else if (this == next && next == *"]" && this_context & LC_WIKILINK)
             return Tokenizer_handle_wikilink_end(self);
-        }
         else if (this == *"=" && !(self->global & GL_HEADING)) {
             last = *PyUnicode_AS_UNICODE(Tokenizer_read_backwards(self, 1));
             if (last == *"\n" || last == *"") {
                 if (Tokenizer_parse_heading(self))
                     return NULL;
             }
-            else {
+            else
                 Tokenizer_write_text(self, this);
-            }
         }
-        else if (this == *"=" && this_context & LC_HEADING) {
+        else if (this == *"=" && this_context & LC_HEADING)
             return (PyObject*) Tokenizer_handle_heading_end(self);
-        }
-        else if (this == *"\n" && this_context & LC_HEADING) {
+        else if (this == *"\n" && this_context & LC_HEADING)
             return Tokenizer_fail_route(self);
-        }
         else if (this == *"&") {
             if (Tokenizer_parse_entity(self))
                 return NULL;
@@ -1388,14 +1373,11 @@ Tokenizer_parse(Tokenizer* self, int context)
                 if (Tokenizer_parse_comment(self))
                     return NULL;
             }
-            else {
+            else
                 Tokenizer_write_text(self, this);
-            }
         }
-        else {
+        else
             Tokenizer_write_text(self, this);
-        }
-
         self->head++;
     }
 }
@@ -1414,9 +1396,8 @@ Tokenizer_tokenize(Tokenizer* self, PyObject* args)
         const char* encoded;
         Py_ssize_t size;
 
-        if (!PyArg_ParseTuple(args, "s#", &encoded, &size)) {
+        if (!PyArg_ParseTuple(args, "s#", &encoded, &size))
             return NULL;
-        }
 
         PyObject* temp;
         temp = PyUnicode_FromStringAndSize(encoded, size);
@@ -1434,7 +1415,6 @@ Tokenizer_tokenize(Tokenizer* self, PyObject* args)
     }
 
     self->length = PyList_GET_SIZE(self->text);
-
     return Tokenizer_parse(self, 0);
 }
 
@@ -1453,16 +1433,19 @@ init_tokenizer(void)
     PyModule_AddObject(module, "CTokenizer", (PyObject*) &TokenizerType);
 
     PyObject* htmlentitydefs = PyImport_ImportModule("htmlentitydefs");
-    if (!htmlentitydefs) return;
+    if (!htmlentitydefs)
+        return;
 
     PyObject* defmap = PyObject_GetAttrString(htmlentitydefs, "entitydefs");
-    if (!defmap) return;
+    if (!defmap)
+        return;
     Py_DECREF(htmlentitydefs);
 
     unsigned numdefs = (unsigned) PyDict_Size(defmap);
     entitydefs = calloc(numdefs + 1, sizeof(char*));
     PyObject* deflist = PyDict_Keys(defmap);
-    if (!deflist) return;
+    if (!deflist)
+        return;
     Py_DECREF(defmap);
 
     unsigned i;
@@ -1478,7 +1461,8 @@ init_tokenizer(void)
     PyObject* globals = PyEval_GetGlobals();
     PyObject* locals = PyEval_GetLocals();
     PyObject* fromlist = PyList_New(1);
-    if (!fromlist) return;
+    if (!fromlist)
+        return;
     PyObject* submodname = PyBytes_FromString("tokens");
     if (!submodname) {
         Py_DECREF(fromlist);
@@ -1488,7 +1472,8 @@ init_tokenizer(void)
 
     PyObject* tokmodule = PyImport_ImportModuleLevel(name, globals, locals, fromlist, 0);
     Py_DECREF(fromlist);
-    if (!tokmodule) return;
+    if (!tokmodule)
+        return;
 
     tokens = PyObject_GetAttrString(tokmodule, "tokens");
     Py_DECREF(tokmodule);

From 3a6335d96915a555399577b191e891a4c4a987f3 Mon Sep 17 00:00:00 2001
From: Ben Kurtovic <ben.kurtovic@verizon.net>
Date: Sat, 17 Nov 2012 17:18:00 -0500
Subject: [PATCH 43/47] Clean up init_tokenizer().

---
 mwparserfromhell/parser/tokenizer.c | 50 ++++++++++++++++---------------------
 1 file changed, 22 insertions(+), 28 deletions(-)

diff --git a/mwparserfromhell/parser/tokenizer.c b/mwparserfromhell/parser/tokenizer.c
index 40f91a1..1efa65d 100644
--- a/mwparserfromhell/parser/tokenizer.c
+++ b/mwparserfromhell/parser/tokenizer.c
@@ -1421,7 +1421,9 @@ Tokenizer_tokenize(Tokenizer* self, PyObject* args)
 PyMODINIT_FUNC
 init_tokenizer(void)
 {
-    PyObject* module;
+    PyObject *module, *tempmodule, *defmap, *deflist, *globals, *locals, *fromlist, *modname;
+    unsigned numdefs, i;
+    char* name;
 
     TokenizerType.tp_new = PyType_GenericNew;
     if (PyType_Ready(&TokenizerType) < 0)
@@ -1432,51 +1434,43 @@ init_tokenizer(void)
     Py_INCREF(&TokenizerType);
     PyModule_AddObject(module, "CTokenizer", (PyObject*) &TokenizerType);
 
-    PyObject* htmlentitydefs = PyImport_ImportModule("htmlentitydefs");
-    if (!htmlentitydefs)
+    tempmodule = PyImport_ImportModule("htmlentitydefs");
+    if (!tempmodule)
         return;
-
-    PyObject* defmap = PyObject_GetAttrString(htmlentitydefs, "entitydefs");
+    defmap = PyObject_GetAttrString(tempmodule, "entitydefs");
     if (!defmap)
         return;
-    Py_DECREF(htmlentitydefs);
-
-    unsigned numdefs = (unsigned) PyDict_Size(defmap);
-    entitydefs = calloc(numdefs + 1, sizeof(char*));
-    PyObject* deflist = PyDict_Keys(defmap);
+    Py_DECREF(tempmodule);
+    deflist = PyDict_Keys(defmap);
     if (!deflist)
         return;
     Py_DECREF(defmap);
 
-    unsigned i;
-    for (i = 0; i < numdefs; i++) {
+    numdefs = (unsigned) PyList_GET_SIZE(defmap);
+    entitydefs = calloc(numdefs + 1, sizeof(char*));
+    for (i = 0; i < numdefs; i++)
         entitydefs[i] = PyString_AsString(PyList_GET_ITEM(deflist, i));
-    }
     Py_DECREF(deflist);
 
     EMPTY = PyUnicode_FromString("");
     NOARGS = PyTuple_New(0);
 
-    char* name = "mwparserfromhell.parser";
-    PyObject* globals = PyEval_GetGlobals();
-    PyObject* locals = PyEval_GetLocals();
-    PyObject* fromlist = PyList_New(1);
+    name = "mwparserfromhell.parser";
+    globals = PyEval_GetGlobals();
+    locals = PyEval_GetLocals();
+    fromlist = PyList_New(1);
     if (!fromlist)
         return;
-    PyObject* submodname = PyBytes_FromString("tokens");
-    if (!submodname) {
-        Py_DECREF(fromlist);
+    modname = PyBytes_FromString("tokens");
+    if (!modname)
         return;
-    }
-    PyList_SET_ITEM(fromlist, 0, submodname);
-
-    PyObject* tokmodule = PyImport_ImportModuleLevel(name, globals, locals, fromlist, 0);
+    PyList_SET_ITEM(fromlist, 0, modname);
+    tempmodule = PyImport_ImportModuleLevel(name, globals, locals, fromlist, 0);
     Py_DECREF(fromlist);
-    if (!tokmodule)
+    if (!tempmodule)
         return;
-
-    tokens = PyObject_GetAttrString(tokmodule, "tokens");
-    Py_DECREF(tokmodule);
+    tokens = PyObject_GetAttrString(tempmodule, "tokens");
+    Py_DECREF(tempmodule);
 
     Text = PyObject_GetAttrString(tokens, "Text");
 

From 88a2b950306cabac98e7354979f7eb8da74f41ab Mon Sep 17 00:00:00 2001
From: Ben Kurtovic <ben.kurtovic@verizon.net>
Date: Sat, 17 Nov 2012 17:54:10 -0500
Subject: [PATCH 44/47] Remove a bunch of white space.

---
 mwparserfromhell/parser/tokenizer.c | 205 ++++++++++++------------------------
 1 file changed, 65 insertions(+), 140 deletions(-)

diff --git a/mwparserfromhell/parser/tokenizer.c b/mwparserfromhell/parser/tokenizer.c
index 1efa65d..733bd61 100644
--- a/mwparserfromhell/parser/tokenizer.c
+++ b/mwparserfromhell/parser/tokenizer.c
@@ -82,7 +82,6 @@ Tokenizer_init(Tokenizer* self, PyObject* args, PyObject* kwds)
     static char* kwlist[] = {NULL};
     if (!PyArg_ParseTupleAndKeywords(args, kwds, "", kwlist))
         return -1;
-
     self->text = Py_None;
     Py_INCREF(Py_None);
     self->topstack = NULL;
@@ -144,7 +143,6 @@ Tokenizer_push_textbuffer(Tokenizer* self)
     PyObject* text = Textbuffer_render(buffer);
     if (!text)
         return -1;
-
     PyObject* kwargs = PyDict_New();
     if (!kwargs) {
         Py_DECREF(text);
@@ -152,18 +150,15 @@ Tokenizer_push_textbuffer(Tokenizer* self)
     }
     PyDict_SetItemString(kwargs, "text", text);
     Py_DECREF(text);
-
     PyObject* token = PyObject_Call(Text, NOARGS, kwargs);
     Py_DECREF(kwargs);
     if (!token)
         return -1;
-
     if (PyList_Append(self->topstack->stack, token)) {
         Py_DECREF(token);
         return -1;
     }
     Py_DECREF(token);
-
     Textbuffer_dealloc(buffer);
     self->topstack->textbuffer = Textbuffer_new();
     if (!self->topstack->textbuffer)
@@ -189,10 +184,8 @@ Tokenizer_pop(Tokenizer* self)
 {
     if (Tokenizer_push_textbuffer(self))
         return NULL;
-
     PyObject* stack = self->topstack->stack;
     Py_INCREF(stack);
-
     Tokenizer_delete_top_of_stack(self);
     return stack;
 }
@@ -206,11 +199,9 @@ Tokenizer_pop_keeping_context(Tokenizer* self)
 {
     if (Tokenizer_push_textbuffer(self))
         return NULL;
-
     PyObject* stack = self->topstack->stack;
     Py_INCREF(stack);
     int context = self->topstack->context;
-
     Tokenizer_delete_top_of_stack(self);
     self->topstack->context = context;
     return stack;
@@ -282,35 +273,36 @@ static int
 Tokenizer_write_all(Tokenizer* self, PyObject* tokenlist)
 {
     int pushed = 0;
+    PyObject *stack, *token, *left, *right, *text;
+    struct Textbuffer* buffer;
+    Py_ssize_t size;
+
     if (PyList_GET_SIZE(tokenlist) > 0) {
-        PyObject* token = PyList_GET_ITEM(tokenlist, 0);
+        token = PyList_GET_ITEM(tokenlist, 0);
         switch (PyObject_IsInstance(token, Text)) {
             case 0:
                 break;
             case 1: {
                 pushed = 1;
-                struct Textbuffer* buffer = self->topstack->textbuffer;
+                buffer = self->topstack->textbuffer;
                 if (buffer->size == 0 && !buffer->next)
                     break;
-                PyObject* left = Textbuffer_render(buffer);
+                left = Textbuffer_render(buffer);
                 if (!left)
                     return -1;
-                PyObject* right = PyObject_GetAttrString(token, "text");
+                right = PyObject_GetAttrString(token, "text");
                 if (!right)
                     return -1;
-
-                PyObject* text = PyUnicode_Concat(left, right);
+                text = PyUnicode_Concat(left, right);
                 Py_DECREF(left);
                 Py_DECREF(right);
                 if (!text)
                     return -1;
-
                 if (PyObject_SetAttrString(token, "text", text)) {
                     Py_DECREF(text);
                     return -1;
                 }
                 Py_DECREF(text);
-
                 Textbuffer_dealloc(buffer);
                 self->topstack->textbuffer = Textbuffer_new();
                 if (!self->topstack->textbuffer)
@@ -325,9 +317,8 @@ Tokenizer_write_all(Tokenizer* self, PyObject* tokenlist)
         if (Tokenizer_push_textbuffer(self))
             return -1;
     }
-
-    PyObject* stack = self->topstack->stack;
-    Py_ssize_t size = PyList_GET_SIZE(stack);
+    stack = self->topstack->stack;
+    size = PyList_GET_SIZE(stack);
     if (PyList_SetSlice(stack, size, size, tokenlist))
         return -1;
     return 0;
@@ -351,7 +342,6 @@ Tokenizer_write_text_then_stack(Tokenizer* self, const char* text)
         }
         i++;
     }
-
     if (stack) {
         if (PyList_GET_SIZE(stack) > 0) {
             if (Tokenizer_write_all(self, stack)) {
@@ -361,7 +351,6 @@ Tokenizer_write_text_then_stack(Tokenizer* self, const char* text)
         }
         Py_DECREF(stack);
     }
-
     self->head--;
     return 0;
 }
@@ -396,23 +385,22 @@ Tokenizer_read_backwards(Tokenizer* self, Py_ssize_t delta)
 static int
 Tokenizer_parse_template_or_argument(Tokenizer* self)
 {
-    self->head += 2;
     unsigned int braces = 2, i;
+    PyObject *tokenlist;
 
+    self->head += 2;
     while (Tokenizer_READ(self, 0) == *"{") {
         self->head++;
         braces++;
     }
     if (Tokenizer_push(self, 0))
         return -1;
-
     while (braces) {
         if (braces == 1) {
             if (Tokenizer_write_text_then_stack(self, "{"))
                 return -1;
             return 0;
         }
-
         if (braces == 2) {
             if (Tokenizer_parse_template(self))
                 return -1;
@@ -425,15 +413,12 @@ Tokenizer_parse_template_or_argument(Tokenizer* self)
             }
             break;
         }
-
         if (Tokenizer_parse_argument(self))
             return -1;
-
         if (BAD_ROUTE) {
             RESET_ROUTE();
             if (Tokenizer_parse_template(self))
                 return -1;
-
             if (BAD_ROUTE) {
                 RESET_ROUTE();
                 char text[braces + 1];
@@ -443,7 +428,6 @@ Tokenizer_parse_template_or_argument(Tokenizer* self)
                     Py_XDECREF(text);
                     return -1;
                 }
-
                 Py_XDECREF(text);
                 return 0;
             }
@@ -452,19 +436,16 @@ Tokenizer_parse_template_or_argument(Tokenizer* self)
         }
         else
             braces -= 3;
-
         if (braces)
             self->head++;
     }
-
-    PyObject* tokenlist = Tokenizer_pop(self);
+    tokenlist = Tokenizer_pop(self);
     if (!tokenlist)
         return -1;
     if (Tokenizer_write_all(self, tokenlist)) {
         Py_DECREF(tokenlist);
         return -1;
     }
-
     Py_DECREF(tokenlist);
     return 0;
 }
@@ -485,30 +466,25 @@ Tokenizer_parse_template(Tokenizer* self)
     }
     if (!template)
         return -1;
-
     token = PyObject_CallObject(TemplateOpen, NULL);
     if (!token) {
         Py_DECREF(template);
         return -1;
     }
-
     if (Tokenizer_write_first(self, token)) {
         Py_DECREF(token);
         Py_DECREF(template);
         return -1;
     }
     Py_DECREF(token);
-
     if (Tokenizer_write_all(self, template)) {
         Py_DECREF(template);
         return -1;
     }
     Py_DECREF(template);
-
     token = PyObject_CallObject(TemplateClose, NULL);
     if (!token)
         return -1;
-
     if (Tokenizer_write(self, token)) {
         Py_DECREF(token);
         return -1;
@@ -533,30 +509,25 @@ Tokenizer_parse_argument(Tokenizer* self)
     }
     if (!argument)
         return -1;
-
     token = PyObject_CallObject(ArgumentOpen, NULL);
     if (!token) {
         Py_DECREF(argument);
         return -1;
     }
-
     if (Tokenizer_write_first(self, token)) {
         Py_DECREF(token);
         Py_DECREF(argument);
         return -1;
     }
     Py_DECREF(token);
-
     if (Tokenizer_write_all(self, argument)) {
         Py_DECREF(argument);
         return -1;
     }
     Py_DECREF(argument);
-
     token = PyObject_CallObject(ArgumentClose, NULL);
     if (!token)
         return -1;
-
     if (Tokenizer_write(self, token)) {
         Py_DECREF(token);
         return -1;
@@ -571,13 +542,14 @@ Tokenizer_parse_argument(Tokenizer* self)
 static int
 Tokenizer_handle_template_param(Tokenizer* self)
 {
+    PyObject *stack, *token;
+
     if (self->topstack->context & LC_TEMPLATE_NAME)
         self->topstack->context ^= LC_TEMPLATE_NAME;
     else if (self->topstack->context & LC_TEMPLATE_PARAM_VALUE)
         self->topstack->context ^= LC_TEMPLATE_PARAM_VALUE;
-
     if (self->topstack->context & LC_TEMPLATE_PARAM_KEY) {
-        PyObject* stack = Tokenizer_pop_keeping_context(self);
+        stack = Tokenizer_pop_keeping_context(self);
         if (!stack)
             return -1;
         if (Tokenizer_write_all(self, stack)) {
@@ -589,16 +561,14 @@ Tokenizer_handle_template_param(Tokenizer* self)
     else
         self->topstack->context |= LC_TEMPLATE_PARAM_KEY;
 
-    PyObject* token = PyObject_CallObject(TemplateParamSeparator, NULL);
+    token = PyObject_CallObject(TemplateParamSeparator, NULL);
     if (!token)
         return -1;
-
     if (Tokenizer_write(self, token)) {
         Py_DECREF(token);
         return -1;
     }
     Py_DECREF(token);
-
     if (Tokenizer_push(self, self->topstack->context))
         return -1;
     return 0;
@@ -610,7 +580,9 @@ Tokenizer_handle_template_param(Tokenizer* self)
 static int
 Tokenizer_handle_template_param_value(Tokenizer* self)
 {
-    PyObject* stack = Tokenizer_pop_keeping_context(self);
+    PyObject *stack, *token;
+
+    stack = Tokenizer_pop_keeping_context(self);
     if (!stack)
         return -1;
     if (Tokenizer_write_all(self, stack)) {
@@ -618,11 +590,9 @@ Tokenizer_handle_template_param_value(Tokenizer* self)
         return -1;
     }
     Py_DECREF(stack);
-
     self->topstack->context ^= LC_TEMPLATE_PARAM_KEY;
     self->topstack->context |= LC_TEMPLATE_PARAM_VALUE;
-
-    PyObject* token = PyObject_CallObject(TemplateParamEquals, NULL);
+    token = PyObject_CallObject(TemplateParamEquals, NULL);
     if (!token)
         return -1;
     if (Tokenizer_write(self, token)) {
@@ -640,6 +610,7 @@ static PyObject*
 Tokenizer_handle_template_end(Tokenizer* self)
 {
     PyObject* stack;
+
     if (self->topstack->context & LC_TEMPLATE_PARAM_KEY) {
         stack = Tokenizer_pop_keeping_context(self);
         if (!stack)
@@ -663,7 +634,6 @@ Tokenizer_handle_argument_separator(Tokenizer* self)
 {
     self->topstack->context ^= LC_ARGUMENT_NAME;
     self->topstack->context |= LC_ARGUMENT_DEFAULT;
-
     PyObject* token = PyObject_CallObject(ArgumentSeparator, NULL);
     if (!token)
         return -1;
@@ -693,16 +663,15 @@ static int
 Tokenizer_parse_wikilink(Tokenizer* self)
 {
     Py_ssize_t reset;
-    PyObject *token, *wikilink;
+    PyObject *wikilink, *token;
+    int i;
 
     self->head += 2;
     reset = self->head - 1;
     wikilink = Tokenizer_parse(self, LC_WIKILINK_TITLE);
-
     if (BAD_ROUTE) {
         RESET_ROUTE();
         self->head = reset;
-        int i;
         for (i = 0; i < 2; i++) {
             if (Tokenizer_write_text(self, *"["))
                 return -1;
@@ -711,26 +680,22 @@ Tokenizer_parse_wikilink(Tokenizer* self)
     }
     if (!wikilink)
         return -1;
-
     token = PyObject_CallObject(WikilinkOpen, NULL);
     if (!token) {
         Py_DECREF(wikilink);
         return -1;
     }
-
     if (Tokenizer_write(self, token)) {
         Py_DECREF(token);
         Py_DECREF(wikilink);
         return -1;
     }
     Py_DECREF(token);
-
     if (Tokenizer_write_all(self, wikilink)) {
         Py_DECREF(wikilink);
         return -1;
     }
     Py_DECREF(wikilink);
-
     token = PyObject_CallObject(WikilinkClose, NULL);
     if (!token)
         return -1;
@@ -750,7 +715,6 @@ Tokenizer_handle_wikilink_separator(Tokenizer* self)
 {
     self->topstack->context ^= LC_WIKILINK_TITLE;
     self->topstack->context |= LC_WIKILINK_TEXT;
-
     PyObject* token = PyObject_CallObject(WikilinkSeparator, NULL);
     if (!token)
         return -1;
@@ -779,19 +743,19 @@ Tokenizer_handle_wikilink_end(Tokenizer* self)
 static int
 Tokenizer_parse_heading(Tokenizer* self)
 {
-    self->global |= GL_HEADING;
     Py_ssize_t reset = self->head;
-    self->head += 1;
-    int best = 1, i;
+    int best = 1, i, context, diff;
+    HeadingData *heading;
+    PyObject *level, *kwargs, *token;
 
+    self->global |= GL_HEADING;
+    self->head += 1;
     while (Tokenizer_READ(self, 0) == *"=") {
         best++;
         self->head++;
     }
-
-    int context = LC_HEADING_LEVEL_1 << (best > 5 ? 5 : best - 1);
-    HeadingData* heading = (HeadingData*) Tokenizer_parse(self, context);
-
+    context = LC_HEADING_LEVEL_1 << (best > 5 ? 5 : best - 1);
+    heading = (HeadingData*) Tokenizer_parse(self, context);
     if (BAD_ROUTE) {
         RESET_ROUTE();
         self->head = reset + best - 1;
@@ -804,14 +768,13 @@ Tokenizer_parse_heading(Tokenizer* self)
         return 0;
     }
 
-    PyObject* level = PyInt_FromSsize_t(heading->level);
+    level = PyInt_FromSsize_t(heading->level);
     if (!level) {
         Py_DECREF(heading->title);
         free(heading);
         return -1;
     }
-
-    PyObject* kwargs = PyDict_New();
+    kwargs = PyDict_New();
     if (!kwargs) {
         Py_DECREF(level);
         Py_DECREF(heading->title);
@@ -820,15 +783,13 @@ Tokenizer_parse_heading(Tokenizer* self)
     }
     PyDict_SetItemString(kwargs, "level", level);
     Py_DECREF(level);
-
-    PyObject* token = PyObject_Call(HeadingStart, NOARGS, kwargs);
+    token = PyObject_Call(HeadingStart, NOARGS, kwargs);
     Py_DECREF(kwargs);
     if (!token) {
         Py_DECREF(heading->title);
         free(heading);
         return -1;
     }
-
     if (Tokenizer_write(self, token)) {
         Py_DECREF(token);
         Py_DECREF(heading->title);
@@ -836,9 +797,8 @@ Tokenizer_parse_heading(Tokenizer* self)
         return -1;
     }
     Py_DECREF(token);
-
     if (heading->level < best) {
-        int diff = best - heading->level;
+        diff = best - heading->level;
         char difftext[diff + 1];
         for (i = 0; i < diff; i++) difftext[i] = *"=";
         difftext[diff] = *"";
@@ -848,7 +808,6 @@ Tokenizer_parse_heading(Tokenizer* self)
             return -1;
         }
     }
-
     if (Tokenizer_write_all(self, heading->title)) {
         Py_DECREF(heading->title);
         free(heading);
@@ -856,7 +815,6 @@ Tokenizer_parse_heading(Tokenizer* self)
     }
     Py_DECREF(heading->title);
     free(heading);
-
     token = PyObject_CallObject(HeadingEnd, NULL);
     if (!token)
         return -1;
@@ -875,24 +833,24 @@ Tokenizer_parse_heading(Tokenizer* self)
 static HeadingData*
 Tokenizer_handle_heading_end(Tokenizer* self)
 {
-    Py_ssize_t reset = self->head;
-    self->head += 1;
-    Py_ssize_t best = 1;
-    int i;
+    Py_ssize_t reset = self->head, best;
+    int i, current, level, diff;
+    HeadingData *after, *heading;
+    PyObject *stack;
 
+    self->head += 1;
+    best = 1;
     while (Tokenizer_READ(self, 0) == *"=") {
         best++;
         self->head++;
     }
-
-    int current = log2(self->topstack->context / LC_HEADING_LEVEL_1) + 1;
-    int level = current > best ? (best > 6 ? 6 : best) : (current > 6 ? 6 : current);
-    HeadingData* after = (HeadingData*) Tokenizer_parse(self, self->topstack->context);
-
+    current = log2(self->topstack->context / LC_HEADING_LEVEL_1) + 1;
+    level = current > best ? (best > 6 ? 6 : best) : (current > 6 ? 6 : current);
+    after = (HeadingData*) Tokenizer_parse(self, self->topstack->context);
     if (BAD_ROUTE) {
         RESET_ROUTE();
         if (level < best) {
-            int diff = best - level;
+            diff = best - level;
             char difftext[diff + 1];
             for (i = 0; i < diff; i++) difftext[i] = *"=";
             difftext[diff] = *"";
@@ -901,7 +859,6 @@ Tokenizer_handle_heading_end(Tokenizer* self)
         }
         self->head = reset + best - 1;
     }
-
     else {
         char text[best + 1];
         for (i = 0; i < best; i++) text[i] = *"=";
@@ -920,12 +877,10 @@ Tokenizer_handle_heading_end(Tokenizer* self)
         level = after->level;
         free(after);
     }
-
-    PyObject* stack = Tokenizer_pop(self);
+    stack = Tokenizer_pop(self);
     if (!stack)
         return NULL;
-
-    HeadingData* heading = malloc(sizeof(HeadingData));
+    heading = malloc(sizeof(HeadingData));
     if (!heading) {
         PyErr_NoMemory();
         return NULL;
@@ -946,6 +901,12 @@ Tokenizer_really_parse_entity(Tokenizer* self)
     int numeric, hexadecimal, i, j, test;
     char *valid, *text, *def;
 
+    #define FAIL_ROUTE_AND_EXIT() { \
+        Tokenizer_fail_route(self); \
+        free(text);                 \
+        return 0;                   \
+    }
+
     token = PyObject_CallObject(HTMLEntityStart, NULL);
     if (!token)
         return -1;
@@ -954,9 +915,7 @@ Tokenizer_really_parse_entity(Tokenizer* self)
         return -1;
     }
     Py_DECREF(token);
-
     self->head++;
-
     this = Tokenizer_READ(self, 0);
     if (this == *"") {
         Tokenizer_fail_route(self);
@@ -972,7 +931,6 @@ Tokenizer_really_parse_entity(Tokenizer* self)
             return -1;
         }
         Py_DECREF(token);
-
         self->head++;
         this = Tokenizer_READ(self, 0);
         if (this == *"") {
@@ -985,7 +943,7 @@ Tokenizer_really_parse_entity(Tokenizer* self)
             if (!kwargs)
                 return -1;
             PyDict_SetItemString(kwargs, "char", Tokenizer_read(self, 0));
-            PyObject* token = PyObject_Call(HTMLEntityHex, NOARGS, kwargs);
+            token = PyObject_Call(HTMLEntityHex, NOARGS, kwargs);
             Py_DECREF(kwargs);
             if (!token)
                 return -1;
@@ -1001,26 +959,17 @@ Tokenizer_really_parse_entity(Tokenizer* self)
     }
     else
         numeric = hexadecimal = 0;
-
     if (hexadecimal)
         valid = "0123456789abcdefABCDEF";
     else if (numeric)
         valid = "0123456789";
     else
         valid = "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ";
-
     text = malloc(MAX_ENTITY_SIZE * sizeof(char));
     if (!text) {
         PyErr_NoMemory();
         return -1;
     }
-
-    #define FAIL_ROUTE_AND_EXIT() { \
-        Tokenizer_fail_route(self); \
-        free(text);                 \
-        return 0;                   \
-    }
-
     i = 0;
     while (1) {
         this = Tokenizer_READ(self, 0);
@@ -1051,7 +1000,6 @@ Tokenizer_really_parse_entity(Tokenizer* self)
         self->head++;
         i++;
     }
-
     if (numeric) {
         sscanf(text, (hexadecimal ? "%x" : "%d"), &test);
         if (test < 1 || test > 0x10FFFF)
@@ -1068,14 +1016,12 @@ Tokenizer_really_parse_entity(Tokenizer* self)
             i++;
         }
     }
-
     textobj = PyUnicode_FromString(text);
     if (!textobj) {
         free(text);
         return -1;
     }
     free(text);
-
     kwargs = PyDict_New();
     if (!kwargs) {
         Py_DECREF(textobj);
@@ -1092,7 +1038,6 @@ Tokenizer_really_parse_entity(Tokenizer* self)
         return -1;
     }
     Py_DECREF(token);
-
     token = PyObject_CallObject(HTMLEntityEnd, NULL);
     if (!token)
         return -1;
@@ -1111,11 +1056,12 @@ static int
 Tokenizer_parse_entity(Tokenizer* self)
 {
     Py_ssize_t reset = self->head;
+    PyObject *tokenlist;
+
     if (Tokenizer_push(self, 0))
         return -1;
     if (Tokenizer_really_parse_entity(self))
         return -1;
-
     if (BAD_ROUTE) {
         RESET_ROUTE();
         self->head = reset;
@@ -1123,8 +1069,7 @@ Tokenizer_parse_entity(Tokenizer* self)
             return -1;
         return 0;
     }
-
-    PyObject* tokenlist = Tokenizer_pop(self);
+    tokenlist = Tokenizer_pop(self);
     if (!tokenlist)
         return -1;
     if (Tokenizer_write_all(self, tokenlist)) {
@@ -1141,17 +1086,17 @@ Tokenizer_parse_entity(Tokenizer* self)
 static int
 Tokenizer_parse_comment(Tokenizer* self)
 {
-    self->head += 4;
-    Py_ssize_t reset = self->head - 1;
-
-    PyObject *token;
-    PyObject *comment = Tokenizer_parse(self, LC_WIKILINK_TITLE);
+    Py_ssize_t reset = self->head + 3;
+    PyObject *token, *comment;
+    int i;
 
+    self->head += 4;
+    comment = Tokenizer_parse(self, LC_WIKILINK_TITLE);
     if (BAD_ROUTE) {
         RESET_ROUTE();
         self->head = reset;
         const char* text = "<!--";
-        int i = 0;
+        i = 0;
         while (1) {
             if (!text[i])
                 return 0;
@@ -1165,13 +1110,11 @@ Tokenizer_parse_comment(Tokenizer* self)
     }
     if (!comment)
         return -1;
-
     token = PyObject_CallObject(CommentStart, NULL);
     if (!token) {
         Py_DECREF(comment);
         return -1;
     }
-
     if (Tokenizer_write(self, token)) {
         Py_DECREF(token);
         Py_DECREF(comment);
@@ -1183,7 +1126,6 @@ Tokenizer_parse_comment(Tokenizer* self)
         return -1;
     }
     Py_DECREF(comment);
-
     token = PyObject_CallObject(CommentEnd, NULL);
     if (!token)
         return -1;
@@ -1206,7 +1148,6 @@ Tokenizer_verify_safe(Tokenizer* self, int context, Py_UNICODE data)
         Tokenizer_fail_route(self);
         return;
     }
-
     if (context & (LC_TEMPLATE_NAME | LC_WIKILINK_TITLE)) {
         if (data == *"{" || data == *"}" || data == *"[" || data == *"]") {
             self->topstack->context |= LC_FAIL_NEXT;
@@ -1233,7 +1174,6 @@ Tokenizer_verify_safe(Tokenizer* self, int context, Py_UNICODE data)
         else if (data == *"}")
             self->topstack->context |= LC_FAIL_ON_RBRACE;
     }
-
     if (context & LC_HAS_TEXT) {
         if (context & LC_FAIL_ON_TEXT) {
             if (!Py_UNICODE_ISSPACE(data)) {
@@ -1263,17 +1203,14 @@ Tokenizer_parse(Tokenizer* self, int context)
 
     if (Tokenizer_push(self, context))
         return NULL;
-
     while (1) {
         this = Tokenizer_READ(self, 0);
         this_context = self->topstack->context;
-
         if (this_context & unsafe_contexts) {
             Tokenizer_verify_safe(self, this_context, this);
             if (BAD_ROUTE)
                 return NULL;
         }
-
         is_marker = 0;
         for (i = 0; i < NUM_MARKERS; i++) {
             if (*MARKERS[i] == this) {
@@ -1281,13 +1218,11 @@ Tokenizer_parse(Tokenizer* self, int context)
                 break;
             }
         }
-
         if (!is_marker) {
             Tokenizer_write_text(self, this);
             self->head++;
             continue;
         }
-
         if (this == *"") {
             if (this_context & LC_TEMPLATE_PARAM_KEY) {
                 PyObject* trash = Tokenizer_pop(self);
@@ -1297,9 +1232,7 @@ Tokenizer_parse(Tokenizer* self, int context)
                 return Tokenizer_fail_route(self);
             return Tokenizer_pop(self);
         }
-
         next = Tokenizer_READ(self, 1);
-
         if (this_context & LC_COMMENT) {
             if (this == next && next == *"-") {
                 if (Tokenizer_READ(self, 2) == *">")
@@ -1388,22 +1321,18 @@ Tokenizer_parse(Tokenizer* self, int context)
 static PyObject*
 Tokenizer_tokenize(Tokenizer* self, PyObject* args)
 {
-    PyObject* text;
+    PyObject *text, *temp;
 
     if (!PyArg_ParseTuple(args, "U", &text)) {
         /* Failed to parse a Unicode object; try a string instead. */
         PyErr_Clear();
         const char* encoded;
         Py_ssize_t size;
-
         if (!PyArg_ParseTuple(args, "s#", &encoded, &size))
             return NULL;
-
-        PyObject* temp;
         temp = PyUnicode_FromStringAndSize(encoded, size);
         if (!text)
             return NULL;
-
         Py_XDECREF(self->text);
         text = PySequence_Fast(temp, "expected a sequence");
         Py_XDECREF(temp);
@@ -1413,7 +1342,6 @@ Tokenizer_tokenize(Tokenizer* self, PyObject* args)
         Py_XDECREF(self->text);
         self->text = PySequence_Fast(text, "expected a sequence");
     }
-
     self->length = PyList_GET_SIZE(self->text);
     return Tokenizer_parse(self, 0);
 }
@@ -1428,9 +1356,7 @@ init_tokenizer(void)
     TokenizerType.tp_new = PyType_GenericNew;
     if (PyType_Ready(&TokenizerType) < 0)
         return;
-
     module = Py_InitModule("_tokenizer", module_methods);
-
     Py_INCREF(&TokenizerType);
     PyModule_AddObject(module, "CTokenizer", (PyObject*) &TokenizerType);
 
@@ -1445,7 +1371,6 @@ init_tokenizer(void)
     if (!deflist)
         return;
     Py_DECREF(defmap);
-
     numdefs = (unsigned) PyList_GET_SIZE(defmap);
     entitydefs = calloc(numdefs + 1, sizeof(char*));
     for (i = 0; i < numdefs; i++)

From 1abdb478c911b7f9acf14fff47e467560425406f Mon Sep 17 00:00:00 2001
From: Ben Kurtovic <ben.kurtovic@verizon.net>
Date: Sat, 17 Nov 2012 20:34:41 -0500
Subject: [PATCH 45/47] Fix a couple bugs.

---
 mwparserfromhell/parser/tokenizer.c | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/mwparserfromhell/parser/tokenizer.c b/mwparserfromhell/parser/tokenizer.c
index 733bd61..0016515 100644
--- a/mwparserfromhell/parser/tokenizer.c
+++ b/mwparserfromhell/parser/tokenizer.c
@@ -965,7 +965,7 @@ Tokenizer_really_parse_entity(Tokenizer* self)
         valid = "0123456789";
     else
         valid = "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ";
-    text = malloc(MAX_ENTITY_SIZE * sizeof(char));
+    text = calloc(MAX_ENTITY_SIZE, sizeof(char));
     if (!text) {
         PyErr_NoMemory();
         return -1;
@@ -1091,7 +1091,7 @@ Tokenizer_parse_comment(Tokenizer* self)
     int i;
 
     self->head += 4;
-    comment = Tokenizer_parse(self, LC_WIKILINK_TITLE);
+    comment = Tokenizer_parse(self, LC_COMMENT);
     if (BAD_ROUTE) {
         RESET_ROUTE();
         self->head = reset;
@@ -1351,7 +1351,7 @@ init_tokenizer(void)
 {
     PyObject *module, *tempmodule, *defmap, *deflist, *globals, *locals, *fromlist, *modname;
     unsigned numdefs, i;
-    char* name;
+    char *name;
 
     TokenizerType.tp_new = PyType_GenericNew;
     if (PyType_Ready(&TokenizerType) < 0)

From 4a725b7ac5ec983a2efcd8bb3c3786beab175b61 Mon Sep 17 00:00:00 2001
From: Ben Kurtovic <ben.kurtovic@verizon.net>
Date: Sat, 17 Nov 2012 21:31:52 -0500
Subject: [PATCH 46/47] Fix another couple bugs regarding template contexts and
 verify_safe().

---
 mwparserfromhell/parser/tokenizer.c | 18 +++++++++++++++---
 mwparserfromhell/parser/tokenizer.h |  1 +
 2 files changed, 16 insertions(+), 3 deletions(-)

diff --git a/mwparserfromhell/parser/tokenizer.c b/mwparserfromhell/parser/tokenizer.c
index 0016515..57c6a62 100644
--- a/mwparserfromhell/parser/tokenizer.c
+++ b/mwparserfromhell/parser/tokenizer.c
@@ -1153,18 +1153,30 @@ Tokenizer_verify_safe(Tokenizer* self, int context, Py_UNICODE data)
             self->topstack->context |= LC_FAIL_NEXT;
             return;
         }
+        if (data == *"|") {
+            if (context & LC_FAIL_ON_TEXT) {
+                self->topstack->context ^= LC_FAIL_ON_TEXT;
+                return;
+            }
+        }
     }
     else if (context & (LC_TEMPLATE_PARAM_KEY | LC_ARGUMENT_NAME)) {
-        if (context & LC_FAIL_ON_LBRACE) {
-            if (data == *"{") {
+        if (context & LC_FAIL_ON_EQUALS) {
+            if (data == *"=") {
                 self->topstack->context |= LC_FAIL_NEXT;
                 return;
             }
+        }
+        else if (context & LC_FAIL_ON_LBRACE) {
+            if (data == *"{") {
+                self->topstack->context |= (context & LC_TEMPLATE) ? LC_FAIL_ON_EQUALS : LC_FAIL_NEXT;
+                return;
+            }
             self->topstack->context ^= LC_FAIL_ON_LBRACE;
         }
         else if (context & LC_FAIL_ON_RBRACE) {
             if (data == *"}") {
-                self->topstack->context |= LC_FAIL_NEXT;
+                self->topstack->context |= (context & LC_TEMPLATE) ? LC_FAIL_ON_EQUALS : LC_FAIL_NEXT;
                 return;
             }
             self->topstack->context ^= LC_FAIL_ON_RBRACE;
diff --git a/mwparserfromhell/parser/tokenizer.h b/mwparserfromhell/parser/tokenizer.h
index 67c39cd..2484d4f 100644
--- a/mwparserfromhell/parser/tokenizer.h
+++ b/mwparserfromhell/parser/tokenizer.h
@@ -119,6 +119,7 @@ static PyObject* TagCloseClose;
 #define LC_FAIL_NEXT            0x10000
 #define LC_FAIL_ON_LBRACE       0x20000
 #define LC_FAIL_ON_RBRACE       0x40000
+#define LC_FAIL_ON_EQUALS       0x80000
 
 /* Global contexts: */
 

From ca11d77efd682b62d97be8851cda9afcdea69d7f Mon Sep 17 00:00:00 2001
From: Ben Kurtovic <ben.kurtovic@verizon.net>
Date: Sat, 17 Nov 2012 21:49:07 -0500
Subject: [PATCH 47/47] Shorten some longer lines.

---
 mwparserfromhell/parser/tokenizer.c | 16 ++++++++++++----
 1 file changed, 12 insertions(+), 4 deletions(-)

diff --git a/mwparserfromhell/parser/tokenizer.c b/mwparserfromhell/parser/tokenizer.c
index 57c6a62..702e1a3 100644
--- a/mwparserfromhell/parser/tokenizer.c
+++ b/mwparserfromhell/parser/tokenizer.c
@@ -1169,14 +1169,20 @@ Tokenizer_verify_safe(Tokenizer* self, int context, Py_UNICODE data)
         }
         else if (context & LC_FAIL_ON_LBRACE) {
             if (data == *"{") {
-                self->topstack->context |= (context & LC_TEMPLATE) ? LC_FAIL_ON_EQUALS : LC_FAIL_NEXT;
+                if (context & LC_TEMPLATE)
+                    self->topstack->context |= LC_FAIL_ON_EQUALS;
+                else
+                    self->topstack->context |= LC_FAIL_NEXT;
                 return;
             }
             self->topstack->context ^= LC_FAIL_ON_LBRACE;
         }
         else if (context & LC_FAIL_ON_RBRACE) {
             if (data == *"}") {
-                self->topstack->context |= (context & LC_TEMPLATE) ? LC_FAIL_ON_EQUALS : LC_FAIL_NEXT;
+                if (context & LC_TEMPLATE)
+                    self->topstack->context |= LC_FAIL_ON_EQUALS;
+                else
+                    self->topstack->context |= LC_FAIL_NEXT;
                 return;
             }
             self->topstack->context ^= LC_FAIL_ON_RBRACE;
@@ -1208,8 +1214,10 @@ Tokenizer_verify_safe(Tokenizer* self, int context, Py_UNICODE data)
 static PyObject*
 Tokenizer_parse(Tokenizer* self, int context)
 {
-    static int fail_contexts = LC_TEMPLATE | LC_ARGUMENT | LC_WIKILINK | LC_HEADING | LC_COMMENT;
-    static int unsafe_contexts = LC_TEMPLATE_NAME | LC_WIKILINK_TITLE | LC_TEMPLATE_PARAM_KEY | LC_ARGUMENT_NAME;
+    static int fail_contexts = (LC_TEMPLATE | LC_ARGUMENT | LC_WIKILINK |
+                                LC_HEADING | LC_COMMENT);
+    static int unsafe_contexts = (LC_TEMPLATE_NAME | LC_WIKILINK_TITLE |
+                                  LC_TEMPLATE_PARAM_KEY | LC_ARGUMENT_NAME);
     int this_context, is_marker, i;
     Py_UNICODE this, next, next_next, last;