From df9f7388b65185210deb5a5402e84750f820f0d6 Mon Sep 17 00:00:00 2001
From: Ben Kurtovic <ben.kurtovic@verizon.net>
Date: Wed, 14 Aug 2013 15:00:45 -0400
Subject: [PATCH 01/12] emit_FAST(), emit_first_FAST(); update comment parsing

---
 mwparserfromhell/parser/tokenizer.c | 328 +++++++++++-------------------------
 mwparserfromhell/parser/tokenizer.h |  86 +++++-----
 2 files changed, 140 insertions(+), 274 deletions(-)

diff --git a/mwparserfromhell/parser/tokenizer.c b/mwparserfromhell/parser/tokenizer.c
index eff000a..a78c6d9 100644
--- a/mwparserfromhell/parser/tokenizer.c
+++ b/mwparserfromhell/parser/tokenizer.c
@@ -368,6 +368,40 @@ static int Tokenizer_emit_first(Tokenizer* self, PyObject* token)
     return 0;
 }
 
+static int Tokenizer_emit_FAST(Tokenizer* self, PyObject* token)
+{
+    PyObject* instance;
+
+    if (Tokenizer_push_textbuffer(self))
+        return -1;
+    instance = PyObject_CallObject(token, NULL);
+    if (!instance)
+        return -1;
+    if (PyList_Append(self->topstack->stack, instance)) {
+        Py_DECREF(instance);
+        return -1;
+    }
+    Py_DECREF(instance);
+    return 0;
+}
+
+static int Tokenizer_emit_first_FAST(Tokenizer* self, PyObject* token)
+{
+    PyObject* instance;
+
+    if (Tokenizer_push_textbuffer(self))
+        return -1;
+    instance = PyObject_CallObject(token, NULL);
+    if (!instance)
+        return -1;
+    if (PyList_Insert(self->topstack->stack, 0, instance)) {
+        Py_DECREF(instance);
+        return -1;
+    }
+    Py_DECREF(instance);
+    return 0;
+}
+
 /*
     Write a Unicode codepoint to the current textbuffer.
 */
@@ -503,7 +537,7 @@ static PyObject* Tokenizer_read_backwards(Tokenizer* self, Py_ssize_t delta)
 */
 static int Tokenizer_parse_template(Tokenizer* self)
 {
-    PyObject *template, *token;
+    PyObject *template;
     Py_ssize_t reset = self->head;
 
     template = Tokenizer_parse(self, LC_TEMPLATE_NAME, 1);
@@ -513,30 +547,17 @@ static int Tokenizer_parse_template(Tokenizer* self)
     }
     if (!template)
         return -1;
-    token = PyObject_CallObject(TemplateOpen, NULL);
-    if (!token) {
+    if (Tokenizer_emit_first_FAST(self, TemplateOpen)) {
         Py_DECREF(template);
         return -1;
     }
-    if (Tokenizer_emit_first(self, token)) {
-        Py_DECREF(token);
-        Py_DECREF(template);
-        return -1;
-    }
-    Py_DECREF(token);
     if (Tokenizer_emit_all(self, template)) {
         Py_DECREF(template);
         return -1;
     }
     Py_DECREF(template);
-    token = PyObject_CallObject(TemplateClose, NULL);
-    if (!token)
-        return -1;
-    if (Tokenizer_emit(self, token)) {
-        Py_DECREF(token);
+    if (Tokenizer_emit_FAST(self, TemplateClose))
         return -1;
-    }
-    Py_DECREF(token);
     return 0;
 }
 
@@ -545,7 +566,7 @@ static int Tokenizer_parse_template(Tokenizer* self)
 */
 static int Tokenizer_parse_argument(Tokenizer* self)
 {
-    PyObject *argument, *token;
+    PyObject *argument;
     Py_ssize_t reset = self->head;
 
     argument = Tokenizer_parse(self, LC_ARGUMENT_NAME, 1);
@@ -555,30 +576,17 @@ static int Tokenizer_parse_argument(Tokenizer* self)
     }
     if (!argument)
         return -1;
-    token = PyObject_CallObject(ArgumentOpen, NULL);
-    if (!token) {
-        Py_DECREF(argument);
-        return -1;
-    }
-    if (Tokenizer_emit_first(self, token)) {
-        Py_DECREF(token);
+    if (Tokenizer_emit_first_FAST(self, ArgumentOpen)) {
         Py_DECREF(argument);
         return -1;
     }
-    Py_DECREF(token);
     if (Tokenizer_emit_all(self, argument)) {
         Py_DECREF(argument);
         return -1;
     }
     Py_DECREF(argument);
-    token = PyObject_CallObject(ArgumentClose, NULL);
-    if (!token)
+    if (Tokenizer_emit_FAST(self, ArgumentClose))
         return -1;
-    if (Tokenizer_emit(self, token)) {
-        Py_DECREF(token);
-        return -1;
-    }
-    Py_DECREF(token);
     return 0;
 }
 
@@ -658,7 +666,7 @@ static int Tokenizer_parse_template_or_argument(Tokenizer* self)
 */
 static int Tokenizer_handle_template_param(Tokenizer* self)
 {
-    PyObject *stack, *token;
+    PyObject *stack;
 
     if (self->topstack->context & LC_TEMPLATE_NAME)
         self->topstack->context ^= LC_TEMPLATE_NAME;
@@ -676,15 +684,8 @@ static int Tokenizer_handle_template_param(Tokenizer* self)
     }
     else
         self->topstack->context |= LC_TEMPLATE_PARAM_KEY;
-
-    token = PyObject_CallObject(TemplateParamSeparator, NULL);
-    if (!token)
-        return -1;
-    if (Tokenizer_emit(self, token)) {
-        Py_DECREF(token);
+    if (Tokenizer_emit_FAST(self, TemplateParamSeparator))
         return -1;
-    }
-    Py_DECREF(token);
     if (Tokenizer_push(self, self->topstack->context))
         return -1;
     return 0;
@@ -695,7 +696,7 @@ static int Tokenizer_handle_template_param(Tokenizer* self)
 */
 static int Tokenizer_handle_template_param_value(Tokenizer* self)
 {
-    PyObject *stack, *token;
+    PyObject *stack;
 
     stack = Tokenizer_pop_keeping_context(self);
     if (!stack)
@@ -707,14 +708,8 @@ static int Tokenizer_handle_template_param_value(Tokenizer* self)
     Py_DECREF(stack);
     self->topstack->context ^= LC_TEMPLATE_PARAM_KEY;
     self->topstack->context |= LC_TEMPLATE_PARAM_VALUE;
-    token = PyObject_CallObject(TemplateParamEquals, NULL);
-    if (!token)
+    if (Tokenizer_emit_FAST(self, TemplateParamEquals))
         return -1;
-    if (Tokenizer_emit(self, token)) {
-        Py_DECREF(token);
-        return -1;
-    }
-    Py_DECREF(token);
     return 0;
 }
 
@@ -745,17 +740,10 @@ static PyObject* Tokenizer_handle_template_end(Tokenizer* self)
 */
 static int Tokenizer_handle_argument_separator(Tokenizer* self)
 {
-    PyObject* token;
     self->topstack->context ^= LC_ARGUMENT_NAME;
     self->topstack->context |= LC_ARGUMENT_DEFAULT;
-    token = PyObject_CallObject(ArgumentSeparator, NULL);
-    if (!token)
-        return -1;
-    if (Tokenizer_emit(self, token)) {
-        Py_DECREF(token);
+    if (Tokenizer_emit_FAST(self, ArgumentSeparator))
         return -1;
-    }
-    Py_DECREF(token);
     return 0;
 }
 
@@ -765,6 +753,7 @@ static int Tokenizer_handle_argument_separator(Tokenizer* self)
 static PyObject* Tokenizer_handle_argument_end(Tokenizer* self)
 {
     PyObject* stack = Tokenizer_pop(self);
+
     self->head += 2;
     return stack;
 }
@@ -775,7 +764,7 @@ static PyObject* Tokenizer_handle_argument_end(Tokenizer* self)
 static int Tokenizer_parse_wikilink(Tokenizer* self)
 {
     Py_ssize_t reset;
-    PyObject *wikilink, *token;
+    PyObject *wikilink;
 
     self->head += 2;
     reset = self->head - 1;
@@ -789,30 +778,17 @@ static int Tokenizer_parse_wikilink(Tokenizer* self)
     }
     if (!wikilink)
         return -1;
-    token = PyObject_CallObject(WikilinkOpen, NULL);
-    if (!token) {
+    if (Tokenizer_emit_FAST(self, WikilinkOpen)) {
         Py_DECREF(wikilink);
         return -1;
     }
-    if (Tokenizer_emit(self, token)) {
-        Py_DECREF(token);
-        Py_DECREF(wikilink);
-        return -1;
-    }
-    Py_DECREF(token);
     if (Tokenizer_emit_all(self, wikilink)) {
         Py_DECREF(wikilink);
         return -1;
     }
     Py_DECREF(wikilink);
-    token = PyObject_CallObject(WikilinkClose, NULL);
-    if (!token)
-        return -1;
-    if (Tokenizer_emit(self, token)) {
-        Py_DECREF(token);
+    if (Tokenizer_emit_FAST(self, WikilinkClose))
         return -1;
-    }
-    Py_DECREF(token);
     if (self->topstack->context & LC_FAIL_NEXT)
         self->topstack->context ^= LC_FAIL_NEXT;
     return 0;
@@ -823,17 +799,10 @@ static int Tokenizer_parse_wikilink(Tokenizer* self)
 */
 static int Tokenizer_handle_wikilink_separator(Tokenizer* self)
 {
-    PyObject* token;
     self->topstack->context ^= LC_WIKILINK_TITLE;
     self->topstack->context |= LC_WIKILINK_TEXT;
-    token = PyObject_CallObject(WikilinkSeparator, NULL);
-    if (!token)
-        return -1;
-    if (Tokenizer_emit(self, token)) {
-        Py_DECREF(token);
+    if (Tokenizer_emit_FAST(self, WikilinkSeparator))
         return -1;
-    }
-    Py_DECREF(token);
     return 0;
 }
 
@@ -921,14 +890,8 @@ static int Tokenizer_parse_heading(Tokenizer* self)
     }
     Py_DECREF(heading->title);
     free(heading);
-    token = PyObject_CallObject(HeadingEnd, NULL);
-    if (!token)
+    if (Tokenizer_emit_FAST(self, HeadingEnd))
         return -1;
-    if (Tokenizer_emit(self, token)) {
-        Py_DECREF(token);
-        return -1;
-    }
-    Py_DECREF(token);
     self->global ^= GL_HEADING;
     return 0;
 }
@@ -1010,14 +973,8 @@ static int Tokenizer_really_parse_entity(Tokenizer* self)
         return 0;                   \
     }
 
-    token = PyObject_CallObject(HTMLEntityStart, NULL);
-    if (!token)
+    if (Tokenizer_emit_FAST(self, HTMLEntityStart))
         return -1;
-    if (Tokenizer_emit(self, token)) {
-        Py_DECREF(token);
-        return -1;
-    }
-    Py_DECREF(token);
     self->head++;
     this = Tokenizer_READ(self, 0);
     if (this == *"") {
@@ -1026,14 +983,8 @@ static int Tokenizer_really_parse_entity(Tokenizer* self)
     }
     if (this == *"#") {
         numeric = 1;
-        token = PyObject_CallObject(HTMLEntityNumeric, NULL);
-        if (!token)
+        if (Tokenizer_emit_FAST(self, HTMLEntityNumeric))
             return -1;
-        if (Tokenizer_emit(self, token)) {
-            Py_DECREF(token);
-            return -1;
-        }
-        Py_DECREF(token);
         self->head++;
         this = Tokenizer_READ(self, 0);
         if (this == *"") {
@@ -1156,14 +1107,8 @@ static int Tokenizer_really_parse_entity(Tokenizer* self)
         return -1;
     }
     Py_DECREF(token);
-    token = PyObject_CallObject(HTMLEntityEnd, NULL);
-    if (!token)
-        return -1;
-    if (Tokenizer_emit(self, token)) {
-        Py_DECREF(token);
+    if (Tokenizer_emit_FAST(self, HTMLEntityEnd))
         return -1;
-    }
-    Py_DECREF(token);
     return 0;
 }
 
@@ -1203,45 +1148,39 @@ static int Tokenizer_parse_entity(Tokenizer* self)
 static int Tokenizer_parse_comment(Tokenizer* self)
 {
     Py_ssize_t reset = self->head + 3;
-    PyObject *token, *comment;
+    PyObject *comment;
+    Py_UNICODE this;
 
     self->head += 4;
-    comment = Tokenizer_parse(self, LC_COMMENT, 1);
-    if (BAD_ROUTE) {
-        RESET_ROUTE();
-        self->head = reset;
-        if (Tokenizer_emit_text(self, "<!--"))
-            return -1;
-        return 0;
-    }
-    if (!comment)
-        return -1;
-    token = PyObject_CallObject(CommentStart, NULL);
-    if (!token) {
-        Py_DECREF(comment);
-        return -1;
-    }
-    if (Tokenizer_emit(self, token)) {
-        Py_DECREF(token);
-        Py_DECREF(comment);
-        return -1;
-    }
-    Py_DECREF(token);
-    if (Tokenizer_emit_all(self, comment)) {
-        Py_DECREF(comment);
-        return -1;
-    }
-    Py_DECREF(comment);
-    token = PyObject_CallObject(CommentEnd, NULL);
-    if (!token)
-        return -1;
-    if (Tokenizer_emit(self, token)) {
-        Py_DECREF(token);
+    if (Tokenizer_push(self, 0))
         return -1;
+    while (1) {
+        this = Tokenizer_READ(self, 0);
+        if (this == *"") {
+            comment = Tokenizer_pop(self);
+            Py_XDECREF(comment);
+            self->head = reset;
+            return Tokenizer_emit_text(self, "<!--");
+        }
+        if (this == *"-" && Tokenizer_READ(self, 1) == this &&
+                            Tokenizer_READ(self, 2) == *">") {
+            if (Tokenizer_emit_first_FAST(self, CommentStart))
+                return -1;
+            if (Tokenizer_emit_FAST(self, CommentEnd))
+                return -1;
+            comment = Tokenizer_pop(self);
+            if (!comment)
+                return -1;
+            if (Tokenizer_emit_all(self, comment))
+                return -1;
+            Py_DECREF(comment);
+            self->head += 2;
+            return 0;
+        }
+        if (Tokenizer_emit_char(self, this))
+            return -1;
+        self->head++;
     }
-    Py_DECREF(token);
-    self->head += 2;
-    return 0;
 }
 
 /*
@@ -1253,14 +1192,8 @@ static int Tokenizer_push_tag_buffer(Tokenizer* self, TagData* data)
              *pad_after_eq;
 
     if (data->context & TAG_QUOTED) {
-        token = PyObject_CallObject(TagAttrQuote, NULL);
-        if (!token)
+        if (Tokenizer_emit_first_FAST(self, TagAttrQuote))
             return -1;
-        if (Tokenizer_emit_first(self, token)) {
-            Py_DECREF(token);
-            return -1;
-        }
-        Py_DECREF(token);
         tokens = Tokenizer_pop(self);
         if (!tokens)
             return -1;
@@ -1370,7 +1303,7 @@ static int Tokenizer_handle_tag_text(Tokenizer* self, Py_UNICODE text)
 static int
 Tokenizer_handle_tag_data(Tokenizer* self, TagData* data, Py_UNICODE chunk)
 {
-    PyObject *trash, *token;
+    PyObject *trash;
     int first_time, i, is_marker = 0, escaped;
 
     if (data->context & TAG_NAME) {
@@ -1414,14 +1347,8 @@ Tokenizer_handle_tag_data(Tokenizer* self, TagData* data, Py_UNICODE chunk)
     else if (data->context & TAG_ATTR_NAME) {
         if (chunk == *"=") {
             data->context = TAG_ATTR_VALUE | TAG_NOTE_QUOTE;
-            token = PyObject_CallObject(TagAttrEquals, NULL);
-            if (!token)
+            if (Tokenizer_emit_FAST(self, TagAttrEquals))
                 return -1;
-            if (Tokenizer_emit(self, token)) {
-                Py_DECREF(token);
-                return -1;
-            }
-            Py_DECREF(token);
             return 0;
         }
         if (data->context & TAG_NOTE_EQUALS) {
@@ -1495,16 +1422,8 @@ Tokenizer_handle_tag_close_open(Tokenizer* self, TagData* data, PyObject* cls)
 */
 static int Tokenizer_handle_tag_open_close(Tokenizer* self)
 {
-    PyObject* token;
-
-    token = PyObject_CallObject(TagOpenClose, NULL);
-    if (!token)
+    if (Tokenizer_emit_FAST(self, TagOpenClose))
         return -1;
-    if (Tokenizer_emit(self, token)) {
-        Py_DECREF(token);
-        return -1;
-    }
-    Py_DECREF(token);
     if (Tokenizer_push(self, LC_TAG_CLOSE))
         return -1;
     self->head++;
@@ -1516,7 +1435,7 @@ static int Tokenizer_handle_tag_open_close(Tokenizer* self)
 */
 static PyObject* Tokenizer_handle_tag_close_close(Tokenizer* self)
 {
-    PyObject *closing, *first, *so, *sc, *token;
+    PyObject *closing, *first, *so, *sc;
     int valid = 1;
 
     closing = Tokenizer_pop(self);
@@ -1557,14 +1476,8 @@ static PyObject* Tokenizer_handle_tag_close_close(Tokenizer* self)
         return NULL;
     }
     Py_DECREF(closing);
-    token = PyObject_CallObject(TagCloseClose, NULL);
-    if (!token)
-        return NULL;
-    if (Tokenizer_emit(self, token)) {
-        Py_DECREF(token);
+    if (Tokenizer_emit_FAST(self, TagCloseClose))
         return NULL;
-    }
-    Py_DECREF(token);
     return Tokenizer_pop(self);
 }
 
@@ -1684,17 +1597,10 @@ static PyObject* Tokenizer_really_parse_tag(Tokenizer* self)
         TagData_dealloc(data);
         return NULL;
     }
-    token = PyObject_CallObject(TagOpenOpen, NULL);
-    if (!token) {
+    if (Tokenizer_emit_FAST(self, TagOpenOpen)) {
         TagData_dealloc(data);
         return NULL;
     }
-    if (Tokenizer_emit(self, token)) {
-        Py_DECREF(token);
-        TagData_dealloc(data);
-        return NULL;
-    }
-    Py_DECREF(token);
     while (1) {
         this = Tokenizer_READ(self, 0);
         next = Tokenizer_READ(self, 1);
@@ -1864,35 +1770,17 @@ static int Tokenizer_emit_style_tag(Tokenizer* self, const char* tag,
     Py_DECREF(token);
     if (Tokenizer_emit_text(self, tag))
         return -1;
-    token = PyObject_CallObject(TagCloseOpen, NULL);
-    if (!token)
-        return -1;
-    if (Tokenizer_emit(self, token)) {
-        Py_DECREF(token);
+    if (Tokenizer_emit_FAST(self, TagCloseOpen))
         return -1;
-    }
-    Py_DECREF(token);
     if (Tokenizer_emit_all(self, body))
         return -1;
-    token = PyObject_CallObject(TagOpenClose, NULL);
-    if (!token)
-        return -1;
-    if (Tokenizer_emit(self, token)) {
-        Py_DECREF(token);
+    Py_DECREF(body);
+    if (Tokenizer_emit_FAST(self, TagOpenClose))
         return -1;
-    }
-    Py_DECREF(token);
     if (Tokenizer_emit_text(self, tag))
         return -1;
-    token = PyObject_CallObject(TagCloseClose, NULL);
-    if (!token)
-        return -1;
-    if (Tokenizer_emit(self, token)) {
-        Py_DECREF(token);
+    if (Tokenizer_emit_FAST(self, TagCloseClose))
         return -1;
-    }
-    Py_DECREF(token);
-    Py_DECREF(body);
     return 0;
 }
 
@@ -2108,14 +1996,8 @@ static int Tokenizer_handle_list_marker(Tokenizer* self)
     Py_DECREF(token);
     if (Tokenizer_emit_text(self, GET_HTML_TAG(code)))
         return -1;
-    token = PyObject_CallObject(TagCloseSelfclose, NULL);
-    if (!token)
-        return -1;
-    if (Tokenizer_emit(self, token)) {
-        Py_DECREF(token);
+    if (Tokenizer_emit_FAST(self, TagCloseSelfclose))
         return -1;
-    }
-    Py_DECREF(token);
     return 0;
 }
 
@@ -2181,14 +2063,8 @@ static int Tokenizer_handle_hr(Tokenizer* self)
     Py_DECREF(token);
     if (Tokenizer_emit_text(self, "hr"))
         return -1;
-    token = PyObject_CallObject(TagCloseSelfclose, NULL);
-    if (!token)
-        return -1;
-    if (Tokenizer_emit(self, token)) {
-        Py_DECREF(token);
+    if (Tokenizer_emit_FAST(self, TagCloseSelfclose))
         return -1;
-    }
-    Py_DECREF(token);
     return 0;
 }
 
@@ -2209,7 +2085,7 @@ static int Tokenizer_handle_dl_term(Tokenizer* self)
 static PyObject* Tokenizer_handle_end(Tokenizer* self, int context)
 {
     static int fail_contexts = (LC_TEMPLATE | LC_ARGUMENT | LC_WIKILINK |
-                                LC_HEADING | LC_COMMENT | LC_TAG | LC_STYLE);
+                                LC_HEADING | LC_TAG | LC_STYLE);
     static int double_fail = (LC_TEMPLATE_PARAM_KEY | LC_TAG_CLOSE);
     PyObject *token, *text, *trash;
     int single;
@@ -2359,15 +2235,7 @@ static PyObject* Tokenizer_parse(Tokenizer* self, int context, int push)
             return Tokenizer_handle_end(self, this_context);
         next = Tokenizer_READ(self, 1);
         last = Tokenizer_READ_BACKWARDS(self, 1);
-        if (this_context & LC_COMMENT) {
-            if (this == next && next == *"-") {
-                if (Tokenizer_READ(self, 2) == *">")
-                    return Tokenizer_pop(self);
-            }
-            if (Tokenizer_emit_char(self, this))
-                return NULL;
-        }
-        else if (this == next && next == *"{") {
+        if (this == next && next == *"{") {
             if (Tokenizer_CAN_RECURSE(self)) {
                 if (Tokenizer_parse_template_or_argument(self))
                     return NULL;
diff --git a/mwparserfromhell/parser/tokenizer.h b/mwparserfromhell/parser/tokenizer.h
index 4136285..cc6dc09 100644
--- a/mwparserfromhell/parser/tokenizer.h
+++ b/mwparserfromhell/parser/tokenizer.h
@@ -103,50 +103,48 @@ static PyObject* TagCloseClose;
 
 /* Local contexts: */
 
-#define LC_TEMPLATE             0x00000007
-#define LC_TEMPLATE_NAME        0x00000001
-#define LC_TEMPLATE_PARAM_KEY   0x00000002
-#define LC_TEMPLATE_PARAM_VALUE 0x00000004
-
-#define LC_ARGUMENT             0x00000018
-#define LC_ARGUMENT_NAME        0x00000008
-#define LC_ARGUMENT_DEFAULT     0x00000010
-
-#define LC_WIKILINK             0x00000060
-#define LC_WIKILINK_TITLE       0x00000020
-#define LC_WIKILINK_TEXT        0x00000040
-
-#define LC_HEADING              0x00001F80
-#define LC_HEADING_LEVEL_1      0x00000080
-#define LC_HEADING_LEVEL_2      0x00000100
-#define LC_HEADING_LEVEL_3      0x00000200
-#define LC_HEADING_LEVEL_4      0x00000400
-#define LC_HEADING_LEVEL_5      0x00000800
-#define LC_HEADING_LEVEL_6      0x00001000
-
-#define LC_COMMENT              0x00002000
-
-#define LC_TAG                  0x0003C000
-#define LC_TAG_OPEN             0x00004000
-#define LC_TAG_ATTR             0x00008000
-#define LC_TAG_BODY             0x00010000
-#define LC_TAG_CLOSE            0x00020000
-
-#define LC_STYLE                0x003C0000
-#define LC_STYLE_ITALICS        0x00040000
-#define LC_STYLE_BOLD           0x00080000
-#define LC_STYLE_PASS_AGAIN     0x00100000
-#define LC_STYLE_SECOND_PASS    0x00200000
-
-#define LC_DLTERM               0x00400000
-
-#define LC_SAFETY_CHECK         0x1F800000
-#define LC_HAS_TEXT             0x00800000
-#define LC_FAIL_ON_TEXT         0x01000000
-#define LC_FAIL_NEXT            0x02000000
-#define LC_FAIL_ON_LBRACE       0x04000000
-#define LC_FAIL_ON_RBRACE       0x08000000
-#define LC_FAIL_ON_EQUALS       0x10000000
+#define LC_TEMPLATE             0x0000007
+#define LC_TEMPLATE_NAME        0x0000001
+#define LC_TEMPLATE_PARAM_KEY   0x0000002
+#define LC_TEMPLATE_PARAM_VALUE 0x0000004
+
+#define LC_ARGUMENT             0x0000018
+#define LC_ARGUMENT_NAME        0x0000008
+#define LC_ARGUMENT_DEFAULT     0x0000010
+
+#define LC_WIKILINK             0x0000060
+#define LC_WIKILINK_TITLE       0x0000020
+#define LC_WIKILINK_TEXT        0x0000040
+
+#define LC_HEADING              0x0001F80
+#define LC_HEADING_LEVEL_1      0x0000080
+#define LC_HEADING_LEVEL_2      0x0000100
+#define LC_HEADING_LEVEL_3      0x0000200
+#define LC_HEADING_LEVEL_4      0x0000400
+#define LC_HEADING_LEVEL_5      0x0000800
+#define LC_HEADING_LEVEL_6      0x0001000
+
+#define LC_TAG                  0x001E000
+#define LC_TAG_OPEN             0x0002000
+#define LC_TAG_ATTR             0x0004000
+#define LC_TAG_BODY             0x0008000
+#define LC_TAG_CLOSE            0x0010000
+
+#define LC_STYLE                0x01E0000
+#define LC_STYLE_ITALICS        0x0020000
+#define LC_STYLE_BOLD           0x0040000
+#define LC_STYLE_PASS_AGAIN     0x0080000
+#define LC_STYLE_SECOND_PASS    0x0100000
+
+#define LC_DLTERM               0x0200000
+
+#define LC_SAFETY_CHECK         0xFC00000
+#define LC_HAS_TEXT             0x0400000
+#define LC_FAIL_ON_TEXT         0x0800000
+#define LC_FAIL_NEXT            0x1000000
+#define LC_FAIL_ON_LBRACE       0x2000000
+#define LC_FAIL_ON_RBRACE       0x4000000
+#define LC_FAIL_ON_EQUALS       0x8000000
 
 /* Global contexts: */
 

From 51ac97de04108a3256467170952e043dde379f26 Mon Sep 17 00:00:00 2001
From: Ben Kurtovic <ben.kurtovic@verizon.net>
Date: Wed, 14 Aug 2013 15:06:24 -0400
Subject: [PATCH 02/12] Make macros out of the failing/unsafe contexts.

---
 mwparserfromhell/parser/tokenizer.c | 14 ++++----------
 mwparserfromhell/parser/tokenizer.h |  6 ++++++
 2 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/mwparserfromhell/parser/tokenizer.c b/mwparserfromhell/parser/tokenizer.c
index a78c6d9..50bb3b0 100644
--- a/mwparserfromhell/parser/tokenizer.c
+++ b/mwparserfromhell/parser/tokenizer.c
@@ -2084,13 +2084,10 @@ static int Tokenizer_handle_dl_term(Tokenizer* self)
 */
 static PyObject* Tokenizer_handle_end(Tokenizer* self, int context)
 {
-    static int fail_contexts = (LC_TEMPLATE | LC_ARGUMENT | LC_WIKILINK |
-                                LC_HEADING | LC_TAG | LC_STYLE);
-    static int double_fail = (LC_TEMPLATE_PARAM_KEY | LC_TAG_CLOSE);
     PyObject *token, *text, *trash;
     int single;
 
-    if (context & fail_contexts) {
+    if (context & AGG_FAIL) {
         if (context & LC_TAG_BODY) {
             token = PyList_GET_ITEM(self->topstack->stack, 1);
             text = PyObject_GetAttrString(token, "text");
@@ -2101,7 +2098,7 @@ static PyObject* Tokenizer_handle_end(Tokenizer* self, int context)
             if (single)
                 return Tokenizer_handle_single_tag_end(self);
         }
-        else if (context & double_fail) {
+        else if (context & AGG_DOUBLE) {
             trash = Tokenizer_pop(self);
             Py_XDECREF(trash);
         }
@@ -2195,9 +2192,6 @@ static int Tokenizer_verify_safe(Tokenizer* self, int context, Py_UNICODE data)
 */
 static PyObject* Tokenizer_parse(Tokenizer* self, int context, int push)
 {
-    static int unsafe_contexts = (LC_TEMPLATE_NAME | LC_WIKILINK_TITLE |
-                                  LC_TEMPLATE_PARAM_KEY | LC_ARGUMENT_NAME);
-    static int double_unsafe = (LC_TEMPLATE_PARAM_KEY | LC_TAG_CLOSE);
     int this_context, is_marker, i;
     Py_UNICODE this, next, next_next, last;
     PyObject* temp;
@@ -2209,9 +2203,9 @@ static PyObject* Tokenizer_parse(Tokenizer* self, int context, int push)
     while (1) {
         this = Tokenizer_READ(self, 0);
         this_context = self->topstack->context;
-        if (this_context & unsafe_contexts) {
+        if (this_context & AGG_UNSAFE) {
             if (Tokenizer_verify_safe(self, this_context, this) < 0) {
-                if (this_context & double_unsafe) {
+                if (this_context & AGG_DOUBLE) {
                     temp = Tokenizer_pop(self);
                     Py_XDECREF(temp);
                 }
diff --git a/mwparserfromhell/parser/tokenizer.h b/mwparserfromhell/parser/tokenizer.h
index cc6dc09..1ac9168 100644
--- a/mwparserfromhell/parser/tokenizer.h
+++ b/mwparserfromhell/parser/tokenizer.h
@@ -150,6 +150,12 @@ static PyObject* TagCloseClose;
 
 #define GL_HEADING 0x1
 
+/* Aggregate contexts: */
+
+#define AGG_FAIL   (LC_TEMPLATE | LC_ARGUMENT | LC_WIKILINK | LC_HEADING | LC_TAG | LC_STYLE)
+#define AGG_UNSAFE (LC_TEMPLATE_NAME | LC_WIKILINK_TITLE | LC_TEMPLATE_PARAM_KEY | LC_ARGUMENT_NAME)
+#define AGG_DOUBLE (LC_TEMPLATE_PARAM_KEY | LC_TAG_CLOSE)
+
 /* Tag contexts: */
 
 #define TAG_NAME        0x01

From 6036dc9d62684d39cf423893371b80c21a86098d Mon Sep 17 00:00:00 2001
From: Ben Kurtovic <ben.kurtovic@verizon.net>
Date: Wed, 14 Aug 2013 17:57:52 -0400
Subject: [PATCH 03/12] Finish new emit_first() and emit_first_kwargs()

---
 mwparserfromhell/parser/tokenizer.c | 64 ++++++++++++++++++++++---------------
 1 file changed, 39 insertions(+), 25 deletions(-)

diff --git a/mwparserfromhell/parser/tokenizer.c b/mwparserfromhell/parser/tokenizer.c
index 50bb3b0..dd36ab8 100644
--- a/mwparserfromhell/parser/tokenizer.c
+++ b/mwparserfromhell/parser/tokenizer.c
@@ -347,28 +347,39 @@ static void* Tokenizer_fail_route(Tokenizer* self)
 /*
     Write a token to the end of the current token stack.
 */
-static int Tokenizer_emit(Tokenizer* self, PyObject* token)
+static int Tokenizer_emit_FAST(Tokenizer* self, PyObject* token)
 {
+    PyObject* instance;
+
     if (Tokenizer_push_textbuffer(self))
         return -1;
-    if (PyList_Append(self->topstack->stack, token))
+    instance = PyObject_CallObject(token, NULL);
+    if (!instance)
         return -1;
+    if (PyList_Append(self->topstack->stack, instance)) {
+        Py_DECREF(instance);
+        return -1;
+    }
+    Py_DECREF(instance);
     return 0;
 }
 
 /*
-    Write a token to the beginning of the current token stack.
+    Write a token to the end of the current token stack.
 */
-static int Tokenizer_emit_first(Tokenizer* self, PyObject* token)
+static int Tokenizer_emit(Tokenizer* self, PyObject* token)
 {
     if (Tokenizer_push_textbuffer(self))
         return -1;
-    if (PyList_Insert(self->topstack->stack, 0, token))
+    if (PyList_Append(self->topstack->stack, token))
         return -1;
     return 0;
 }
 
-static int Tokenizer_emit_FAST(Tokenizer* self, PyObject* token)
+/*
+    Write a token to the beginning of the current token stack.
+*/
+static int Tokenizer_emit_first(Tokenizer* self, PyObject* token)
 {
     PyObject* instance;
 
@@ -377,7 +388,7 @@ static int Tokenizer_emit_FAST(Tokenizer* self, PyObject* token)
     instance = PyObject_CallObject(token, NULL);
     if (!instance)
         return -1;
-    if (PyList_Append(self->topstack->stack, instance)) {
+    if (PyList_Insert(self->topstack->stack, 0, instance)) {
         Py_DECREF(instance);
         return -1;
     }
@@ -385,20 +396,31 @@ static int Tokenizer_emit_FAST(Tokenizer* self, PyObject* token)
     return 0;
 }
 
-static int Tokenizer_emit_first_FAST(Tokenizer* self, PyObject* token)
+/*
+    Write a token to the beginning of the current token stack, with kwargs.
+    Steals a reference to kwargs.
+*/
+static int Tokenizer_emit_first_kwargs(Tokenizer* self, PyObject* token,
+                                       PyObject* kwargs)
 {
     PyObject* instance;
 
-    if (Tokenizer_push_textbuffer(self))
+    if (Tokenizer_push_textbuffer(self)) {
+        Py_DECREF(kwargs);
         return -1;
-    instance = PyObject_CallObject(token, NULL);
-    if (!instance)
+    }
+    instance = PyObject_Call(token, NOARGS, kwargs);
+    if (!instance) {
+        Py_DECREF(kwargs);
         return -1;
+    }
     if (PyList_Insert(self->topstack->stack, 0, instance)) {
         Py_DECREF(instance);
+        Py_DECREF(kwargs);
         return -1;
     }
     Py_DECREF(instance);
+    Py_DECREF(kwargs);
     return 0;
 }
 
@@ -547,7 +569,7 @@ static int Tokenizer_parse_template(Tokenizer* self)
     }
     if (!template)
         return -1;
-    if (Tokenizer_emit_first_FAST(self, TemplateOpen)) {
+    if (Tokenizer_emit_first(self, TemplateOpen)) {
         Py_DECREF(template);
         return -1;
     }
@@ -576,7 +598,7 @@ static int Tokenizer_parse_argument(Tokenizer* self)
     }
     if (!argument)
         return -1;
-    if (Tokenizer_emit_first_FAST(self, ArgumentOpen)) {
+    if (Tokenizer_emit_first(self, ArgumentOpen)) {
         Py_DECREF(argument);
         return -1;
     }
@@ -1164,7 +1186,7 @@ static int Tokenizer_parse_comment(Tokenizer* self)
         }
         if (this == *"-" && Tokenizer_READ(self, 1) == this &&
                             Tokenizer_READ(self, 2) == *">") {
-            if (Tokenizer_emit_first_FAST(self, CommentStart))
+            if (Tokenizer_emit_first(self, CommentStart))
                 return -1;
             if (Tokenizer_emit_FAST(self, CommentEnd))
                 return -1;
@@ -1188,11 +1210,10 @@ static int Tokenizer_parse_comment(Tokenizer* self)
 */
 static int Tokenizer_push_tag_buffer(Tokenizer* self, TagData* data)
 {
-    PyObject *token, *tokens, *kwargs, *pad_first, *pad_before_eq,
-             *pad_after_eq;
+    PyObject *tokens, *kwargs, *pad_first, *pad_before_eq, *pad_after_eq;
 
     if (data->context & TAG_QUOTED) {
-        if (Tokenizer_emit_first_FAST(self, TagAttrQuote))
+        if (Tokenizer_emit_first(self, TagAttrQuote))
             return -1;
         tokens = Tokenizer_pop(self);
         if (!tokens)
@@ -1217,15 +1238,8 @@ static int Tokenizer_push_tag_buffer(Tokenizer* self, TagData* data)
     Py_DECREF(pad_first);
     Py_DECREF(pad_before_eq);
     Py_DECREF(pad_after_eq);
-    token = PyObject_Call(TagAttrStart, NOARGS, kwargs);
-    Py_DECREF(kwargs);
-    if (!token)
-        return -1;
-    if (Tokenizer_emit_first(self, token)) {
-        Py_DECREF(token);
+    if (Tokenizer_emit_first_kwargs(self, TagAttrStart, kwargs))
         return -1;
-    }
-    Py_DECREF(token);
     tokens = Tokenizer_pop(self);
     if (!tokens)
         return -1;

From a07a96d4babd558cb6a1ec00300f2975597c06de Mon Sep 17 00:00:00 2001
From: Ben Kurtovic <ben.kurtovic@verizon.net>
Date: Wed, 14 Aug 2013 18:11:58 -0400
Subject: [PATCH 04/12] Finish emit()'s kwargs version.

---
 mwparserfromhell/parser/tokenizer.c | 114 ++++++++++--------------------------
 1 file changed, 32 insertions(+), 82 deletions(-)

diff --git a/mwparserfromhell/parser/tokenizer.c b/mwparserfromhell/parser/tokenizer.c
index dd36ab8..1ddb76b 100644
--- a/mwparserfromhell/parser/tokenizer.c
+++ b/mwparserfromhell/parser/tokenizer.c
@@ -367,12 +367,26 @@ static int Tokenizer_emit_FAST(Tokenizer* self, PyObject* token)
 /*
     Write a token to the end of the current token stack.
 */
-static int Tokenizer_emit(Tokenizer* self, PyObject* token)
+static int Tokenizer_emit(Tokenizer* self, PyObject* token, PyObject* kwargs)
 {
-    if (Tokenizer_push_textbuffer(self))
+    PyObject* instance;
+
+    if (Tokenizer_push_textbuffer(self)) {
+        Py_DECREF(kwargs);
         return -1;
-    if (PyList_Append(self->topstack->stack, token))
+    }
+    instance = PyObject_Call(token, NOARGS, kwargs);
+    if (!instance) {
+        Py_DECREF(kwargs);
         return -1;
+    }
+    if (PyList_Append(self->topstack->stack, instance)) {
+        Py_DECREF(instance);
+        Py_DECREF(kwargs);
+        return -1;
+    }
+    Py_DECREF(instance);
+    Py_DECREF(kwargs);
     return 0;
 }
 
@@ -846,7 +860,7 @@ static int Tokenizer_parse_heading(Tokenizer* self)
     Py_ssize_t reset = self->head;
     int best = 1, i, context, diff;
     HeadingData *heading;
-    PyObject *level, *kwargs, *token;
+    PyObject *level, *kwargs;
 
     self->global |= GL_HEADING;
     self->head += 1;
@@ -881,20 +895,11 @@ static int Tokenizer_parse_heading(Tokenizer* self)
     }
     PyDict_SetItemString(kwargs, "level", level);
     Py_DECREF(level);
-    token = PyObject_Call(HeadingStart, NOARGS, kwargs);
-    Py_DECREF(kwargs);
-    if (!token) {
+    if (Tokenizer_emit(self, HeadingStart, kwargs)) {
         Py_DECREF(heading->title);
         free(heading);
         return -1;
     }
-    if (Tokenizer_emit(self, token)) {
-        Py_DECREF(token);
-        Py_DECREF(heading->title);
-        free(heading);
-        return -1;
-    }
-    Py_DECREF(token);
     if (heading->level < best) {
         diff = best - heading->level;
         for (i = 0; i < diff; i++) {
@@ -984,7 +989,7 @@ static HeadingData* Tokenizer_handle_heading_end(Tokenizer* self)
 */
 static int Tokenizer_really_parse_entity(Tokenizer* self)
 {
-    PyObject *token, *kwargs, *textobj;
+    PyObject *kwargs, *textobj;
     Py_UNICODE this;
     int numeric, hexadecimal, i, j, zeroes, test;
     char *valid, *text, *buffer, *def;
@@ -1019,15 +1024,8 @@ static int Tokenizer_really_parse_entity(Tokenizer* self)
             if (!kwargs)
                 return -1;
             PyDict_SetItemString(kwargs, "char", Tokenizer_read(self, 0));
-            token = PyObject_Call(HTMLEntityHex, NOARGS, kwargs);
-            Py_DECREF(kwargs);
-            if (!token)
+            if (Tokenizer_emit(self, HTMLEntityHex, kwargs))
                 return -1;
-            if (Tokenizer_emit(self, token)) {
-                Py_DECREF(token);
-                return -1;
-            }
-            Py_DECREF(token);
             self->head++;
         }
         else
@@ -1120,15 +1118,8 @@ static int Tokenizer_really_parse_entity(Tokenizer* self)
     }
     PyDict_SetItemString(kwargs, "text", textobj);
     Py_DECREF(textobj);
-    token = PyObject_Call(Text, NOARGS, kwargs);
-    Py_DECREF(kwargs);
-    if (!token)
-        return -1;
-    if (Tokenizer_emit(self, token)) {
-        Py_DECREF(token);
+    if (Tokenizer_emit(self, Text, kwargs))
         return -1;
-    }
-    Py_DECREF(token);
     if (Tokenizer_emit_FAST(self, HTMLEntityEnd))
         return -1;
     return 0;
@@ -1402,7 +1393,7 @@ Tokenizer_handle_tag_data(Tokenizer* self, TagData* data, Py_UNICODE chunk)
 static int
 Tokenizer_handle_tag_close_open(Tokenizer* self, TagData* data, PyObject* cls)
 {
-    PyObject *padding, *kwargs, *token;
+    PyObject *padding, *kwargs;
 
     if (data->context & (TAG_ATTR_NAME | TAG_ATTR_VALUE)) {
         if (Tokenizer_push_tag_buffer(self, data))
@@ -1418,15 +1409,8 @@ Tokenizer_handle_tag_close_open(Tokenizer* self, TagData* data, PyObject* cls)
     }
     PyDict_SetItemString(kwargs, "padding", padding);
     Py_DECREF(padding);
-    token = PyObject_Call(cls, NOARGS, kwargs);
-    Py_DECREF(kwargs);
-    if (!token)
-        return -1;
-    if (Tokenizer_emit(self, token)) {
-        Py_DECREF(token);
+    if (Tokenizer_emit(self, cls, kwargs))
         return -1;
-    }
-    Py_DECREF(token);
     self->head++;
     return 0;
 }
@@ -1523,7 +1507,7 @@ static PyObject* Tokenizer_handle_blacklisted_tag(Tokenizer* self)
 */
 static PyObject* Tokenizer_handle_single_only_tag_end(Tokenizer* self)
 {
-    PyObject *top, *padding, *kwargs, *token;
+    PyObject *top, *padding, *kwargs;
 
     top = PyObject_CallMethod(self->topstack->stack, "pop", NULL);
     if (!top)
@@ -1540,15 +1524,8 @@ static PyObject* Tokenizer_handle_single_only_tag_end(Tokenizer* self)
     PyDict_SetItemString(kwargs, "padding", padding);
     PyDict_SetItemString(kwargs, "implicit", Py_True);
     Py_DECREF(padding);
-    token = PyObject_Call(TagCloseSelfclose, NOARGS, kwargs);
-    Py_DECREF(kwargs);
-    if (!token)
+    if (Tokenizer_emit(self, TagCloseSelfclose, kwargs))
         return NULL;
-    if (Tokenizer_emit(self, token)) {
-        Py_DECREF(token);
-        return NULL;
-    }
-    Py_DECREF(token);
     self->head--;  // Offset displacement done by handle_tag_close_open
     return Tokenizer_pop(self);
 }
@@ -1759,7 +1736,7 @@ static int Tokenizer_parse_tag(Tokenizer* self)
 static int Tokenizer_emit_style_tag(Tokenizer* self, const char* tag,
                                     const char* ticks, PyObject* body)
 {
-    PyObject *markup, *kwargs, *token;
+    PyObject *markup, *kwargs;
 
     markup = PyBytes_FromString(ticks);
     if (!markup)
@@ -1771,17 +1748,8 @@ static int Tokenizer_emit_style_tag(Tokenizer* self, const char* tag,
     }
     PyDict_SetItemString(kwargs, "wiki_markup", markup);
     Py_DECREF(markup);
-    token = PyObject_Call(TagOpenOpen, NOARGS, kwargs);
-    if (!token) {
-        Py_DECREF(kwargs);
-        return -1;
-    }
-    Py_DECREF(kwargs);
-    if (Tokenizer_emit(self, token)) {
-        Py_DECREF(token);
+    if (Tokenizer_emit(self, TagOpenOpen, kwargs))
         return -1;
-    }
-    Py_DECREF(token);
     if (Tokenizer_emit_text(self, tag))
         return -1;
     if (Tokenizer_emit_FAST(self, TagCloseOpen))
@@ -1988,7 +1956,7 @@ static PyObject* Tokenizer_parse_style(Tokenizer* self)
 */
 static int Tokenizer_handle_list_marker(Tokenizer* self)
 {
-    PyObject *markup = Tokenizer_read(self, 0), *kwargs, *token;
+    PyObject *markup = Tokenizer_read(self, 0), *kwargs;
     Py_UNICODE code = *PyUnicode_AS_UNICODE(markup);
 
     if (code == *";")
@@ -1997,17 +1965,8 @@ static int Tokenizer_handle_list_marker(Tokenizer* self)
     if (!kwargs)
         return -1;
     PyDict_SetItemString(kwargs, "wiki_markup", markup);
-    token = PyObject_Call(TagOpenOpen, NOARGS, kwargs);
-    if (!token) {
-        Py_DECREF(kwargs);
-        return -1;
-    }
-    Py_DECREF(kwargs);
-    if (Tokenizer_emit(self, token)) {
-        Py_DECREF(token);
+    if (Tokenizer_emit(self, TagOpenOpen, kwargs))
         return -1;
-    }
-    Py_DECREF(token);
     if (Tokenizer_emit_text(self, GET_HTML_TAG(code)))
         return -1;
     if (Tokenizer_emit_FAST(self, TagCloseSelfclose))
@@ -2039,7 +1998,7 @@ static int Tokenizer_handle_list(Tokenizer* self)
 */
 static int Tokenizer_handle_hr(Tokenizer* self)
 {
-    PyObject *markup, *kwargs, *token;
+    PyObject *markup, *kwargs;
     Textbuffer *buffer = Textbuffer_new();
     int i;
 
@@ -2064,17 +2023,8 @@ static int Tokenizer_handle_hr(Tokenizer* self)
         return -1;
     PyDict_SetItemString(kwargs, "wiki_markup", markup);
     Py_DECREF(markup);
-    token = PyObject_Call(TagOpenOpen, NOARGS, kwargs);
-    if (!token) {
-        Py_DECREF(kwargs);
-        return -1;
-    }
-    Py_DECREF(kwargs);
-    if (Tokenizer_emit(self, token)) {
-        Py_DECREF(token);
+    if (Tokenizer_emit(self, TagOpenOpen, kwargs))
         return -1;
-    }
-    Py_DECREF(token);
     if (Tokenizer_emit_text(self, "hr"))
         return -1;
     if (Tokenizer_emit_FAST(self, TagCloseSelfclose))

From 95efa7dde9022c7582bbff44f86ff710f6985f27 Mon Sep 17 00:00:00 2001
From: Ben Kurtovic <ben.kurtovic@verizon.net>
Date: Wed, 14 Aug 2013 18:13:02 -0400
Subject: [PATCH 05/12] emit_FAST() -> emit(); emit() -> emit_kwargs()

---
 mwparserfromhell/parser/tokenizer.c | 65 +++++++++++++++++++------------------
 1 file changed, 33 insertions(+), 32 deletions(-)

diff --git a/mwparserfromhell/parser/tokenizer.c b/mwparserfromhell/parser/tokenizer.c
index 1ddb76b..cf11462 100644
--- a/mwparserfromhell/parser/tokenizer.c
+++ b/mwparserfromhell/parser/tokenizer.c
@@ -347,7 +347,7 @@ static void* Tokenizer_fail_route(Tokenizer* self)
 /*
     Write a token to the end of the current token stack.
 */
-static int Tokenizer_emit_FAST(Tokenizer* self, PyObject* token)
+static int Tokenizer_emit(Tokenizer* self, PyObject* token)
 {
     PyObject* instance;
 
@@ -367,7 +367,8 @@ static int Tokenizer_emit_FAST(Tokenizer* self, PyObject* token)
 /*
     Write a token to the end of the current token stack.
 */
-static int Tokenizer_emit(Tokenizer* self, PyObject* token, PyObject* kwargs)
+static int Tokenizer_emit_kwargs(Tokenizer* self, PyObject* token,
+                                 PyObject* kwargs)
 {
     PyObject* instance;
 
@@ -592,7 +593,7 @@ static int Tokenizer_parse_template(Tokenizer* self)
         return -1;
     }
     Py_DECREF(template);
-    if (Tokenizer_emit_FAST(self, TemplateClose))
+    if (Tokenizer_emit(self, TemplateClose))
         return -1;
     return 0;
 }
@@ -621,7 +622,7 @@ static int Tokenizer_parse_argument(Tokenizer* self)
         return -1;
     }
     Py_DECREF(argument);
-    if (Tokenizer_emit_FAST(self, ArgumentClose))
+    if (Tokenizer_emit(self, ArgumentClose))
         return -1;
     return 0;
 }
@@ -720,7 +721,7 @@ static int Tokenizer_handle_template_param(Tokenizer* self)
     }
     else
         self->topstack->context |= LC_TEMPLATE_PARAM_KEY;
-    if (Tokenizer_emit_FAST(self, TemplateParamSeparator))
+    if (Tokenizer_emit(self, TemplateParamSeparator))
         return -1;
     if (Tokenizer_push(self, self->topstack->context))
         return -1;
@@ -744,7 +745,7 @@ static int Tokenizer_handle_template_param_value(Tokenizer* self)
     Py_DECREF(stack);
     self->topstack->context ^= LC_TEMPLATE_PARAM_KEY;
     self->topstack->context |= LC_TEMPLATE_PARAM_VALUE;
-    if (Tokenizer_emit_FAST(self, TemplateParamEquals))
+    if (Tokenizer_emit(self, TemplateParamEquals))
         return -1;
     return 0;
 }
@@ -778,7 +779,7 @@ static int Tokenizer_handle_argument_separator(Tokenizer* self)
 {
     self->topstack->context ^= LC_ARGUMENT_NAME;
     self->topstack->context |= LC_ARGUMENT_DEFAULT;
-    if (Tokenizer_emit_FAST(self, ArgumentSeparator))
+    if (Tokenizer_emit(self, ArgumentSeparator))
         return -1;
     return 0;
 }
@@ -814,7 +815,7 @@ static int Tokenizer_parse_wikilink(Tokenizer* self)
     }
     if (!wikilink)
         return -1;
-    if (Tokenizer_emit_FAST(self, WikilinkOpen)) {
+    if (Tokenizer_emit(self, WikilinkOpen)) {
         Py_DECREF(wikilink);
         return -1;
     }
@@ -823,7 +824,7 @@ static int Tokenizer_parse_wikilink(Tokenizer* self)
         return -1;
     }
     Py_DECREF(wikilink);
-    if (Tokenizer_emit_FAST(self, WikilinkClose))
+    if (Tokenizer_emit(self, WikilinkClose))
         return -1;
     if (self->topstack->context & LC_FAIL_NEXT)
         self->topstack->context ^= LC_FAIL_NEXT;
@@ -837,7 +838,7 @@ static int Tokenizer_handle_wikilink_separator(Tokenizer* self)
 {
     self->topstack->context ^= LC_WIKILINK_TITLE;
     self->topstack->context |= LC_WIKILINK_TEXT;
-    if (Tokenizer_emit_FAST(self, WikilinkSeparator))
+    if (Tokenizer_emit(self, WikilinkSeparator))
         return -1;
     return 0;
 }
@@ -895,7 +896,7 @@ static int Tokenizer_parse_heading(Tokenizer* self)
     }
     PyDict_SetItemString(kwargs, "level", level);
     Py_DECREF(level);
-    if (Tokenizer_emit(self, HeadingStart, kwargs)) {
+    if (Tokenizer_emit_kwargs(self, HeadingStart, kwargs)) {
         Py_DECREF(heading->title);
         free(heading);
         return -1;
@@ -917,7 +918,7 @@ static int Tokenizer_parse_heading(Tokenizer* self)
     }
     Py_DECREF(heading->title);
     free(heading);
-    if (Tokenizer_emit_FAST(self, HeadingEnd))
+    if (Tokenizer_emit(self, HeadingEnd))
         return -1;
     self->global ^= GL_HEADING;
     return 0;
@@ -1000,7 +1001,7 @@ static int Tokenizer_really_parse_entity(Tokenizer* self)
         return 0;                   \
     }
 
-    if (Tokenizer_emit_FAST(self, HTMLEntityStart))
+    if (Tokenizer_emit(self, HTMLEntityStart))
         return -1;
     self->head++;
     this = Tokenizer_READ(self, 0);
@@ -1010,7 +1011,7 @@ static int Tokenizer_really_parse_entity(Tokenizer* self)
     }
     if (this == *"#") {
         numeric = 1;
-        if (Tokenizer_emit_FAST(self, HTMLEntityNumeric))
+        if (Tokenizer_emit(self, HTMLEntityNumeric))
             return -1;
         self->head++;
         this = Tokenizer_READ(self, 0);
@@ -1024,7 +1025,7 @@ static int Tokenizer_really_parse_entity(Tokenizer* self)
             if (!kwargs)
                 return -1;
             PyDict_SetItemString(kwargs, "char", Tokenizer_read(self, 0));
-            if (Tokenizer_emit(self, HTMLEntityHex, kwargs))
+            if (Tokenizer_emit_kwargs(self, HTMLEntityHex, kwargs))
                 return -1;
             self->head++;
         }
@@ -1118,9 +1119,9 @@ static int Tokenizer_really_parse_entity(Tokenizer* self)
     }
     PyDict_SetItemString(kwargs, "text", textobj);
     Py_DECREF(textobj);
-    if (Tokenizer_emit(self, Text, kwargs))
+    if (Tokenizer_emit_kwargs(self, Text, kwargs))
         return -1;
-    if (Tokenizer_emit_FAST(self, HTMLEntityEnd))
+    if (Tokenizer_emit(self, HTMLEntityEnd))
         return -1;
     return 0;
 }
@@ -1179,7 +1180,7 @@ static int Tokenizer_parse_comment(Tokenizer* self)
                             Tokenizer_READ(self, 2) == *">") {
             if (Tokenizer_emit_first(self, CommentStart))
                 return -1;
-            if (Tokenizer_emit_FAST(self, CommentEnd))
+            if (Tokenizer_emit(self, CommentEnd))
                 return -1;
             comment = Tokenizer_pop(self);
             if (!comment)
@@ -1352,7 +1353,7 @@ Tokenizer_handle_tag_data(Tokenizer* self, TagData* data, Py_UNICODE chunk)
     else if (data->context & TAG_ATTR_NAME) {
         if (chunk == *"=") {
             data->context = TAG_ATTR_VALUE | TAG_NOTE_QUOTE;
-            if (Tokenizer_emit_FAST(self, TagAttrEquals))
+            if (Tokenizer_emit(self, TagAttrEquals))
                 return -1;
             return 0;
         }
@@ -1409,7 +1410,7 @@ Tokenizer_handle_tag_close_open(Tokenizer* self, TagData* data, PyObject* cls)
     }
     PyDict_SetItemString(kwargs, "padding", padding);
     Py_DECREF(padding);
-    if (Tokenizer_emit(self, cls, kwargs))
+    if (Tokenizer_emit_kwargs(self, cls, kwargs))
         return -1;
     self->head++;
     return 0;
@@ -1420,7 +1421,7 @@ Tokenizer_handle_tag_close_open(Tokenizer* self, TagData* data, PyObject* cls)
 */
 static int Tokenizer_handle_tag_open_close(Tokenizer* self)
 {
-    if (Tokenizer_emit_FAST(self, TagOpenClose))
+    if (Tokenizer_emit(self, TagOpenClose))
         return -1;
     if (Tokenizer_push(self, LC_TAG_CLOSE))
         return -1;
@@ -1474,7 +1475,7 @@ static PyObject* Tokenizer_handle_tag_close_close(Tokenizer* self)
         return NULL;
     }
     Py_DECREF(closing);
-    if (Tokenizer_emit_FAST(self, TagCloseClose))
+    if (Tokenizer_emit(self, TagCloseClose))
         return NULL;
     return Tokenizer_pop(self);
 }
@@ -1524,7 +1525,7 @@ static PyObject* Tokenizer_handle_single_only_tag_end(Tokenizer* self)
     PyDict_SetItemString(kwargs, "padding", padding);
     PyDict_SetItemString(kwargs, "implicit", Py_True);
     Py_DECREF(padding);
-    if (Tokenizer_emit(self, TagCloseSelfclose, kwargs))
+    if (Tokenizer_emit_kwargs(self, TagCloseSelfclose, kwargs))
         return NULL;
     self->head--;  // Offset displacement done by handle_tag_close_open
     return Tokenizer_pop(self);
@@ -1588,7 +1589,7 @@ static PyObject* Tokenizer_really_parse_tag(Tokenizer* self)
         TagData_dealloc(data);
         return NULL;
     }
-    if (Tokenizer_emit_FAST(self, TagOpenOpen)) {
+    if (Tokenizer_emit(self, TagOpenOpen)) {
         TagData_dealloc(data);
         return NULL;
     }
@@ -1748,20 +1749,20 @@ static int Tokenizer_emit_style_tag(Tokenizer* self, const char* tag,
     }
     PyDict_SetItemString(kwargs, "wiki_markup", markup);
     Py_DECREF(markup);
-    if (Tokenizer_emit(self, TagOpenOpen, kwargs))
+    if (Tokenizer_emit_kwargs(self, TagOpenOpen, kwargs))
         return -1;
     if (Tokenizer_emit_text(self, tag))
         return -1;
-    if (Tokenizer_emit_FAST(self, TagCloseOpen))
+    if (Tokenizer_emit(self, TagCloseOpen))
         return -1;
     if (Tokenizer_emit_all(self, body))
         return -1;
     Py_DECREF(body);
-    if (Tokenizer_emit_FAST(self, TagOpenClose))
+    if (Tokenizer_emit(self, TagOpenClose))
         return -1;
     if (Tokenizer_emit_text(self, tag))
         return -1;
-    if (Tokenizer_emit_FAST(self, TagCloseClose))
+    if (Tokenizer_emit(self, TagCloseClose))
         return -1;
     return 0;
 }
@@ -1965,11 +1966,11 @@ static int Tokenizer_handle_list_marker(Tokenizer* self)
     if (!kwargs)
         return -1;
     PyDict_SetItemString(kwargs, "wiki_markup", markup);
-    if (Tokenizer_emit(self, TagOpenOpen, kwargs))
+    if (Tokenizer_emit_kwargs(self, TagOpenOpen, kwargs))
         return -1;
     if (Tokenizer_emit_text(self, GET_HTML_TAG(code)))
         return -1;
-    if (Tokenizer_emit_FAST(self, TagCloseSelfclose))
+    if (Tokenizer_emit(self, TagCloseSelfclose))
         return -1;
     return 0;
 }
@@ -2023,11 +2024,11 @@ static int Tokenizer_handle_hr(Tokenizer* self)
         return -1;
     PyDict_SetItemString(kwargs, "wiki_markup", markup);
     Py_DECREF(markup);
-    if (Tokenizer_emit(self, TagOpenOpen, kwargs))
+    if (Tokenizer_emit_kwargs(self, TagOpenOpen, kwargs))
         return -1;
     if (Tokenizer_emit_text(self, "hr"))
         return -1;
-    if (Tokenizer_emit_FAST(self, TagCloseSelfclose))
+    if (Tokenizer_emit(self, TagCloseSelfclose))
         return -1;
     return 0;
 }

From ebf99d722c68d0a9e91f312fefcbce856519d159 Mon Sep 17 00:00:00 2001
From: Ben Kurtovic <ben.kurtovic@verizon.net>
Date: Wed, 14 Aug 2013 19:01:12 -0400
Subject: [PATCH 06/12] Combine emit()/emit_first() internally.

---
 mwparserfromhell/parser/tokenizer.c | 60 +++++--------------------------------
 mwparserfromhell/parser/tokenizer.h |  7 ++++-
 2 files changed, 13 insertions(+), 54 deletions(-)

diff --git a/mwparserfromhell/parser/tokenizer.c b/mwparserfromhell/parser/tokenizer.c
index cf11462..4df61d8 100644
--- a/mwparserfromhell/parser/tokenizer.c
+++ b/mwparserfromhell/parser/tokenizer.c
@@ -347,7 +347,7 @@ static void* Tokenizer_fail_route(Tokenizer* self)
 /*
     Write a token to the end of the current token stack.
 */
-static int Tokenizer_emit(Tokenizer* self, PyObject* token)
+static int Tokenizer_emit_token(Tokenizer* self, PyObject* token, int first)
 {
     PyObject* instance;
 
@@ -356,7 +356,8 @@ static int Tokenizer_emit(Tokenizer* self, PyObject* token)
     instance = PyObject_CallObject(token, NULL);
     if (!instance)
         return -1;
-    if (PyList_Append(self->topstack->stack, instance)) {
+    if (first ? PyList_Insert(self->topstack->stack, 0, instance) :
+                PyList_Append(self->topstack->stack, instance)) {
         Py_DECREF(instance);
         return -1;
     }
@@ -367,8 +368,8 @@ static int Tokenizer_emit(Tokenizer* self, PyObject* token)
 /*
     Write a token to the end of the current token stack.
 */
-static int Tokenizer_emit_kwargs(Tokenizer* self, PyObject* token,
-                                 PyObject* kwargs)
+static int Tokenizer_emit_token_kwargs(Tokenizer* self, PyObject* token,
+                                       PyObject* kwargs, int first)
 {
     PyObject* instance;
 
@@ -381,55 +382,8 @@ static int Tokenizer_emit_kwargs(Tokenizer* self, PyObject* token,
         Py_DECREF(kwargs);
         return -1;
     }
-    if (PyList_Append(self->topstack->stack, instance)) {
-        Py_DECREF(instance);
-        Py_DECREF(kwargs);
-        return -1;
-    }
-    Py_DECREF(instance);
-    Py_DECREF(kwargs);
-    return 0;
-}
-
-/*
-    Write a token to the beginning of the current token stack.
-*/
-static int Tokenizer_emit_first(Tokenizer* self, PyObject* token)
-{
-    PyObject* instance;
-
-    if (Tokenizer_push_textbuffer(self))
-        return -1;
-    instance = PyObject_CallObject(token, NULL);
-    if (!instance)
-        return -1;
-    if (PyList_Insert(self->topstack->stack, 0, instance)) {
-        Py_DECREF(instance);
-        return -1;
-    }
-    Py_DECREF(instance);
-    return 0;
-}
-
-/*
-    Write a token to the beginning of the current token stack, with kwargs.
-    Steals a reference to kwargs.
-*/
-static int Tokenizer_emit_first_kwargs(Tokenizer* self, PyObject* token,
-                                       PyObject* kwargs)
-{
-    PyObject* instance;
-
-    if (Tokenizer_push_textbuffer(self)) {
-        Py_DECREF(kwargs);
-        return -1;
-    }
-    instance = PyObject_Call(token, NOARGS, kwargs);
-    if (!instance) {
-        Py_DECREF(kwargs);
-        return -1;
-    }
-    if (PyList_Insert(self->topstack->stack, 0, instance)) {
+    if (first ? PyList_Insert(self->topstack->stack, 0, instance):
+                PyList_Append(self->topstack->stack, instance)) {
         Py_DECREF(instance);
         Py_DECREF(kwargs);
         return -1;
diff --git a/mwparserfromhell/parser/tokenizer.h b/mwparserfromhell/parser/tokenizer.h
index 1ac9168..d5f755d 100644
--- a/mwparserfromhell/parser/tokenizer.h
+++ b/mwparserfromhell/parser/tokenizer.h
@@ -214,13 +214,18 @@ typedef struct {
 } Tokenizer;
 
 
-/* Macros for accessing Tokenizer data: */
+/* Macros related to Tokenizer functions: */
 
 #define Tokenizer_READ(self, delta) (*PyUnicode_AS_UNICODE(Tokenizer_read(self, delta)))
 #define Tokenizer_READ_BACKWARDS(self, delta) \
                 (*PyUnicode_AS_UNICODE(Tokenizer_read_backwards(self, delta)))
 #define Tokenizer_CAN_RECURSE(self) (self->depth < MAX_DEPTH && self->cycles < MAX_CYCLES)
 
+#define Tokenizer_emit(self, token) Tokenizer_emit_token(self, token, 0)
+#define Tokenizer_emit_first(self, token) Tokenizer_emit_token(self, token, 1)
+#define Tokenizer_emit_kwargs(self, token, kwargs) Tokenizer_emit_token_kwargs(self, token, kwargs, 0)
+#define Tokenizer_emit_first_kwargs(self, token, kwargs) Tokenizer_emit_token_kwargs(self, token, kwargs, 1)
+
 
 /* Macros for accessing HTML tag definitions: */
 

From 25d53cacf8abc76a55cbf1af1b77b4cb9b6b0f5c Mon Sep 17 00:00:00 2001
From: Ben Kurtovic <ben.kurtovic@verizon.net>
Date: Wed, 14 Aug 2013 23:54:06 -0400
Subject: [PATCH 07/12] Begin porting C tokenizer to Python 3.

---
 mwparserfromhell/parser/tokenizer.c | 70 ++++++++++++++++++++++++++-----------
 mwparserfromhell/parser/tokenizer.h | 27 +++++++-------
 setup.py                            |  5 +--
 3 files changed, 65 insertions(+), 37 deletions(-)

diff --git a/mwparserfromhell/parser/tokenizer.c b/mwparserfromhell/parser/tokenizer.c
index 4df61d8..60223e1 100644
--- a/mwparserfromhell/parser/tokenizer.c
+++ b/mwparserfromhell/parser/tokenizer.c
@@ -207,7 +207,7 @@ static void Tokenizer_dealloc(Tokenizer* self)
         free(this);
         this = next;
     }
-    self->ob_type->tp_free((PyObject*) self);
+    Py_TYPE(self)->tp_free((PyObject*) self);
 }
 
 static int Tokenizer_init(Tokenizer* self, PyObject* args, PyObject* kwds)
@@ -835,7 +835,11 @@ static int Tokenizer_parse_heading(Tokenizer* self)
         self->global ^= GL_HEADING;
         return 0;
     }
+#ifdef IS_PY3K
+    level = PyLong_FromSsize_t(heading->level);
+#else
     level = PyInt_FromSsize_t(heading->level);
+#endif
     if (!level) {
         Py_DECREF(heading->title);
         free(heading);
@@ -2299,30 +2303,40 @@ static PyObject* Tokenizer_tokenize(Tokenizer* self, PyObject* args)
     return Tokenizer_parse(self, 0, 1);
 }
 
-static void load_entitydefs(void)
+static int load_entitydefs(void)
 {
     PyObject *tempmod, *defmap, *deflist;
     unsigned numdefs, i;
 
+#ifdef IS_PY3K
+    tempmod = PyImport_ImportModule("html.entities");
+#else
     tempmod = PyImport_ImportModule("htmlentitydefs");
+#endif
     if (!tempmod)
-        return;
+        return -1;
     defmap = PyObject_GetAttrString(tempmod, "entitydefs");
     if (!defmap)
-        return;
+        return -1;
     Py_DECREF(tempmod);
     deflist = PyDict_Keys(defmap);
     if (!deflist)
-        return;
+        return -1;
     Py_DECREF(defmap);
     numdefs = (unsigned) PyList_GET_SIZE(defmap);
     entitydefs = calloc(numdefs + 1, sizeof(char*));
-    for (i = 0; i < numdefs; i++)
+    if (!entitydefs)
+        return -1;
+    for (i = 0; i < numdefs; i++) {
         entitydefs[i] = PyBytes_AsString(PyList_GET_ITEM(deflist, i));
+        if (!entitydefs[i])
+            return -1;
+    }
     Py_DECREF(deflist);
+    return 0;
 }
 
-static void load_tokens(void)
+static int load_tokens(void)
 {
     PyObject *tempmod, *tokens,
              *globals = PyEval_GetGlobals(),
@@ -2332,12 +2346,12 @@ static void load_tokens(void)
     char *name = "mwparserfromhell.parser";
 
     if (!fromlist || !modname)
-        return;
+        return -1;
     PyList_SET_ITEM(fromlist, 0, modname);
     tempmod = PyImport_ImportModuleLevel(name, globals, locals, fromlist, 0);
     Py_DECREF(fromlist);
     if (!tempmod)
-        return;
+        return -1;
     tokens = PyObject_GetAttrString(tempmod, "tokens");
     Py_DECREF(tempmod);
 
@@ -2379,9 +2393,10 @@ static void load_tokens(void)
     TagCloseClose = PyObject_GetAttrString(tokens, "TagCloseClose");
 
     Py_DECREF(tokens);
+    return 0;
 }
 
-static void load_tag_defs(void)
+static int load_tag_defs(void)
 {
     PyObject *tempmod,
              *globals = PyEval_GetGlobals(),
@@ -2391,33 +2406,48 @@ static void load_tag_defs(void)
     char *name = "mwparserfromhell";
 
     if (!fromlist || !modname)
-        return;
+        return -1;
     PyList_SET_ITEM(fromlist, 0, modname);
     tempmod = PyImport_ImportModuleLevel(name, globals, locals, fromlist, 0);
     Py_DECREF(fromlist);
     if (!tempmod)
-        return;
+        return -1;
     tag_defs = PyObject_GetAttrString(tempmod, "tag_defs");
     Py_DECREF(tempmod);
+    return 0;
 }
 
-PyMODINIT_FUNC init_tokenizer(void)
+#ifdef IS_PY3K
+    #define INIT_ERROR return NULL
+    PyMODINIT_FUNC PyInit__tokenizer(void)
+#else
+    #define INIT_ERROR return
+    PyMODINIT_FUNC init_tokenizer(void)
+#endif
 {
     PyObject *module;
 
     TokenizerType.tp_new = PyType_GenericNew;
     if (PyType_Ready(&TokenizerType) < 0)
-        return;
-    module = Py_InitModule("_tokenizer", module_methods);
+        INIT_ERROR;
+#ifdef IS_PY3K
+    module = PyModule_Create(&module_def);
+#else
+    module = Py_InitModule("_tokenizer", NULL);
+#endif
+    if (!module)
+        INIT_ERROR;
     Py_INCREF(&TokenizerType);
     PyModule_AddObject(module, "CTokenizer", (PyObject*) &TokenizerType);
     Py_INCREF(Py_True);
     PyDict_SetItemString(TokenizerType.tp_dict, "USES_C", Py_True);
-
     EMPTY = PyUnicode_FromString("");
     NOARGS = PyTuple_New(0);
-
-    load_entitydefs();
-    load_tokens();
-    load_tag_defs();
+    if (!EMPTY || !NOARGS)
+        INIT_ERROR;
+    if (load_entitydefs() || load_tokens() || load_tag_defs())
+        INIT_ERROR;
+#ifdef IS_PY3K
+    return module;
+#endif
 }
diff --git a/mwparserfromhell/parser/tokenizer.h b/mwparserfromhell/parser/tokenizer.h
index d5f755d..2bf6973 100644
--- a/mwparserfromhell/parser/tokenizer.h
+++ b/mwparserfromhell/parser/tokenizer.h
@@ -28,6 +28,7 @@ SOFTWARE.
 #include <Python.h>
 #include <math.h>
 #include <structmember.h>
+#include <bytesobject.h>
 
 #if PY_MAJOR_VERSION >= 3
 #define IS_PY3K
@@ -253,27 +254,18 @@ static PyObject* Tokenizer_tokenize(Tokenizer*, PyObject*);
 
 /* More structs for creating the Tokenizer type: */
 
-static PyMethodDef
-Tokenizer_methods[] = {
+static PyMethodDef Tokenizer_methods[] = {
     {"tokenize", (PyCFunction) Tokenizer_tokenize, METH_VARARGS,
     "Build a list of tokens from a string of wikicode and return it."},
     {NULL}
 };
 
-static PyMemberDef
-Tokenizer_members[] = {
+static PyMemberDef Tokenizer_members[] = {
     {NULL}
 };
 
-static PyMethodDef
-module_methods[] = {
-    {NULL}
-};
-
-static PyTypeObject
-TokenizerType = {
-    PyObject_HEAD_INIT(NULL)
-    0,                                                      /* ob_size */
+static PyTypeObject TokenizerType = {
+    PyVarObject_HEAD_INIT(NULL, 0)
     "_tokenizer.CTokenizer",                                /* tp_name */
     sizeof(Tokenizer),                                      /* tp_basicsize */
     0,                                                      /* tp_itemsize */
@@ -312,3 +304,12 @@ TokenizerType = {
     0,                                                      /* tp_alloc */
     Tokenizer_new,                                          /* tp_new */
 };
+
+#ifdef IS_PY3K
+static PyModuleDef module_def = {
+    PyModuleDef_HEAD_INIT,
+    "_tokenizer",
+    "Creates a list of tokens from a string of wikicode.",
+    -1, NULL, NULL, NULL, NULL, NULL
+};
+#endif
diff --git a/setup.py b/setup.py
index 8b4ae86..5e6d779 100644
--- a/setup.py
+++ b/setup.py
@@ -29,16 +29,13 @@ from mwparserfromhell.compat import py3k
 with open("README.rst") as fp:
     long_docs = fp.read()
 
-# builder = Extension("mwparserfromhell.parser._builder",
-#                     sources = ["mwparserfromhell/parser/builder.c"])
-
 tokenizer = Extension("mwparserfromhell.parser._tokenizer",
                       sources = ["mwparserfromhell/parser/tokenizer.c"])
 
 setup(
     name = "mwparserfromhell",
     packages = find_packages(exclude=("tests",)),
-    ext_modules = [] if py3k else [tokenizer],
+    ext_modules = [tokenizer],
     test_suite = "tests",
     version = __version__,
     author = "Ben Kurtovic",

From e02ad8239f16d78e216e8f809ee162c8213e2287 Mon Sep 17 00:00:00 2001
From: Ben Kurtovic <ben.kurtovic@verizon.net>
Date: Thu, 15 Aug 2013 00:11:42 -0400
Subject: [PATCH 08/12] Make load_entitydefs() work on Python 3.

---
 mwparserfromhell/parser/tokenizer.c | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/mwparserfromhell/parser/tokenizer.c b/mwparserfromhell/parser/tokenizer.c
index 60223e1..b3ad3ec 100644
--- a/mwparserfromhell/parser/tokenizer.c
+++ b/mwparserfromhell/parser/tokenizer.c
@@ -2307,6 +2307,9 @@ static int load_entitydefs(void)
 {
     PyObject *tempmod, *defmap, *deflist;
     unsigned numdefs, i;
+#ifdef IS_PY3K
+    PyObject *string;
+#endif
 
 #ifdef IS_PY3K
     tempmod = PyImport_ImportModule("html.entities");
@@ -2328,7 +2331,15 @@ static int load_entitydefs(void)
     if (!entitydefs)
         return -1;
     for (i = 0; i < numdefs; i++) {
+#ifdef IS_PY3K
+        string = PyUnicode_AsASCIIString(PyList_GET_ITEM(deflist, i));
+        if (!string)
+            return -1;
+        entitydefs[i] = PyBytes_AsString(string);
+        Py_DECREF(string);
+#else
         entitydefs[i] = PyBytes_AsString(PyList_GET_ITEM(deflist, i));
+#endif
         if (!entitydefs[i])
             return -1;
     }

From b5ec7f3bebc71ba161c7a1533032abb5cb6211f0 Mon Sep 17 00:00:00 2001
From: Ben Kurtovic <ben.kurtovic@verizon.net>
Date: Thu, 15 Aug 2013 00:33:22 -0400
Subject: [PATCH 09/12] Fix py3k module importing; stick a bunch of macros in
 one place.

---
 mwparserfromhell/parser/tokenizer.c | 24 +++++-------------------
 mwparserfromhell/parser/tokenizer.h | 17 +++++++++++++++++
 2 files changed, 22 insertions(+), 19 deletions(-)

diff --git a/mwparserfromhell/parser/tokenizer.c b/mwparserfromhell/parser/tokenizer.c
index b3ad3ec..f6eea84 100644
--- a/mwparserfromhell/parser/tokenizer.c
+++ b/mwparserfromhell/parser/tokenizer.c
@@ -2311,11 +2311,7 @@ static int load_entitydefs(void)
     PyObject *string;
 #endif
 
-#ifdef IS_PY3K
-    tempmod = PyImport_ImportModule("html.entities");
-#else
-    tempmod = PyImport_ImportModule("htmlentitydefs");
-#endif
+    tempmod = PyImport_ImportModule(ENTITYDEFS_MODULE);
     if (!tempmod)
         return -1;
     defmap = PyObject_GetAttrString(tempmod, "entitydefs");
@@ -2353,7 +2349,7 @@ static int load_tokens(void)
              *globals = PyEval_GetGlobals(),
              *locals = PyEval_GetLocals(),
              *fromlist = PyList_New(1),
-             *modname = PyBytes_FromString("tokens");
+             *modname = IMPORT_NAME_FUNC("tokens");
     char *name = "mwparserfromhell.parser";
 
     if (!fromlist || !modname)
@@ -2413,7 +2409,7 @@ static int load_tag_defs(void)
              *globals = PyEval_GetGlobals(),
              *locals = PyEval_GetLocals(),
              *fromlist = PyList_New(1),
-             *modname = PyBytes_FromString("tag_defs");
+             *modname = IMPORT_NAME_FUNC("tag_defs");
     char *name = "mwparserfromhell";
 
     if (!fromlist || !modname)
@@ -2428,24 +2424,14 @@ static int load_tag_defs(void)
     return 0;
 }
 
-#ifdef IS_PY3K
-    #define INIT_ERROR return NULL
-    PyMODINIT_FUNC PyInit__tokenizer(void)
-#else
-    #define INIT_ERROR return
-    PyMODINIT_FUNC init_tokenizer(void)
-#endif
+PyMODINIT_FUNC INIT_FUNC_NAME(void)
 {
     PyObject *module;
 
     TokenizerType.tp_new = PyType_GenericNew;
     if (PyType_Ready(&TokenizerType) < 0)
         INIT_ERROR;
-#ifdef IS_PY3K
-    module = PyModule_Create(&module_def);
-#else
-    module = Py_InitModule("_tokenizer", NULL);
-#endif
+    module = CREATE_MODULE;
     if (!module)
         INIT_ERROR;
     Py_INCREF(&TokenizerType);
diff --git a/mwparserfromhell/parser/tokenizer.h b/mwparserfromhell/parser/tokenizer.h
index 2bf6973..1229688 100644
--- a/mwparserfromhell/parser/tokenizer.h
+++ b/mwparserfromhell/parser/tokenizer.h
@@ -252,6 +252,23 @@ static PyObject* Tokenizer_parse(Tokenizer*, int, int);
 static PyObject* Tokenizer_tokenize(Tokenizer*, PyObject*);
 
 
+/* Macros for Python 2/3 compatibility: */
+
+#ifdef IS_PY3K
+    #define IMPORT_NAME_FUNC  PyUnicode_FromString
+    #define CREATE_MODULE     PyModule_Create(&module_def);
+    #define ENTITYDEFS_MODULE "html.entities"
+    #define INIT_FUNC_NAME    PyInit__tokenizer
+    #define INIT_ERROR        return NULL
+#else
+    #define IMPORT_NAME_FUNC  PyBytes_FromString
+    #define CREATE_MODULE     Py_InitModule("_tokenizer", NULL);
+    #define ENTITYDEFS_MODULE "htmlentitydefs"
+    #define INIT_FUNC_NAME    init_tokenizer
+    #define INIT_ERROR        return
+#endif
+
+
 /* More structs for creating the Tokenizer type: */
 
 static PyMethodDef Tokenizer_methods[] = {

From db86176c085063f1b5227dc3e99c003ef443f0fc Mon Sep 17 00:00:00 2001
From: Ben Kurtovic <ben.kurtovic@verizon.net>
Date: Thu, 15 Aug 2013 00:41:24 -0400
Subject: [PATCH 10/12] wiki_markup attr should be unicode, not bytes

---
 mwparserfromhell/parser/tokenizer.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/mwparserfromhell/parser/tokenizer.c b/mwparserfromhell/parser/tokenizer.c
index f6eea84..c08fb11 100644
--- a/mwparserfromhell/parser/tokenizer.c
+++ b/mwparserfromhell/parser/tokenizer.c
@@ -1697,7 +1697,7 @@ static int Tokenizer_emit_style_tag(Tokenizer* self, const char* tag,
 {
     PyObject *markup, *kwargs;
 
-    markup = PyBytes_FromString(ticks);
+    markup = PyUnicode_FromString(ticks);
     if (!markup)
         return -1;
     kwargs = PyDict_New();

From 5e8e050ca3421e20c0462df071ff83e8c47ff703 Mon Sep 17 00:00:00 2001
From: Ben Kurtovic <ben.kurtovic@verizon.net>
Date: Thu, 15 Aug 2013 00:53:04 -0400
Subject: [PATCH 11/12] A few tweaks; py3k support now complete.

---
 mwparserfromhell/parser/tokenizer.c | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/mwparserfromhell/parser/tokenizer.c b/mwparserfromhell/parser/tokenizer.c
index c08fb11..88ca3f2 100644
--- a/mwparserfromhell/parser/tokenizer.c
+++ b/mwparserfromhell/parser/tokenizer.c
@@ -1017,7 +1017,7 @@ static int Tokenizer_really_parse_entity(Tokenizer* self)
             self->head++;
             continue;
         }
-        if (i >= 8)
+        if (i >= MAX_ENTITY_SIZE)
             FAIL_ROUTE_AND_EXIT()
         for (j = 0; j < NUM_MARKERS; j++) {
             if (this == *MARKERS[j])
@@ -2332,7 +2332,6 @@ static int load_entitydefs(void)
         if (!string)
             return -1;
         entitydefs[i] = PyBytes_AsString(string);
-        Py_DECREF(string);
 #else
         entitydefs[i] = PyBytes_AsString(PyList_GET_ITEM(deflist, i));
 #endif

From 8923d96a57b35d3b911733719099107671ff84d6 Mon Sep 17 00:00:00 2001
From: Ben Kurtovic <ben.kurtovic@verizon.net>
Date: Thu, 15 Aug 2013 01:19:39 -0400
Subject: [PATCH 12/12] More unification.

---
 mwparserfromhell/parser/tokenizer.c | 16 ++++++----------
 mwparserfromhell/parser/tokenizer.h |  2 ++
 2 files changed, 8 insertions(+), 10 deletions(-)

diff --git a/mwparserfromhell/parser/tokenizer.c b/mwparserfromhell/parser/tokenizer.c
index 88ca3f2..8dace5a 100644
--- a/mwparserfromhell/parser/tokenizer.c
+++ b/mwparserfromhell/parser/tokenizer.c
@@ -835,11 +835,7 @@ static int Tokenizer_parse_heading(Tokenizer* self)
         self->global ^= GL_HEADING;
         return 0;
     }
-#ifdef IS_PY3K
-    level = PyLong_FromSsize_t(heading->level);
-#else
-    level = PyInt_FromSsize_t(heading->level);
-#endif
+    level = NEW_INT_FUNC(heading->level);
     if (!level) {
         Py_DECREF(heading->title);
         free(heading);
@@ -2279,7 +2275,11 @@ static PyObject* Tokenizer_tokenize(Tokenizer* self, PyObject* args)
 {
     PyObject *text, *temp;
 
-    if (!PyArg_ParseTuple(args, "U", &text)) {
+    if (PyArg_ParseTuple(args, "U", &text)) {
+        Py_XDECREF(self->text);
+        self->text = PySequence_Fast(text, "expected a sequence");
+    }
+    else {
         const char* encoded;
         Py_ssize_t size;
         /* Failed to parse a Unicode object; try a string instead. */
@@ -2294,10 +2294,6 @@ static PyObject* Tokenizer_tokenize(Tokenizer* self, PyObject* args)
         Py_XDECREF(temp);
         self->text = text;
     }
-    else {
-        Py_XDECREF(self->text);
-        self->text = PySequence_Fast(text, "expected a sequence");
-    }
     self->head = self->global = self->depth = self->cycles = 0;
     self->length = PyList_GET_SIZE(self->text);
     return Tokenizer_parse(self, 0, 1);
diff --git a/mwparserfromhell/parser/tokenizer.h b/mwparserfromhell/parser/tokenizer.h
index 1229688..264360e 100644
--- a/mwparserfromhell/parser/tokenizer.h
+++ b/mwparserfromhell/parser/tokenizer.h
@@ -255,12 +255,14 @@ static PyObject* Tokenizer_tokenize(Tokenizer*, PyObject*);
 /* Macros for Python 2/3 compatibility: */
 
 #ifdef IS_PY3K
+    #define NEW_INT_FUNC      PyLong_FromSsize_t
     #define IMPORT_NAME_FUNC  PyUnicode_FromString
     #define CREATE_MODULE     PyModule_Create(&module_def);
     #define ENTITYDEFS_MODULE "html.entities"
     #define INIT_FUNC_NAME    PyInit__tokenizer
     #define INIT_ERROR        return NULL
 #else
+    #define NEW_INT_FUNC      PyInt_FromSsize_t
     #define IMPORT_NAME_FUNC  PyBytes_FromString
     #define CREATE_MODULE     Py_InitModule("_tokenizer", NULL);
     #define ENTITYDEFS_MODULE "htmlentitydefs"