mwparserfromhell/mwparserfromhell/parser/tokenizer.c

/*
Tokenizer for MWParserFromHell
Copyright (C) 2012-2014 Ben Kurtovic <ben.kurtovic@gmail.com>

Permission is hereby granted, free of charge, to any person obtaining a copy of
this software and associated documentation files (the "Software"), to deal in
the Software without restriction, including without limitation the rights to
use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
of the Software, and to permit persons to whom the Software is furnished to do
so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
*/

#include "tokenizer.h"

/*
    Determine whether the given Py_UNICODE is a marker.
*/
static int is_marker(Py_UNICODE this)
{
    int i;

    for (i = 0; i < NUM_MARKERS; i++) {
        if (MARKERS[i] == this)
            return 1;
    }
    return 0;
}

/*
    Given a context, return the heading level encoded within it.
*/
static int heading_level_from_context(int n)
{
    int level;

    n /= LC_HEADING_LEVEL_1;
    for (level = 1; n > 1; n >>= 1)
        level++;
    return level;
}

/*
    Call the given function in definitions.py, using 'in1', 'in2', and 'in3' as
    parameters, and return its output as a bool.
*/
static int call_def_func(const char* funcname, PyObject* in1, PyObject* in2,
                         PyObject* in3)
{
    PyObject* func = PyObject_GetAttrString(definitions, funcname);
    PyObject* result = PyObject_CallFunctionObjArgs(func, in1, in2, in3, NULL);
    int ans = (result == Py_True) ? 1 : 0;

    Py_DECREF(func);
    Py_DECREF(result);
    return ans;
}

/*
    Sanitize the name of a tag so it can be compared with others for equality.
*/
static PyObject* strip_tag_name(PyObject* token)
{
    PyObject *text, *rstripped, *lowered;

    text = PyObject_GetAttrString(token, "text");
    if (!text)
        return NULL;
    rstripped = PyObject_CallMethod(text, "rstrip", NULL);
    Py_DECREF(text);
    if (!rstripped)
        return NULL;
    lowered = PyObject_CallMethod(rstripped, "lower", NULL);
    Py_DECREF(rstripped);
    return lowered;
}

static Textbuffer* Textbuffer_new(void)
{
    Textbuffer* buffer = malloc(sizeof(Textbuffer));

    if (!buffer) {
        PyErr_NoMemory();
        return NULL;
    }
    buffer->size = 0;
    buffer->data = malloc(sizeof(Py_UNICODE) * TEXTBUFFER_BLOCKSIZE);
    if (!buffer->data) {
        free(buffer);
        PyErr_NoMemory();
        return NULL;
    }
    buffer->prev = buffer->next = NULL;
    return buffer;
}

static void Textbuffer_dealloc(Textbuffer* self)
{
    Textbuffer* next;

    while (self) {
        free(self->data);
        next = self->next;
        free(self);
        self = next;
    }
}

/*
    Write a Unicode codepoint to the given textbuffer.
*/
static int Textbuffer_write(Textbuffer** this, Py_UNICODE code)
{
    Textbuffer* self = *this;

    if (self->size == TEXTBUFFER_BLOCKSIZE) {
        Textbuffer* new = Textbuffer_new();
        if (!new)
            return -1;
        new->next = self;
        self->prev = new;
        *this = self = new;
    }
    self->data[self->size++] = code;
    return 0;
}

/*
    Return the contents of the textbuffer as a Python Unicode object.
*/
static PyObject* Textbuffer_render(Textbuffer* self)
{
    PyObject *result = PyUnicode_FromUnicode(self->data, self->size);
    PyObject *left, *concat;

    while (self->next) {
        self = self->next;
        left = PyUnicode_FromUnicode(self->data, self->size);
        concat = PyUnicode_Concat(left, result);
        Py_DECREF(left);
        Py_DECREF(result);
        result = concat;
    }
    return result;
}

static TagData* TagData_new(void)
{
    TagData *self = malloc(sizeof(TagData));

    #define ALLOC_BUFFER(name)     \
        name = Textbuffer_new();   \
        if (!name) {               \
            TagData_dealloc(self); \
            return NULL;           \
        }

    if (!self) {
        PyErr_NoMemory();
        return NULL;
    }
    self->context = TAG_NAME;
    ALLOC_BUFFER(self->pad_first)
    ALLOC_BUFFER(self->pad_before_eq)
    ALLOC_BUFFER(self->pad_after_eq)
    self->reset = 0;
    return self;
}

static void TagData_dealloc(TagData* self)
{
    #define DEALLOC_BUFFER(name) \
        if (name)                \
            Textbuffer_dealloc(name);

    DEALLOC_BUFFER(self->pad_first);
    DEALLOC_BUFFER(self->pad_before_eq);
    DEALLOC_BUFFER(self->pad_after_eq);
    free(self);
}

static int TagData_reset_buffers(TagData* self)
{
    #define RESET_BUFFER(name)    \
        Textbuffer_dealloc(name); \
        name = Textbuffer_new();  \
        if (!name)                \
            return -1;

    RESET_BUFFER(self->pad_first)
    RESET_BUFFER(self->pad_before_eq)
    RESET_BUFFER(self->pad_after_eq)
    return 0;
}

static PyObject*
Tokenizer_new(PyTypeObject* type, PyObject* args, PyObject* kwds)
{
    Tokenizer* self = (Tokenizer*) type->tp_alloc(type, 0);
    return (PyObject*) self;
}

static void Tokenizer_dealloc(Tokenizer* self)
{
    Stack *this = self->topstack, *next;
    Py_XDECREF(self->text);

    while (this) {
        Py_DECREF(this->stack);
        Textbuffer_dealloc(this->textbuffer);
        next = this->next;
        free(this);
        this = next;
    }
    Py_TYPE(self)->tp_free((PyObject*) self);
}

static int Tokenizer_init(Tokenizer* self, PyObject* args, PyObject* kwds)
{
    static char* kwlist[] = {NULL};

    if (!PyArg_ParseTupleAndKeywords(args, kwds, "", kwlist))
        return -1;
    self->text = Py_None;
    Py_INCREF(Py_None);
    self->topstack = NULL;
    self->head = self->length = self->global = self->depth = self->cycles = 0;
    return 0;
}

/*
    Add a new token stack, context, and textbuffer to the list.
*/
static int Tokenizer_push(Tokenizer* self, int context)
{
    Stack* top = malloc(sizeof(Stack));

    if (!top) {
        PyErr_NoMemory();
        return -1;
    }
    top->stack = PyList_New(0);
    top->context = context;
    top->textbuffer = Textbuffer_new();
    if (!top->textbuffer)
        return -1;
    top->next = self->topstack;
    self->topstack = top;
    self->depth++;
    self->cycles++;
    return 0;
}

/*
    Push the textbuffer onto the stack as a Text node and clear it.
*/
static int Tokenizer_push_textbuffer(Tokenizer* self)
{
    PyObject *text, *kwargs, *token;
    Textbuffer* buffer = self->topstack->textbuffer;

    if (buffer->size == 0 && !buffer->next)
        return 0;
    text = Textbuffer_render(buffer);
    if (!text)
        return -1;
    kwargs = PyDict_New();
    if (!kwargs) {
        Py_DECREF(text);
        return -1;
    }
    PyDict_SetItemString(kwargs, "text", text);
    Py_DECREF(text);
    token = PyObject_Call(Text, NOARGS, kwargs);
    Py_DECREF(kwargs);
    if (!token)
        return -1;
    if (PyList_Append(self->topstack->stack, token)) {
        Py_DECREF(token);
        return -1;
    }
    Py_DECREF(token);
    Textbuffer_dealloc(buffer);
    self->topstack->textbuffer = Textbuffer_new();
    if (!self->topstack->textbuffer)
        return -1;
    return 0;
}

/*
    Pop and deallocate the top token stack/context/textbuffer.
*/
static void Tokenizer_delete_top_of_stack(Tokenizer* self)
{
    Stack* top = self->topstack;

    Py_DECREF(top->stack);
    Textbuffer_dealloc(top->textbuffer);
    self->topstack = top->next;
    free(top);
    self->depth--;
}

/*
    Pop the current stack/context/textbuffer, returing the stack.
*/
static PyObject* Tokenizer_pop(Tokenizer* self)
{
    PyObject* stack;

    if (Tokenizer_push_textbuffer(self))
        return NULL;
    stack = self->topstack->stack;
    Py_INCREF(stack);
    Tokenizer_delete_top_of_stack(self);
    return stack;
}

/*
    Pop the current stack/context/textbuffer, returing the stack. We will also
    replace the underlying stack's context with the current stack's.
*/
static PyObject* Tokenizer_pop_keeping_context(Tokenizer* self)
{
    PyObject* stack;
    int context;

    if (Tokenizer_push_textbuffer(self))
        return NULL;
    stack = self->topstack->stack;
    Py_INCREF(stack);
    context = self->topstack->context;
    Tokenizer_delete_top_of_stack(self);
    self->topstack->context = context;
    return stack;
}

/*
    Fail the current tokenization route. Discards the current
    stack/context/textbuffer and sets the BAD_ROUTE flag.
*/
static void* Tokenizer_fail_route(Tokenizer* self)
{
    int context = self->topstack->context;
    PyObject* stack = Tokenizer_pop(self);

    Py_XDECREF(stack);
    FAIL_ROUTE(context);
    return NULL;
}

/*
    Write a token to the current token stack.
*/
static int Tokenizer_emit_token(Tokenizer* self, PyObject* token, int first)
{
    PyObject* instance;

    if (Tokenizer_push_textbuffer(self))
        return -1;
    instance = PyObject_CallObject(token, NULL);
    if (!instance)
        return -1;
    if (first ? PyList_Insert(self->topstack->stack, 0, instance) :
                PyList_Append(self->topstack->stack, instance)) {
        Py_DECREF(instance);
        return -1;
    }
    Py_DECREF(instance);
    return 0;
}

/*
    Write a token to the current token stack, with kwargs. Steals a reference
    to kwargs.
*/
static int Tokenizer_emit_token_kwargs(Tokenizer* self, PyObject* token,
                                       PyObject* kwargs, int first)
{
    PyObject* instance;

    if (Tokenizer_push_textbuffer(self)) {
        Py_DECREF(kwargs);
        return -1;
    }
    instance = PyObject_Call(token, NOARGS, kwargs);
    if (!instance) {
        Py_DECREF(kwargs);
        return -1;
    }
    if (first ? PyList_Insert(self->topstack->stack, 0, instance):
                PyList_Append(self->topstack->stack, instance)) {
        Py_DECREF(instance);
        Py_DECREF(kwargs);
        return -1;
    }
    Py_DECREF(instance);
    Py_DECREF(kwargs);
    return 0;
}

/*
    Write a Unicode codepoint to the current textbuffer.
*/
static int Tokenizer_emit_char(Tokenizer* self, Py_UNICODE code)
{
    return Textbuffer_write(&(self->topstack->textbuffer), code);
}

/*
    Write a string of text to the current textbuffer.
*/
static int Tokenizer_emit_text(Tokenizer* self, const char* text)
{
    int i = 0;

    while (text[i]) {
        if (Tokenizer_emit_char(self, text[i]))
            return -1;
        i++;
    }
    return 0;
}

/*
    Write the contents of another textbuffer to the current textbuffer,
    deallocating it in the process.
*/
static int
Tokenizer_emit_textbuffer(Tokenizer* self, Textbuffer* buffer, int reverse)
{
    Textbuffer *original = buffer;
    long i;

    if (reverse) {
        do {
            for (i = buffer->size - 1; i >= 0; i--) {
                if (Tokenizer_emit_char(self, buffer->data[i])) {
                    Textbuffer_dealloc(original);
                    return -1;
                }
            }
        } while ((buffer = buffer->next));
    }
    else {
        while (buffer->next)
            buffer = buffer->next;
        do {
            for (i = 0; i < buffer->size; i++) {
                if (Tokenizer_emit_char(self, buffer->data[i])) {
                    Textbuffer_dealloc(original);
                    return -1;
                }
            }
        } while ((buffer = buffer->prev));
    }
    Textbuffer_dealloc(original);
    return 0;
}

/*
    Write a series of tokens to the current stack at once.
*/
static int Tokenizer_emit_all(Tokenizer* self, PyObject* tokenlist)
{
    int pushed = 0;
    PyObject *stack, *token, *left, *right, *text;
    Textbuffer* buffer;
    Py_ssize_t size;

    if (PyList_GET_SIZE(tokenlist) > 0) {
        token = PyList_GET_ITEM(tokenlist, 0);
        switch (PyObject_IsInstance(token, Text)) {
            case 0:
                break;
            case 1: {
                pushed = 1;
                buffer = self->topstack->textbuffer;
                if (buffer->size == 0 && !buffer->next)
                    break;
                left = Textbuffer_render(buffer);
                if (!left)
                    return -1;
                right = PyObject_GetAttrString(token, "text");
                if (!right)
                    return -1;
                text = PyUnicode_Concat(left, right);
                Py_DECREF(left);
                Py_DECREF(right);
                if (!text)
                    return -1;
                if (PyObject_SetAttrString(token, "text", text)) {
                    Py_DECREF(text);
                    return -1;
                }
                Py_DECREF(text);
                Textbuffer_dealloc(buffer);
                self->topstack->textbuffer = Textbuffer_new();
                if (!self->topstack->textbuffer)
                    return -1;
                break;
            }
            case -1:
                return -1;
        }
    }
    if (!pushed) {
        if (Tokenizer_push_textbuffer(self))
            return -1;
    }
    stack = self->topstack->stack;
    size = PyList_GET_SIZE(stack);
    if (PyList_SetSlice(stack, size, size, tokenlist))
        return -1;
    return 0;
}

/*
    Pop the current stack, write text, and then write the stack. 'text' is a
    NULL-terminated array of chars.
*/
static int Tokenizer_emit_text_then_stack(Tokenizer* self, const char* text)
{
    PyObject* stack = Tokenizer_pop(self);

    if (Tokenizer_emit_text(self, text)) {
        Py_DECREF(stack);
        return -1;
    }
    if (stack) {
        if (PyList_GET_SIZE(stack) > 0) {
            if (Tokenizer_emit_all(self, stack)) {
                Py_DECREF(stack);
                return -1;
            }
        }
        Py_DECREF(stack);
    }
    self->head--;
    return 0;
}

/*
    Read the value at a relative point in the wikicode, forwards.
*/
static PyObject* Tokenizer_read(Tokenizer* self, Py_ssize_t delta)
{
    Py_ssize_t index = self->head + delta;

    if (index >= self->length)
        return EMPTY;
    return PyList_GET_ITEM(self->text, index);
}

/*
    Read the value at a relative point in the wikicode, backwards.
*/
static PyObject* Tokenizer_read_backwards(Tokenizer* self, Py_ssize_t delta)
{
    Py_ssize_t index;

    if (delta > self->head)
        return EMPTY;
    index = self->head - delta;
    return PyList_GET_ITEM(self->text, index);
}

/*
    Parse a template at the head of the wikicode string.
*/
static int Tokenizer_parse_template(Tokenizer* self)
{
    PyObject *template;
    Py_ssize_t reset = self->head;

    template = Tokenizer_parse(self, LC_TEMPLATE_NAME, 1);
    if (BAD_ROUTE) {
        self->head = reset;
        return 0;
    }
    if (!template)
        return -1;
    if (Tokenizer_emit_first(self, TemplateOpen)) {
        Py_DECREF(template);
        return -1;
    }
    if (Tokenizer_emit_all(self, template)) {
        Py_DECREF(template);
        return -1;
    }
    Py_DECREF(template);
    if (Tokenizer_emit(self, TemplateClose))
        return -1;
    return 0;
}

/*
    Parse an argument at the head of the wikicode string.
*/
static int Tokenizer_parse_argument(Tokenizer* self)
{
    PyObject *argument;
    Py_ssize_t reset = self->head;

    argument = Tokenizer_parse(self, LC_ARGUMENT_NAME, 1);
    if (BAD_ROUTE) {
        self->head = reset;
        return 0;
    }
    if (!argument)
        return -1;
    if (Tokenizer_emit_first(self, ArgumentOpen)) {
        Py_DECREF(argument);
        return -1;
    }
    if (Tokenizer_emit_all(self, argument)) {
        Py_DECREF(argument);
        return -1;
    }
    Py_DECREF(argument);
    if (Tokenizer_emit(self, ArgumentClose))
        return -1;
    return 0;
}

/*
    Parse a template or argument at the head of the wikicode string.
*/
static int Tokenizer_parse_template_or_argument(Tokenizer* self)
{
    unsigned int braces = 2, i;
    PyObject *tokenlist;

    self->head += 2;
    while (Tokenizer_READ(self, 0) == '{' && braces < MAX_BRACES) {
        self->head++;
        braces++;
    }
    if (Tokenizer_push(self, 0))
        return -1;
    while (braces) {
        if (braces == 1) {
            if (Tokenizer_emit_text_then_stack(self, "{"))
                return -1;
            return 0;
        }
        if (braces == 2) {
            if (Tokenizer_parse_template(self))
                return -1;
            if (BAD_ROUTE) {
                RESET_ROUTE();
                if (Tokenizer_emit_text_then_stack(self, "{{"))
                    return -1;
                return 0;
            }
            break;
        }
        if (Tokenizer_parse_argument(self))
            return -1;
        if (BAD_ROUTE) {
            RESET_ROUTE();
            if (Tokenizer_parse_template(self))
                return -1;
            if (BAD_ROUTE) {
                char text[MAX_BRACES + 1];
                RESET_ROUTE();
                for (i = 0; i < braces; i++) text[i] = '{';
                text[braces] = '\0';
                if (Tokenizer_emit_text_then_stack(self, text)) {
                    Py_XDECREF(text);
                    return -1;
                }
                Py_XDECREF(text);
                return 0;
            }
            else
                braces -= 2;
        }
        else
            braces -= 3;
        if (braces)
            self->head++;
    }
    tokenlist = Tokenizer_pop(self);
    if (!tokenlist)
        return -1;
    if (Tokenizer_emit_all(self, tokenlist)) {
        Py_DECREF(tokenlist);
        return -1;
    }
    Py_DECREF(tokenlist);
    if (self->topstack->context & LC_FAIL_NEXT)
        self->topstack->context ^= LC_FAIL_NEXT;
    return 0;
}

/*
    Handle a template parameter at the head of the string.
*/
static int Tokenizer_handle_template_param(Tokenizer* self)
{
    PyObject *stack;

    if (self->topstack->context & LC_TEMPLATE_NAME)
        self->topstack->context ^= LC_TEMPLATE_NAME;
    else if (self->topstack->context & LC_TEMPLATE_PARAM_VALUE)
        self->topstack->context ^= LC_TEMPLATE_PARAM_VALUE;
    if (self->topstack->context & LC_TEMPLATE_PARAM_KEY) {
        stack = Tokenizer_pop_keeping_context(self);
        if (!stack)
            return -1;
        if (Tokenizer_emit_all(self, stack)) {
            Py_DECREF(stack);
            return -1;
        }
        Py_DECREF(stack);
    }
    else
        self->topstack->context |= LC_TEMPLATE_PARAM_KEY;
    if (Tokenizer_emit(self, TemplateParamSeparator))
        return -1;
    if (Tokenizer_push(self, self->topstack->context))
        return -1;
    return 0;
}

/*
    Handle a template parameter's value at the head of the string.
*/
static int Tokenizer_handle_template_param_value(Tokenizer* self)
{
    PyObject *stack;

    stack = Tokenizer_pop_keeping_context(self);
    if (!stack)
        return -1;
    if (Tokenizer_emit_all(self, stack)) {
        Py_DECREF(stack);
        return -1;
    }
    Py_DECREF(stack);
    self->topstack->context ^= LC_TEMPLATE_PARAM_KEY;
    self->topstack->context |= LC_TEMPLATE_PARAM_VALUE;
    if (Tokenizer_emit(self, TemplateParamEquals))
        return -1;
    return 0;
}

/*
    Handle the end of a template at the head of the string.
*/
static PyObject* Tokenizer_handle_template_end(Tokenizer* self)
{
    PyObject* stack;

    if (self->topstack->context & LC_TEMPLATE_PARAM_KEY) {
        stack = Tokenizer_pop_keeping_context(self);
        if (!stack)
            return NULL;
        if (Tokenizer_emit_all(self, stack)) {
            Py_DECREF(stack);
            return NULL;
        }
        Py_DECREF(stack);
    }
    self->head++;
    stack = Tokenizer_pop(self);
    return stack;
}

/*
    Handle the separator between an argument's name and default.
*/
static int Tokenizer_handle_argument_separator(Tokenizer* self)
{
    self->topstack->context ^= LC_ARGUMENT_NAME;
    self->topstack->context |= LC_ARGUMENT_DEFAULT;
    if (Tokenizer_emit(self, ArgumentSeparator))
        return -1;
    return 0;
}

/*
    Handle the end of an argument at the head of the string.
*/
static PyObject* Tokenizer_handle_argument_end(Tokenizer* self)
{
    PyObject* stack = Tokenizer_pop(self);

    self->head += 2;
    return stack;
}

/*
    Parse an internal wikilink at the head of the wikicode string.
*/
static int Tokenizer_parse_wikilink(Tokenizer* self)
{
    Py_ssize_t reset;
    PyObject *wikilink;

    self->head += 2;
    reset = self->head - 1;
    wikilink = Tokenizer_parse(self, LC_WIKILINK_TITLE, 1);
    if (BAD_ROUTE) {
        RESET_ROUTE();
        self->head = reset;
        if (Tokenizer_emit_text(self, "[["))
            return -1;
        return 0;
    }
    if (!wikilink)
        return -1;
    if (Tokenizer_emit(self, WikilinkOpen)) {
        Py_DECREF(wikilink);
        return -1;
    }
    if (Tokenizer_emit_all(self, wikilink)) {
        Py_DECREF(wikilink);
        return -1;
    }
    Py_DECREF(wikilink);
    if (Tokenizer_emit(self, WikilinkClose))
        return -1;
    if (self->topstack->context & LC_FAIL_NEXT)
        self->topstack->context ^= LC_FAIL_NEXT;
    return 0;
}

/*
    Handle the separator between a wikilink's title and its text.
*/
static int Tokenizer_handle_wikilink_separator(Tokenizer* self)
{
    self->topstack->context ^= LC_WIKILINK_TITLE;
    self->topstack->context |= LC_WIKILINK_TEXT;
    if (Tokenizer_emit(self, WikilinkSeparator))
        return -1;
    return 0;
}

/*
    Handle the end of a wikilink at the head of the string.
*/
static PyObject* Tokenizer_handle_wikilink_end(Tokenizer* self)
{
    PyObject* stack = Tokenizer_pop(self);
    self->head += 1;
    return stack;
}

/*
    Parse the URI scheme of a bracket-enclosed external link.
*/
static int Tokenizer_parse_bracketed_uri_scheme(Tokenizer* self)
{
    static const char* valid = "abcdefghijklmnopqrstuvwxyz0123456789+.-";
    Textbuffer* buffer;
    PyObject* scheme;
    Py_UNICODE this;
    int slashes, i;

    if (Tokenizer_push(self, LC_EXT_LINK_URI))
        return -1;
    if (Tokenizer_READ(self, 0) == '/' && Tokenizer_READ(self, 1) == '/') {
        if (Tokenizer_emit_text(self, "//"))
            return -1;
        self->head += 2;
    }
    else {
        buffer = Textbuffer_new();
        if (!buffer)
            return -1;
        while ((this = Tokenizer_READ(self, 0))) {
            i = 0;
            while (1) {
                if (!valid[i])
                    goto end_of_loop;
                if (this == valid[i])
                    break;
                i++;
            }
            Textbuffer_write(&buffer, this);
            if (Tokenizer_emit_char(self, this)) {
                Textbuffer_dealloc(buffer);
                return -1;
            }
            self->head++;
        }
        end_of_loop:
        if (this != ':') {
            Textbuffer_dealloc(buffer);
            Tokenizer_fail_route(self);
            return 0;
        }
        if (Tokenizer_emit_char(self, ':')) {
            Textbuffer_dealloc(buffer);
            return -1;
        }
        self->head++;
        slashes = (Tokenizer_READ(self, 0) == '/' &&
                   Tokenizer_READ(self, 1) == '/');
        if (slashes) {
            if (Tokenizer_emit_text(self, "//")) {
                Textbuffer_dealloc(buffer);
                return -1;
            }
            self->head += 2;
        }
        scheme = Textbuffer_render(buffer);
        Textbuffer_dealloc(buffer);
        if (!scheme)
            return -1;
        if (!IS_SCHEME(scheme, slashes, 0)) {
            Py_DECREF(scheme);
            Tokenizer_fail_route(self);
            return 0;
        }
        Py_DECREF(scheme);
    }
    return 0;
}

/*
    Parse the URI scheme of a free (no brackets) external link.
*/
static int Tokenizer_parse_free_uri_scheme(Tokenizer* self)
{
    static const char* valid = "abcdefghijklmnopqrstuvwxyz0123456789+.-";
    Textbuffer *scheme_buffer = Textbuffer_new(), *temp_buffer;
    PyObject *scheme;
    Py_UNICODE chunk;
    long i;
    int slashes, j;

    if (!scheme_buffer)
        return -1;
    // We have to backtrack through the textbuffer looking for our scheme since
    // it was just parsed as text:
    temp_buffer = self->topstack->textbuffer;
    while (temp_buffer) {
        for (i = temp_buffer->size - 1; i >= 0; i--) {
            chunk = temp_buffer->data[i];
            if (Py_UNICODE_ISSPACE(chunk) || is_marker(chunk))
                goto end_of_loop;
            j = 0;
            while (1) {
                if (!valid[j]) {
                    Textbuffer_dealloc(scheme_buffer);
                    FAIL_ROUTE(0);
                    return 0;
                }
                if (chunk == valid[j])
                    break;
                j++;
            }
            Textbuffer_write(&scheme_buffer, chunk);
        }
        temp_buffer = temp_buffer->next;
    }
    end_of_loop:
    scheme = Textbuffer_render(scheme_buffer);
    if (!scheme) {
        Textbuffer_dealloc(scheme_buffer);
        return -1;
    }
    slashes = (Tokenizer_READ(self, 0) == '/' &&
               Tokenizer_READ(self, 1) == '/');
    if (!IS_SCHEME(scheme, slashes, 1)) {
        Py_DECREF(scheme);
        Textbuffer_dealloc(scheme_buffer);
        FAIL_ROUTE(0);
        return 0;
    }
    Py_DECREF(scheme);
    if (Tokenizer_push(self, self->topstack->context | LC_EXT_LINK_URI)) {
        Textbuffer_dealloc(scheme_buffer);
        return -1;
    }
    if (Tokenizer_emit_textbuffer(self, scheme_buffer, 1))
        return -1;
    if (Tokenizer_emit_char(self, ':'))
        return -1;
    if (slashes) {
        if (Tokenizer_emit_text(self, "//"))
            return -1;
        self->head += 2;
    }
    return 0;
}

/*
    Handle text in a free external link, including trailing punctuation.
*/
static int
Tokenizer_handle_free_link_text(Tokenizer* self, int* parens,
                                Textbuffer** tail, Py_UNICODE this)
{
    #define PUSH_TAIL_BUFFER(tail, error)                 \
        if ((tail)->size || (tail)->next) {               \
            if (Tokenizer_emit_textbuffer(self, tail, 0)) \
                return error;                             \
            tail = Textbuffer_new();                      \
            if (!(tail))                                  \
                return error;                             \
        }

    if (this == '(' && !(*parens)) {
        *parens = 1;
        PUSH_TAIL_BUFFER(*tail, -1)
    }
    else if (this == ',' || this == ';' || this == '\\' || this == '.' ||
             this == ':' || this == '!' || this == '?' ||
             (!(*parens) && this == ')'))
        return Textbuffer_write(tail, this);
    else
        PUSH_TAIL_BUFFER(*tail, -1)
    return Tokenizer_emit_char(self, this);
}

/*
    Return whether the current head is the end of a free link.
*/
static int
Tokenizer_is_free_link(Tokenizer* self, Py_UNICODE this, Py_UNICODE next)
{
    // Built from Tokenizer_parse()'s end sentinels:
    Py_UNICODE after = Tokenizer_READ(self, 2);
    int ctx = self->topstack->context;

    return (!this || this == '\n' || this == '[' || this == ']' ||
        this == '<' || this == '>'  || (this == '\'' && next == '\'') ||
        (this == '|' && ctx & LC_TEMPLATE) ||
        (this == '=' && ctx & (LC_TEMPLATE_PARAM_KEY | LC_HEADING)) ||
        (this == '}' && next == '}' &&
            (ctx & LC_TEMPLATE || (after == '}' && ctx & LC_ARGUMENT))));
}

/*
    Really parse an external link.
*/
static PyObject*
Tokenizer_really_parse_external_link(Tokenizer* self, int brackets,
                                     Textbuffer** extra)
{
    Py_UNICODE this, next;
    int parens = 0;

    if (brackets ? Tokenizer_parse_bracketed_uri_scheme(self) :
                   Tokenizer_parse_free_uri_scheme(self))
        return NULL;
    if (BAD_ROUTE)
        return NULL;
    this = Tokenizer_READ(self, 0);
    if (!this || this == '\n' || this == ' ' || this == ']')
        return Tokenizer_fail_route(self);
    if (!brackets && this == '[')
        return Tokenizer_fail_route(self);
    while (1) {
        this = Tokenizer_READ(self, 0);
        next = Tokenizer_READ(self, 1);
        if (this == '&') {
            PUSH_TAIL_BUFFER(*extra, NULL)
            if (Tokenizer_parse_entity(self))
                return NULL;
        }
        else if (this == '<' && next == '!'
                 && Tokenizer_READ(self, 2) == '-'
                 && Tokenizer_READ(self, 3) == '-') {
            PUSH_TAIL_BUFFER(*extra, NULL)
            if (Tokenizer_parse_comment(self))
                return NULL;
        }
        else if (!brackets && Tokenizer_is_free_link(self, this, next)) {
            self->head--;
            return Tokenizer_pop(self);
        }
        else if (!this || this == '\n')
            return Tokenizer_fail_route(self);
        else if (this == '{' && next == '{' && Tokenizer_CAN_RECURSE(self)) {
            PUSH_TAIL_BUFFER(*extra, NULL)
            if (Tokenizer_parse_template_or_argument(self))
                return NULL;
        }
        else if (this == ']')
            return Tokenizer_pop(self);
        else if (this == ' ') {
            if (brackets) {
                if (Tokenizer_emit(self, ExternalLinkSeparator))
                    return NULL;
                self->topstack->context ^= LC_EXT_LINK_URI;
                self->topstack->context |= LC_EXT_LINK_TITLE;
                self->head++;
                return Tokenizer_parse(self, 0, 0);
            }
            if (Textbuffer_write(extra, ' '))
                return NULL;
            return Tokenizer_pop(self);
        }
        else if (!brackets) {
            if (Tokenizer_handle_free_link_text(self, &parens, extra, this))
                return NULL;
        }
        else {
            if (Tokenizer_emit_char(self, this))
                return NULL;
        }
        self->head++;
    }
}

/*
    Remove the URI scheme of a new external link from the textbuffer.
*/
static int
Tokenizer_remove_uri_scheme_from_textbuffer(Tokenizer* self, PyObject* link)
{
    PyObject *text = PyObject_GetAttrString(PyList_GET_ITEM(link, 0), "text"),
             *split, *scheme;
    Py_ssize_t length;
    Textbuffer* temp;

    if (!text)
        return -1;
    split = PyObject_CallMethod(text, "split", "si", ":", 1);
    Py_DECREF(text);
    if (!split)
        return -1;
    scheme = PyList_GET_ITEM(split, 0);
    length = PyUnicode_GET_SIZE(scheme);
    while (length) {
        temp = self->topstack->textbuffer;
        if (length <= temp->size) {
            temp->size -= length;
            break;
        }
        length -= temp->size;
        self->topstack->textbuffer = temp->next;
        free(temp->data);
        free(temp);
    }
    Py_DECREF(split);
    return 0;
}

/*
    Parse an external link at the head of the wikicode string.
*/
static int Tokenizer_parse_external_link(Tokenizer* self, int brackets)
{
    #define INVALID_CONTEXT self->topstack->context & AGG_NO_EXT_LINKS
    #define NOT_A_LINK                                        \
        if (!brackets && self->topstack->context & LC_DLTERM) \
            return Tokenizer_handle_dl_term(self);            \
        return Tokenizer_emit_char(self, Tokenizer_READ(self, 0))

    Py_ssize_t reset = self->head;
    PyObject *link, *kwargs;
    Textbuffer *extra = 0;

    if (INVALID_CONTEXT || !(Tokenizer_CAN_RECURSE(self))) {
        NOT_A_LINK;
    }
    extra = Textbuffer_new();
    if (!extra)
        return -1;
    self->head++;
    link = Tokenizer_really_parse_external_link(self, brackets, &extra);
    if (BAD_ROUTE) {
        RESET_ROUTE();
        self->head = reset;
        Textbuffer_dealloc(extra);
        NOT_A_LINK;
    }
    if (!link) {
        Textbuffer_dealloc(extra);
        return -1;
    }
    if (!brackets) {
        if (Tokenizer_remove_uri_scheme_from_textbuffer(self, link)) {
            Textbuffer_dealloc(extra);
            Py_DECREF(link);
            return -1;
        }
    }
    kwargs = PyDict_New();
    if (!kwargs) {
        Textbuffer_dealloc(extra);
        Py_DECREF(link);
        return -1;
    }
    PyDict_SetItemString(kwargs, "brackets", brackets ? Py_True : Py_False);
    if (Tokenizer_emit_kwargs(self, ExternalLinkOpen, kwargs)) {
        Textbuffer_dealloc(extra);
        Py_DECREF(link);
        return -1;
    }
    if (Tokenizer_emit_all(self, link)) {
        Textbuffer_dealloc(extra);
        Py_DECREF(link);
        return -1;
    }
    Py_DECREF(link);
    if (Tokenizer_emit(self, ExternalLinkClose)) {
        Textbuffer_dealloc(extra);
        return -1;
    }
    if (extra->size || extra->next)
        return Tokenizer_emit_textbuffer(self, extra, 0);
    Textbuffer_dealloc(extra);
    return 0;
}

/*
    Parse a section heading at the head of the wikicode string.
*/
static int Tokenizer_parse_heading(Tokenizer* self)
{
    Py_ssize_t reset = self->head;
    int best = 1, i, context, diff;
    HeadingData *heading;
    PyObject *level, *kwargs;

    self->global |= GL_HEADING;
    self->head += 1;
    while (Tokenizer_READ(self, 0) == '=') {
        best++;
        self->head++;
    }
    context = LC_HEADING_LEVEL_1 << (best > 5 ? 5 : best - 1);
    heading = (HeadingData*) Tokenizer_parse(self, context, 1);
    if (BAD_ROUTE) {
        RESET_ROUTE();
        self->head = reset + best - 1;
        for (i = 0; i < best; i++) {
            if (Tokenizer_emit_char(self, '='))
                return -1;
        }
        self->global ^= GL_HEADING;
        return 0;
    }
    level = NEW_INT_FUNC(heading->level);
    if (!level) {
        Py_DECREF(heading->title);
        free(heading);
        return -1;
    }
    kwargs = PyDict_New();
    if (!kwargs) {
        Py_DECREF(level);
        Py_DECREF(heading->title);
        free(heading);
        return -1;
    }
    PyDict_SetItemString(kwargs, "level", level);
    Py_DECREF(level);
    if (Tokenizer_emit_kwargs(self, HeadingStart, kwargs)) {
        Py_DECREF(heading->title);
        free(heading);
        return -1;
    }
    if (heading->level < best) {
        diff = best - heading->level;
        for (i = 0; i < diff; i++) {
            if (Tokenizer_emit_char(self, '=')) {
                Py_DECREF(heading->title);
                free(heading);
                return -1;
            }
        }
    }
    if (Tokenizer_emit_all(self, heading->title)) {
        Py_DECREF(heading->title);
        free(heading);
        return -1;
    }
    Py_DECREF(heading->title);
    free(heading);
    if (Tokenizer_emit(self, HeadingEnd))
        return -1;
    self->global ^= GL_HEADING;
    return 0;
}

/*
    Handle the end of a section heading at the head of the string.
*/
static HeadingData* Tokenizer_handle_heading_end(Tokenizer* self)
{
    Py_ssize_t reset = self->head;
    int best, i, current, level, diff;
    HeadingData *after, *heading;
    PyObject *stack;

    self->head += 1;
    best = 1;
    while (Tokenizer_READ(self, 0) == '=') {
        best++;
        self->head++;
    }
    current = heading_level_from_context(self->topstack->context);
    level = current > best ? (best > 6 ? 6 : best) :
                             (current > 6 ? 6 : current);
    after = (HeadingData*) Tokenizer_parse(self, self->topstack->context, 1);
    if (BAD_ROUTE) {
        RESET_ROUTE();
        if (level < best) {
            diff = best - level;
            for (i = 0; i < diff; i++) {
                if (Tokenizer_emit_char(self, '='))
                    return NULL;
            }
        }
        self->head = reset + best - 1;
    }
    else {
        for (i = 0; i < best; i++) {
            if (Tokenizer_emit_char(self, '=')) {
                Py_DECREF(after->title);
                free(after);
                return NULL;
            }
        }
        if (Tokenizer_emit_all(self, after->title)) {
            Py_DECREF(after->title);
            free(after);
            return NULL;
        }
        Py_DECREF(after->title);
        level = after->level;
        free(after);
    }
    stack = Tokenizer_pop(self);
    if (!stack)
        return NULL;
    heading = malloc(sizeof(HeadingData));
    if (!heading) {
        PyErr_NoMemory();
        return NULL;
    }
    heading->title = stack;
    heading->level = level;
    return heading;
}

/*
    Actually parse an HTML entity and ensure that it is valid.
*/
static int Tokenizer_really_parse_entity(Tokenizer* self)
{
    PyObject *kwargs, *textobj;
    Py_UNICODE this;
    int numeric, hexadecimal, i, j, zeroes, test;
    char *valid, *text, *buffer, *def;

    #define FAIL_ROUTE_AND_EXIT() { \
        Tokenizer_fail_route(self); \
        free(text);                 \
        return 0;                   \
    }

    if (Tokenizer_emit(self, HTMLEntityStart))
        return -1;
    self->head++;
    this = Tokenizer_READ(self, 0);
    if (!this) {
        Tokenizer_fail_route(self);
        return 0;
    }
    if (this == '#') {
        numeric = 1;
        if (Tokenizer_emit(self, HTMLEntityNumeric))
            return -1;
        self->head++;
        this = Tokenizer_READ(self, 0);
        if (!this) {
            Tokenizer_fail_route(self);
            return 0;
        }
        if (this == 'x' || this == 'X') {
            hexadecimal = 1;
            kwargs = PyDict_New();
            if (!kwargs)
                return -1;
            PyDict_SetItemString(kwargs, "char", Tokenizer_read(self, 0));
            if (Tokenizer_emit_kwargs(self, HTMLEntityHex, kwargs))
                return -1;
            self->head++;
        }
        else
            hexadecimal = 0;
    }
    else
        numeric = hexadecimal = 0;
    if (hexadecimal)
        valid = HEXDIGITS;
    else if (numeric)
        valid = DIGITS;
    else
        valid = ALPHANUM;
    text = calloc(MAX_ENTITY_SIZE, sizeof(char));
    if (!text) {
        PyErr_NoMemory();
        return -1;
    }
    i = 0;
    zeroes = 0;
    while (1) {
        this = Tokenizer_READ(self, 0);
        if (this == ';') {
            if (i == 0)
                FAIL_ROUTE_AND_EXIT()
            break;
        }
        if (i == 0 && this == '0') {
            zeroes++;
            self->head++;
            continue;
        }
        if (i >= MAX_ENTITY_SIZE)
            FAIL_ROUTE_AND_EXIT()
        if (is_marker(this))
            FAIL_ROUTE_AND_EXIT()
        j = 0;
        while (1) {
            if (!valid[j])
                FAIL_ROUTE_AND_EXIT()
            if (this == valid[j])
                break;
            j++;
        }
        text[i] = (char) this;
        self->head++;
        i++;
    }
    if (numeric) {
        sscanf(text, (hexadecimal ? "%x" : "%d"), &test);
        if (test < 1 || test > 0x10FFFF)
            FAIL_ROUTE_AND_EXIT()
    }
    else {
        i = 0;
        while (1) {
            def = entitydefs[i];
            if (!def)  // We've reached the end of the defs without finding it
                FAIL_ROUTE_AND_EXIT()
            if (strcmp(text, def) == 0)
                break;
            i++;
        }
    }
    if (zeroes) {
        buffer = calloc(strlen(text) + zeroes + 1, sizeof(char));
        if (!buffer) {
            free(text);
            PyErr_NoMemory();
            return -1;
        }
        for (i = 0; i < zeroes; i++)
            strcat(buffer, "0");
        strcat(buffer, text);
        free(text);
        text = buffer;
    }
    textobj = PyUnicode_FromString(text);
    if (!textobj) {
        free(text);
        return -1;
    }
    free(text);
    kwargs = PyDict_New();
    if (!kwargs) {
        Py_DECREF(textobj);
        return -1;
    }
    PyDict_SetItemString(kwargs, "text", textobj);
    Py_DECREF(textobj);
    if (Tokenizer_emit_kwargs(self, Text, kwargs))
        return -1;
    if (Tokenizer_emit(self, HTMLEntityEnd))
        return -1;
    return 0;
}

/*
    Parse an HTML entity at the head of the wikicode string.
*/
static int Tokenizer_parse_entity(Tokenizer* self)
{
    Py_ssize_t reset = self->head;
    PyObject *tokenlist;

    if (Tokenizer_push(self, 0))
        return -1;
    if (Tokenizer_really_parse_entity(self))
        return -1;
    if (BAD_ROUTE) {
        RESET_ROUTE();
        self->head = reset;
        if (Tokenizer_emit_char(self, '&'))
            return -1;
        return 0;
    }
    tokenlist = Tokenizer_pop(self);
    if (!tokenlist)
        return -1;
    if (Tokenizer_emit_all(self, tokenlist)) {
        Py_DECREF(tokenlist);
        return -1;
    }
    Py_DECREF(tokenlist);
    return 0;
}

/*
    Parse an HTML comment at the head of the wikicode string.
*/
static int Tokenizer_parse_comment(Tokenizer* self)
{
    Py_ssize_t reset = self->head + 3;
    PyObject *comment;
    Py_UNICODE this;

    self->head += 4;
    if (Tokenizer_push(self, 0))
        return -1;
    while (1) {
        this = Tokenizer_READ(self, 0);
        if (!this) {
            comment = Tokenizer_pop(self);
            Py_XDECREF(comment);
            self->head = reset;
            return Tokenizer_emit_text(self, "<!--");
        }
        if (this == '-' && Tokenizer_READ(self, 1) == this &&
                            Tokenizer_READ(self, 2) == '>') {
            if (Tokenizer_emit_first(self, CommentStart))
                return -1;
            if (Tokenizer_emit(self, CommentEnd))
                return -1;
            comment = Tokenizer_pop(self);
            if (!comment)
                return -1;
            if (Tokenizer_emit_all(self, comment))
                return -1;
            Py_DECREF(comment);
            self->head += 2;
            return 0;
        }
        if (Tokenizer_emit_char(self, this))
            return -1;
        self->head++;
    }
}

/*
    Write a pending tag attribute from data to the stack.
*/
static int Tokenizer_push_tag_buffer(Tokenizer* self, TagData* data)
{
    PyObject *tokens, *kwargs, *pad_first, *pad_before_eq, *pad_after_eq;

    if (data->context & TAG_QUOTED) {
        if (Tokenizer_emit_first(self, TagAttrQuote))
            return -1;
        tokens = Tokenizer_pop(self);
        if (!tokens)
            return -1;
        if (Tokenizer_emit_all(self, tokens)) {
            Py_DECREF(tokens);
            return -1;
        }
        Py_DECREF(tokens);
    }
    pad_first = Textbuffer_render(data->pad_first);
    pad_before_eq = Textbuffer_render(data->pad_before_eq);
    pad_after_eq = Textbuffer_render(data->pad_after_eq);
    if (!pad_first || !pad_before_eq || !pad_after_eq)
        return -1;
    kwargs = PyDict_New();
    if (!kwargs)
        return -1;
    PyDict_SetItemString(kwargs, "pad_first", pad_first);
    PyDict_SetItemString(kwargs, "pad_before_eq", pad_before_eq);
    PyDict_SetItemString(kwargs, "pad_after_eq", pad_after_eq);
    Py_DECREF(pad_first);
    Py_DECREF(pad_before_eq);
    Py_DECREF(pad_after_eq);
    if (Tokenizer_emit_first_kwargs(self, TagAttrStart, kwargs))
        return -1;
    tokens = Tokenizer_pop(self);
    if (!tokens)
        return -1;
    if (Tokenizer_emit_all(self, tokens)) {
        Py_DECREF(tokens);
        return -1;
    }
    Py_DECREF(tokens);
    if (TagData_reset_buffers(data))
        return -1;
    return 0;
}

/*
    Handle whitespace inside of an HTML open tag.
*/
static int
Tokenizer_handle_tag_space(Tokenizer* self, TagData* data, Py_UNICODE text)
{
    int ctx = data->context;
    int end_of_value = (ctx & TAG_ATTR_VALUE &&
                        !(ctx & (TAG_QUOTED | TAG_NOTE_QUOTE)));

    if (end_of_value || (ctx & TAG_QUOTED && ctx & TAG_NOTE_SPACE)) {
        if (Tokenizer_push_tag_buffer(self, data))
            return -1;
        data->context = TAG_ATTR_READY;
    }
    else if (ctx & TAG_NOTE_SPACE)
        data->context = TAG_ATTR_READY;
    else if (ctx & TAG_ATTR_NAME) {
        data->context |= TAG_NOTE_EQUALS;
        if (Textbuffer_write(&(data->pad_before_eq), text))
            return -1;
    }
    if (ctx & TAG_QUOTED && !(ctx & TAG_NOTE_SPACE)) {
        if (Tokenizer_emit_char(self, text))
            return -1;
    }
    else if (data->context & TAG_ATTR_READY)
        return Textbuffer_write(&(data->pad_first), text);
    else if (data->context & TAG_ATTR_VALUE)
        return Textbuffer_write(&(data->pad_after_eq), text);
    return 0;
}

/*
    Handle regular text inside of an HTML open tag.
*/
static int Tokenizer_handle_tag_text(Tokenizer* self, Py_UNICODE text)
{
    Py_UNICODE next = Tokenizer_READ(self, 1);

    if (!is_marker(text) || !Tokenizer_CAN_RECURSE(self))
        return Tokenizer_emit_char(self, text);
    else if (text == next && next == '{')
        return Tokenizer_parse_template_or_argument(self);
    else if (text == next && next == '[')
        return Tokenizer_parse_wikilink(self);
    else if (text == '<')
        return Tokenizer_parse_tag(self);
    return Tokenizer_emit_char(self, text);
}

/*
    Handle all sorts of text data inside of an HTML open tag.
*/
static int
Tokenizer_handle_tag_data(Tokenizer* self, TagData* data, Py_UNICODE chunk)
{
    PyObject *trash;
    int first_time, escaped;

    if (data->context & TAG_NAME) {
        first_time = !(data->context & TAG_NOTE_SPACE);
        if (is_marker(chunk) || (Py_UNICODE_ISSPACE(chunk) && first_time)) {
            // Tags must start with text, not spaces
            Tokenizer_fail_route(self);
            return 0;
        }
        else if (first_time)
            data->context |= TAG_NOTE_SPACE;
        else if (Py_UNICODE_ISSPACE(chunk)) {
            data->context = TAG_ATTR_READY;
            return Tokenizer_handle_tag_space(self, data, chunk);
        }
    }
    else if (Py_UNICODE_ISSPACE(chunk))
        return Tokenizer_handle_tag_space(self, data, chunk);
    else if (data->context & TAG_NOTE_SPACE) {
        if (data->context & TAG_QUOTED) {
            data->context = TAG_ATTR_VALUE;
            trash = Tokenizer_pop(self);
            Py_XDECREF(trash);
            self->head = data->reset - 1;  // Will be auto-incremented
        }
        else
            Tokenizer_fail_route(self);
        return 0;
    }
    else if (data->context & TAG_ATTR_READY) {
        data->context = TAG_ATTR_NAME;
        if (Tokenizer_push(self, LC_TAG_ATTR))
            return -1;
    }
    else if (data->context & TAG_ATTR_NAME) {
        if (chunk == '=') {
            data->context = TAG_ATTR_VALUE | TAG_NOTE_QUOTE;
            if (Tokenizer_emit(self, TagAttrEquals))
                return -1;
            return 0;
        }
        if (data->context & TAG_NOTE_EQUALS) {
            if (Tokenizer_push_tag_buffer(self, data))
                return -1;
            data->context = TAG_ATTR_NAME;
            if (Tokenizer_push(self, LC_TAG_ATTR))
                return -1;
        }
    }
    else if (data->context & TAG_ATTR_VALUE) {
        escaped = (Tokenizer_READ_BACKWARDS(self, 1) == '\\' &&
                   Tokenizer_READ_BACKWARDS(self, 2) != '\\');
        if (data->context & TAG_NOTE_QUOTE) {
            data->context ^= TAG_NOTE_QUOTE;
            if (chunk == '"' && !escaped) {
                data->context |= TAG_QUOTED;
                if (Tokenizer_push(self, self->topstack->context))
                    return -1;
                data->reset = self->head;
                return 0;
            }
        }
        else if (data->context & TAG_QUOTED) {
            if (chunk == '"' && !escaped) {
                data->context |= TAG_NOTE_SPACE;
                return 0;
            }
        }
    }
    return Tokenizer_handle_tag_text(self, chunk);
}

/*
    Handle the closing of a open tag (<foo>).
*/
static int
Tokenizer_handle_tag_close_open(Tokenizer* self, TagData* data, PyObject* cls)
{
    PyObject *padding, *kwargs;

    if (data->context & (TAG_ATTR_NAME | TAG_ATTR_VALUE)) {
        if (Tokenizer_push_tag_buffer(self, data))
            return -1;
    }
    padding = Textbuffer_render(data->pad_first);
    if (!padding)
        return -1;
    kwargs = PyDict_New();
    if (!kwargs) {
        Py_DECREF(padding);
        return -1;
    }
    PyDict_SetItemString(kwargs, "padding", padding);
    Py_DECREF(padding);
    if (Tokenizer_emit_kwargs(self, cls, kwargs))
        return -1;
    self->head++;
    return 0;
}

/*
    Handle the opening of a closing tag (</foo>).
*/
static int Tokenizer_handle_tag_open_close(Tokenizer* self)
{
    if (Tokenizer_emit(self, TagOpenClose))
        return -1;
    if (Tokenizer_push(self, LC_TAG_CLOSE))
        return -1;
    self->head++;
    return 0;
}

/*
    Handle the ending of a closing tag (</foo>).
*/
static PyObject* Tokenizer_handle_tag_close_close(Tokenizer* self)
{
    PyObject *closing, *first, *so, *sc;
    int valid = 1;

    closing = Tokenizer_pop(self);
    if (!closing)
        return NULL;
    if (PyList_GET_SIZE(closing) != 1)
        valid = 0;
    else {
        first = PyList_GET_ITEM(closing, 0);
        switch (PyObject_IsInstance(first, Text)) {
            case 0:
                valid = 0;
                break;
            case 1: {
                so = strip_tag_name(first);
                sc = strip_tag_name(PyList_GET_ITEM(self->topstack->stack, 1));
                if (so && sc) {
                    if (PyUnicode_Compare(so, sc))
                        valid = 0;
                    Py_DECREF(so);
                    Py_DECREF(sc);
                    break;
                }
                Py_XDECREF(so);
                Py_XDECREF(sc);
            }
            case -1:
                Py_DECREF(closing);
                return NULL;
        }
    }
    if (!valid) {
        Py_DECREF(closing);
        return Tokenizer_fail_route(self);
    }
    if (Tokenizer_emit_all(self, closing)) {
        Py_DECREF(closing);
        return NULL;
    }
    Py_DECREF(closing);
    if (Tokenizer_emit(self, TagCloseClose))
        return NULL;
    return Tokenizer_pop(self);
}

/*
    Handle the body of an HTML tag that is parser-blacklisted.
*/
static PyObject* Tokenizer_handle_blacklisted_tag(Tokenizer* self)
{
    Py_UNICODE this, next;

    while (1) {
        this = Tokenizer_READ(self, 0);
        next = Tokenizer_READ(self, 1);
        if (!this)
            return Tokenizer_fail_route(self);
        else if (this == '<' && next == '/') {
            if (Tokenizer_handle_tag_open_close(self))
                return NULL;
            self->head++;
            return Tokenizer_parse(self, 0, 0);
        }
        else if (this == '&') {
            if (Tokenizer_parse_entity(self))
                return NULL;
        }
        else if (Tokenizer_emit_char(self, this))
            return NULL;
        self->head++;
    }
}

/*
    Handle the end of an implicitly closing single-only HTML tag.
*/
static PyObject* Tokenizer_handle_single_only_tag_end(Tokenizer* self)
{
    PyObject *top, *padding, *kwargs;

    top = PyObject_CallMethod(self->topstack->stack, "pop", NULL);
    if (!top)
        return NULL;
    padding = PyObject_GetAttrString(top, "padding");
    Py_DECREF(top);
    if (!padding)
        return NULL;
    kwargs = PyDict_New();
    if (!kwargs) {
        Py_DECREF(padding);
        return NULL;
    }
    PyDict_SetItemString(kwargs, "padding", padding);
    PyDict_SetItemString(kwargs, "implicit", Py_True);
    Py_DECREF(padding);
    if (Tokenizer_emit_kwargs(self, TagCloseSelfclose, kwargs))
        return NULL;
    self->head--;  // Offset displacement done by handle_tag_close_open
    return Tokenizer_pop(self);
}

/*
    Handle the stream end when inside a single-supporting HTML tag.
*/
static PyObject* Tokenizer_handle_single_tag_end(Tokenizer* self)
{
    PyObject *token = 0, *padding, *kwargs;
    Py_ssize_t len, index;
    int depth = 1, is_instance;

    len = PyList_GET_SIZE(self->topstack->stack);
    for (index = 2; index < len; index++) {
        token = PyList_GET_ITEM(self->topstack->stack, index);
        is_instance = PyObject_IsInstance(token, TagOpenOpen);
        if (is_instance == -1)
            return NULL;
        else if (is_instance == 1)
            depth++;
        is_instance = PyObject_IsInstance(token, TagCloseOpen);
        if (is_instance == -1)
            return NULL;
        else if (is_instance == 1) {
            depth--;
            if (depth == 0)
                break;
        }
    }
    if (!token || depth > 0)
        return NULL;
    padding = PyObject_GetAttrString(token, "padding");
    if (!padding)
        return NULL;
    kwargs = PyDict_New();
    if (!kwargs) {
        Py_DECREF(padding);
        return NULL;
    }
    PyDict_SetItemString(kwargs, "padding", padding);
    PyDict_SetItemString(kwargs, "implicit", Py_True);
    Py_DECREF(padding);
    token = PyObject_Call(TagCloseSelfclose, NOARGS, kwargs);
    Py_DECREF(kwargs);
    if (!token)
        return NULL;
    if (PyList_SetItem(self->topstack->stack, index, token)) {
        Py_DECREF(token);
        return NULL;
    }
    return Tokenizer_pop(self);
}

/*
    Actually parse an HTML tag, starting with the open (<foo>).
*/
static PyObject* Tokenizer_really_parse_tag(Tokenizer* self)
{
    TagData *data = TagData_new();
    PyObject *token, *text, *trash;
    Py_UNICODE this, next;
    int can_exit;

    if (!data)
        return NULL;
    if (Tokenizer_push(self, LC_TAG_OPEN)) {
        TagData_dealloc(data);
        return NULL;
    }
    if (Tokenizer_emit(self, TagOpenOpen)) {
        TagData_dealloc(data);
        return NULL;
    }
    while (1) {
        this = Tokenizer_READ(self, 0);
        next = Tokenizer_READ(self, 1);
        can_exit = (!(data->context & (TAG_QUOTED | TAG_NAME)) ||
                    data->context & TAG_NOTE_SPACE);
        if (!this) {
            if (self->topstack->context & LC_TAG_ATTR) {
                if (data->context & TAG_QUOTED) {
                    // Unclosed attribute quote: reset, don't die
                    data->context = TAG_ATTR_VALUE;
                    trash = Tokenizer_pop(self);
                    Py_XDECREF(trash);
                    self->head = data->reset;
                    continue;
                }
                trash = Tokenizer_pop(self);
                Py_XDECREF(trash);
            }
            TagData_dealloc(data);
            return Tokenizer_fail_route(self);
        }
        else if (this == '>' && can_exit) {
            if (Tokenizer_handle_tag_close_open(self, data, TagCloseOpen)) {
                TagData_dealloc(data);
                return NULL;
            }
            TagData_dealloc(data);
            self->topstack->context = LC_TAG_BODY;
            token = PyList_GET_ITEM(self->topstack->stack, 1);
            text = PyObject_GetAttrString(token, "text");
            if (!text)
                return NULL;
            if (IS_SINGLE_ONLY(text)) {
                Py_DECREF(text);
                return Tokenizer_handle_single_only_tag_end(self);
            }
            if (IS_PARSABLE(text)) {
                Py_DECREF(text);
                return Tokenizer_parse(self, 0, 0);
            }
            Py_DECREF(text);
            return Tokenizer_handle_blacklisted_tag(self);
        }
        else if (this == '/' && next == '>' && can_exit) {
            if (Tokenizer_handle_tag_close_open(self, data,
                                                TagCloseSelfclose)) {
                TagData_dealloc(data);
                return NULL;
            }
            TagData_dealloc(data);
            return Tokenizer_pop(self);
        }
        else {
            if (Tokenizer_handle_tag_data(self, data, this) || BAD_ROUTE) {
                TagData_dealloc(data);
                return NULL;
            }
        }
        self->head++;
    }
}

/*
    Handle the (possible) start of an implicitly closing single tag.
*/
static int Tokenizer_handle_invalid_tag_start(Tokenizer* self)
{
    Py_ssize_t reset = self->head + 1, pos = 0;
    Textbuffer* buf;
    PyObject *name, *tag;
    Py_UNICODE this;

    self->head += 2;
    buf = Textbuffer_new();
    if (!buf)
        return -1;
    while (1) {
        this = Tokenizer_READ(self, pos);
        if (Py_UNICODE_ISSPACE(this) || is_marker(this)) {
            name = Textbuffer_render(buf);
            if (!name) {
                Textbuffer_dealloc(buf);
                return -1;
            }
            if (!IS_SINGLE_ONLY(name))
                FAIL_ROUTE(0);
            Py_DECREF(name);
            break;
        }
        Textbuffer_write(&buf, this);
        pos++;
    }
    Textbuffer_dealloc(buf);
    if (!BAD_ROUTE)
        tag = Tokenizer_really_parse_tag(self);
    if (BAD_ROUTE) {
        RESET_ROUTE();
        self->head = reset;
        return Tokenizer_emit_text(self, "</");
    }
    if (!tag)
        return -1;
    // Set invalid=True flag of TagOpenOpen
    if (PyObject_SetAttrString(PyList_GET_ITEM(tag, 0), "invalid", Py_True))
        return -1;
    if (Tokenizer_emit_all(self, tag)) {
        Py_DECREF(tag);
        return -1;
    }
    Py_DECREF(tag);
    return 0;
}

/*
    Parse an HTML tag at the head of the wikicode string.
*/
static int Tokenizer_parse_tag(Tokenizer* self)
{
    Py_ssize_t reset = self->head;
    PyObject* tag;

    self->head++;
    tag = Tokenizer_really_parse_tag(self);
    if (BAD_ROUTE) {
        RESET_ROUTE();
        self->head = reset;
        return Tokenizer_emit_char(self, '<');
    }
    if (!tag) {
        return -1;
    }
    if (Tokenizer_emit_all(self, tag)) {
        Py_DECREF(tag);
        return -1;
    }
    Py_DECREF(tag);
    return 0;
}

/*
    Write the body of a tag and the tokens that should surround it.
*/
static int Tokenizer_emit_style_tag(Tokenizer* self, const char* tag,
                                    const char* ticks, PyObject* body)
{
    PyObject *markup, *kwargs;

    markup = PyUnicode_FromString(ticks);
    if (!markup)
        return -1;
    kwargs = PyDict_New();
    if (!kwargs) {
        Py_DECREF(markup);
        return -1;
    }
    PyDict_SetItemString(kwargs, "wiki_markup", markup);
    Py_DECREF(markup);
    if (Tokenizer_emit_kwargs(self, TagOpenOpen, kwargs))
        return -1;
    if (Tokenizer_emit_text(self, tag))
        return -1;
    if (Tokenizer_emit(self, TagCloseOpen))
        return -1;
    if (Tokenizer_emit_all(self, body))
        return -1;
    Py_DECREF(body);
    if (Tokenizer_emit(self, TagOpenClose))
        return -1;
    if (Tokenizer_emit_text(self, tag))
        return -1;
    if (Tokenizer_emit(self, TagCloseClose))
        return -1;
    return 0;
}

/*
    Parse wiki-style italics.
*/
static int Tokenizer_parse_italics(Tokenizer* self)
{
    Py_ssize_t reset = self->head;
    int context;
    PyObject *stack;

    stack = Tokenizer_parse(self, LC_STYLE_ITALICS, 1);
    if (BAD_ROUTE) {
        RESET_ROUTE();
        self->head = reset;
        if (BAD_ROUTE_CONTEXT & LC_STYLE_PASS_AGAIN) {
            context = LC_STYLE_ITALICS | LC_STYLE_SECOND_PASS;
            stack = Tokenizer_parse(self, context, 1);
        }
        else
            return Tokenizer_emit_text(self, "''");
    }
    if (!stack)
        return -1;
    return Tokenizer_emit_style_tag(self, "i", "''", stack);
}

/*
    Parse wiki-style bold.
*/
static int Tokenizer_parse_bold(Tokenizer* self)
{
    Py_ssize_t reset = self->head;
    PyObject *stack;

    stack = Tokenizer_parse(self, LC_STYLE_BOLD, 1);
    if (BAD_ROUTE) {
        RESET_ROUTE();
        self->head = reset;
        if (self->topstack->context & LC_STYLE_SECOND_PASS)
            return Tokenizer_emit_char(self, '\'') ? -1 : 1;
        if (self->topstack->context & LC_STYLE_ITALICS) {
            self->topstack->context |= LC_STYLE_PASS_AGAIN;
            return Tokenizer_emit_text(self, "'''");
        }
        if (Tokenizer_emit_char(self, '\''))
            return -1;
        return Tokenizer_parse_italics(self);
    }
    if (!stack)
        return -1;
    return Tokenizer_emit_style_tag(self, "b", "'''", stack);
}

/*
    Parse wiki-style italics and bold together (i.e., five ticks).
*/
static int Tokenizer_parse_italics_and_bold(Tokenizer* self)
{
    Py_ssize_t reset = self->head;
    PyObject *stack, *stack2;

    stack = Tokenizer_parse(self, LC_STYLE_BOLD, 1);
    if (BAD_ROUTE) {
        RESET_ROUTE();
        self->head = reset;
        stack = Tokenizer_parse(self, LC_STYLE_ITALICS, 1);
        if (BAD_ROUTE) {
            RESET_ROUTE();
            self->head = reset;
            return Tokenizer_emit_text(self, "'''''");
        }
        if (!stack)
            return -1;
        reset = self->head;
        stack2 = Tokenizer_parse(self, LC_STYLE_BOLD, 1);
        if (BAD_ROUTE) {
            RESET_ROUTE();
            self->head = reset;
            if (Tokenizer_emit_text(self, "'''"))
                return -1;
            return Tokenizer_emit_style_tag(self, "i", "''", stack);
        }
        if (!stack2)
            return -1;
        if (Tokenizer_push(self, 0))
            return -1;
        if (Tokenizer_emit_style_tag(self, "i", "''", stack))
            return -1;
        if (Tokenizer_emit_all(self, stack2))
            return -1;
        Py_DECREF(stack2);
        stack2 = Tokenizer_pop(self);
        if (!stack2)
            return -1;
        return Tokenizer_emit_style_tag(self, "b", "'''", stack2);
    }
    if (!stack)
        return -1;
    reset = self->head;
    stack2 = Tokenizer_parse(self, LC_STYLE_ITALICS, 1);
    if (BAD_ROUTE) {
        RESET_ROUTE();
        self->head = reset;
        if (Tokenizer_emit_text(self, "''"))
            return -1;
        return Tokenizer_emit_style_tag(self, "b", "'''", stack);
    }
    if (!stack2)
        return -1;
    if (Tokenizer_push(self, 0))
        return -1;
    if (Tokenizer_emit_style_tag(self, "b", "'''", stack))
        return -1;
    if (Tokenizer_emit_all(self, stack2))
        return -1;
    Py_DECREF(stack2);
    stack2 = Tokenizer_pop(self);
    if (!stack2)
        return -1;
    return Tokenizer_emit_style_tag(self, "i", "''", stack2);
}

/*
    Parse wiki-style formatting (''/''' for italics/bold).
*/
static PyObject* Tokenizer_parse_style(Tokenizer* self)
{
    int context = self->topstack->context, ticks = 2, i;

    self->head += 2;
    while (Tokenizer_READ(self, 0) == '\'') {
        self->head++;
        ticks++;
    }
    if (ticks > 5) {
        for (i = 0; i < ticks - 5; i++) {
            if (Tokenizer_emit_char(self, '\''))
                return NULL;
        }
        ticks = 5;
    }
    else if (ticks == 4) {
        if (Tokenizer_emit_char(self, '\''))
            return NULL;
        ticks = 3;
    }
    if ((context & LC_STYLE_ITALICS && (ticks == 2 || ticks == 5)) ||
           (context & LC_STYLE_BOLD && (ticks == 3 || ticks == 5))) {
        if (ticks == 5)
            self->head -= context & LC_STYLE_ITALICS ? 3 : 2;
        return Tokenizer_pop(self);
    }
    if (!Tokenizer_CAN_RECURSE(self)) {
        if (ticks == 3) {
            if (context & LC_STYLE_SECOND_PASS) {
                if (Tokenizer_emit_char(self, '\''))
                    return NULL;
                return Tokenizer_pop(self);
            }
            if (context & LC_STYLE_ITALICS)
                self->topstack->context |= LC_STYLE_PASS_AGAIN;
        }
        for (i = 0; i < ticks; i++) {
            if (Tokenizer_emit_char(self, '\''))
                return NULL;
        }
    }
    else if (ticks == 2) {
        if (Tokenizer_parse_italics(self))
            return NULL;
    }
    else if (ticks == 3) {
        switch (Tokenizer_parse_bold(self)) {
            case 1:
                return Tokenizer_pop(self);
            case -1:
                return NULL;
        }
    }
    else {
        if (Tokenizer_parse_italics_and_bold(self))
            return NULL;
    }
    self->head--;
    return Py_None;
}

/*
    Handle a list marker at the head (#, *, ;, :).
*/
static int Tokenizer_handle_list_marker(Tokenizer* self)
{
    PyObject *markup = Tokenizer_read(self, 0), *kwargs;
    Py_UNICODE code = *PyUnicode_AS_UNICODE(markup);

    if (code == ';')
        self->topstack->context |= LC_DLTERM;
    kwargs = PyDict_New();
    if (!kwargs)
        return -1;
    PyDict_SetItemString(kwargs, "wiki_markup", markup);
    if (Tokenizer_emit_kwargs(self, TagOpenOpen, kwargs))
        return -1;
    if (Tokenizer_emit_text(self, GET_HTML_TAG(code)))
        return -1;
    if (Tokenizer_emit(self, TagCloseSelfclose))
        return -1;
    return 0;
}

/*
    Handle a wiki-style list (#, *, ;, :).
*/
static int Tokenizer_handle_list(Tokenizer* self)
{
    Py_UNICODE marker = Tokenizer_READ(self, 1);

    if (Tokenizer_handle_list_marker(self))
        return -1;
    while (marker == '#' || marker == '*' || marker == ';' ||
           marker == ':') {
        self->head++;
        if (Tokenizer_handle_list_marker(self))
            return -1;
        marker = Tokenizer_READ(self, 1);
    }
    return 0;
}

/*
    Handle a wiki-style horizontal rule (----) in the string.
*/
static int Tokenizer_handle_hr(Tokenizer* self)
{
    PyObject *markup, *kwargs;
    Textbuffer *buffer = Textbuffer_new();
    int i;

    if (!buffer)
        return -1;
    self->head += 3;
    for (i = 0; i < 4; i++) {
        if (Textbuffer_write(&buffer, '-'))
            return -1;
    }
    while (Tokenizer_READ(self, 1) == '-') {
        if (Textbuffer_write(&buffer, '-'))
            return -1;
        self->head++;
    }
    markup = Textbuffer_render(buffer);
    Textbuffer_dealloc(buffer);
    if (!markup)
        return -1;
    kwargs = PyDict_New();
    if (!kwargs)
        return -1;
    PyDict_SetItemString(kwargs, "wiki_markup", markup);
    Py_DECREF(markup);
    if (Tokenizer_emit_kwargs(self, TagOpenOpen, kwargs))
        return -1;
    if (Tokenizer_emit_text(self, "hr"))
        return -1;
    if (Tokenizer_emit(self, TagCloseSelfclose))
        return -1;
    return 0;
}

/*
    Handle the term in a description list ('foo' in ';foo:bar').
*/
static int Tokenizer_handle_dl_term(Tokenizer* self)
{
    self->topstack->context ^= LC_DLTERM;
    if (Tokenizer_READ(self, 0) == ':')
        return Tokenizer_handle_list_marker(self);
    return Tokenizer_emit_char(self, '\n');
}

/*
    Handle the end of the stream of wikitext.
*/
static PyObject* Tokenizer_handle_end(Tokenizer* self, int context)
{
    PyObject *token, *text, *trash;
    int single;

    if (context & AGG_FAIL) {
        if (context & LC_TAG_BODY) {
            token = PyList_GET_ITEM(self->topstack->stack, 1);
            text = PyObject_GetAttrString(token, "text");
            if (!text)
                return NULL;
            single = IS_SINGLE(text);
            Py_DECREF(text);
            if (single)
                return Tokenizer_handle_single_tag_end(self);
        }
        else if (context & AGG_DOUBLE) {
            trash = Tokenizer_pop(self);
            Py_XDECREF(trash);
        }
        return Tokenizer_fail_route(self);
    }
    return Tokenizer_pop(self);
}

/*
    Make sure we are not trying to write an invalid character. Return 0 if
    everything is safe, or -1 if the route must be failed.
*/
static int Tokenizer_verify_safe(Tokenizer* self, int context, Py_UNICODE data)
{
    if (context & LC_FAIL_NEXT)
        return -1;
    if (context & LC_WIKILINK_TITLE) {
        if (data == ']' || data == '{')
            self->topstack->context |= LC_FAIL_NEXT;
        else if (data == '\n' || data == '[' || data == '}')
            return -1;
        return 0;
    }
    if (context & LC_EXT_LINK_TITLE)
        return (data == '\n') ? -1 : 0;
    if (context & LC_TAG_CLOSE)
        return (data == '<') ? -1 : 0;
    if (context & LC_TEMPLATE_NAME) {
        if (data == '{' || data == '}' || data == '[') {
            self->topstack->context |= LC_FAIL_NEXT;
            return 0;
        }
        if (data == ']') {
            return -1;
        }
        if (data == '|')
            return 0;
        if (context & LC_HAS_TEXT) {
            if (context & LC_FAIL_ON_TEXT) {
                if (!Py_UNICODE_ISSPACE(data))
                    return -1;
            }
            else {
                if (data == '\n')
                    self->topstack->context |= LC_FAIL_ON_TEXT;
            }
        }
        else if (!Py_UNICODE_ISSPACE(data))
            self->topstack->context |= LC_HAS_TEXT;
    }
    else {
        if (context & LC_FAIL_ON_EQUALS) {
            if (data == '=') {
                return -1;
            }
        }
        else if (context & LC_FAIL_ON_LBRACE) {
            if (data == '{' || (Tokenizer_READ(self, -1) == '{' &&
                                 Tokenizer_READ(self, -2) == '{')) {
                if (context & LC_TEMPLATE)
                    self->topstack->context |= LC_FAIL_ON_EQUALS;
                else
                    self->topstack->context |= LC_FAIL_NEXT;
                return 0;
            }
            self->topstack->context ^= LC_FAIL_ON_LBRACE;
        }
        else if (context & LC_FAIL_ON_RBRACE) {
            if (data == '}') {
                if (context & LC_TEMPLATE)
                    self->topstack->context |= LC_FAIL_ON_EQUALS;
                else
                    self->topstack->context |= LC_FAIL_NEXT;
                return 0;
            }
            self->topstack->context ^= LC_FAIL_ON_RBRACE;
        }
        else if (data == '{')
            self->topstack->context |= LC_FAIL_ON_LBRACE;
        else if (data == '}')
            self->topstack->context |= LC_FAIL_ON_RBRACE;
    }
    return 0;
}

/*
    Parse the wikicode string, using context for when to stop. If push is true,
    we will push a new context, otherwise we won't and context will be ignored.
*/
static PyObject* Tokenizer_parse(Tokenizer* self, int context, int push)
{
    int this_context;
    Py_UNICODE this, next, next_next, last;
    PyObject* temp;

    if (push) {
        if (Tokenizer_push(self, context))
            return NULL;
    }
    while (1) {
        this = Tokenizer_READ(self, 0);
        this_context = self->topstack->context;
        if (this_context & AGG_UNSAFE) {
            if (Tokenizer_verify_safe(self, this_context, this) < 0) {
                if (this_context & AGG_DOUBLE) {
                    temp = Tokenizer_pop(self);
                    Py_XDECREF(temp);
                }
                return Tokenizer_fail_route(self);
            }
        }
        if (!is_marker(this)) {
            if (Tokenizer_emit_char(self, this))
                return NULL;
            self->head++;
            continue;
        }
        if (!this)
            return Tokenizer_handle_end(self, this_context);
        next = Tokenizer_READ(self, 1);
        last = Tokenizer_READ_BACKWARDS(self, 1);
        if (this == next && next == '{') {
            if (Tokenizer_CAN_RECURSE(self)) {
                if (Tokenizer_parse_template_or_argument(self))
                    return NULL;
            }
            else if (Tokenizer_emit_char(self, this))
                return NULL;
        }
        else if (this == '|' && this_context & LC_TEMPLATE) {
            if (Tokenizer_handle_template_param(self))
                return NULL;
        }
        else if (this == '=' && this_context & LC_TEMPLATE_PARAM_KEY) {
            if (Tokenizer_handle_template_param_value(self))
                return NULL;
        }
        else if (this == next && next == '}' && this_context & LC_TEMPLATE)
            return Tokenizer_handle_template_end(self);
        else if (this == '|' && this_context & LC_ARGUMENT_NAME) {
            if (Tokenizer_handle_argument_separator(self))
                return NULL;
        }
        else if (this == next && next == '}' && this_context & LC_ARGUMENT) {
            if (Tokenizer_READ(self, 2) == '}') {
                return Tokenizer_handle_argument_end(self);
            }
            if (Tokenizer_emit_char(self, this))
                return NULL;
        }
        else if (this == next && next == '[' && Tokenizer_CAN_RECURSE(self)) {
            if (!(this_context & AGG_NO_WIKILINKS)) {
                if (Tokenizer_parse_wikilink(self))
                    return NULL;
            }
            else if (Tokenizer_emit_char(self, this))
                return NULL;
        }
        else if (this == '|' && this_context & LC_WIKILINK_TITLE) {
            if (Tokenizer_handle_wikilink_separator(self))
                return NULL;
        }
        else if (this == next && next == ']' && this_context & LC_WIKILINK)
            return Tokenizer_handle_wikilink_end(self);
        else if (this == '[') {
            if (Tokenizer_parse_external_link(self, 1))
                return NULL;
        }
        else if (this == ':' && !is_marker(last)) {
            if (Tokenizer_parse_external_link(self, 0))
                return NULL;
        }
        else if (this == ']' && this_context & LC_EXT_LINK_TITLE)
            return Tokenizer_pop(self);
        else if (this == '=' && !(self->global & GL_HEADING)) {
            if (!last || last == '\n') {
                if (Tokenizer_parse_heading(self))
                    return NULL;
            }
            else if (Tokenizer_emit_char(self, this))
                return NULL;
        }
        else if (this == '=' && this_context & LC_HEADING)
            return (PyObject*) Tokenizer_handle_heading_end(self);
        else if (this == '\n' && this_context & LC_HEADING)
            return Tokenizer_fail_route(self);
        else if (this == '&') {
            if (Tokenizer_parse_entity(self))
                return NULL;
        }
        else if (this == '<' && next == '!') {
            next_next = Tokenizer_READ(self, 2);
            if (next_next == Tokenizer_READ(self, 3) && next_next == '-') {
                if (Tokenizer_parse_comment(self))
                    return NULL;
            }
            else if (Tokenizer_emit_char(self, this))
                return NULL;
        }
        else if (this == '<' && next == '/' && Tokenizer_READ(self, 2)) {
            if (this_context & LC_TAG_BODY ?
                Tokenizer_handle_tag_open_close(self) :
                Tokenizer_handle_invalid_tag_start(self))
                return NULL;
        }
        else if (this == '<' && !(this_context & LC_TAG_CLOSE)) {
            if (Tokenizer_CAN_RECURSE(self)) {
                if (Tokenizer_parse_tag(self))
                    return NULL;
            }
            else if (Tokenizer_emit_char(self, this))
                return NULL;
        }
        else if (this == '>' && this_context & LC_TAG_CLOSE)
            return Tokenizer_handle_tag_close_close(self);
        else if (this == next && next == '\'' && !self->skip_style_tags) {
            temp = Tokenizer_parse_style(self);
            if (temp != Py_None)
                return temp;
        }
        else if (!last || last == '\n') {
            if (this == '#' || this == '*' || this == ';' || this == ':') {
                if (Tokenizer_handle_list(self))
                    return NULL;
            }
            else if (this == '-' && this == next &&
                     this == Tokenizer_READ(self, 2) &&
                     this == Tokenizer_READ(self, 3)) {
                if (Tokenizer_handle_hr(self))
                    return NULL;
            }
            else if (Tokenizer_emit_char(self, this))
                return NULL;
        }
        else if ((this == '\n' || this == ':') && this_context & LC_DLTERM) {
            if (Tokenizer_handle_dl_term(self))
                return NULL;
        }
        else if (Tokenizer_emit_char(self, this))
            return NULL;
        self->head++;
    }
}

/*
    Build a list of tokens from a string of wikicode and return it.
*/
static PyObject* Tokenizer_tokenize(Tokenizer* self, PyObject* args)
{
    PyObject *text, *temp, *tokens;
    int context = 0, skip_style_tags = 0;

    if (PyArg_ParseTuple(args, "U|ii", &text, &context, &skip_style_tags)) {
        Py_XDECREF(self->text);
        self->text = PySequence_Fast(text, "expected a sequence");
    }
    else {
        const char* encoded;
        Py_ssize_t size;
        /* Failed to parse a Unicode object; try a string instead. */
        PyErr_Clear();
        if (!PyArg_ParseTuple(args, "s#|ii", &encoded, &size, &context,
                              &skip_style_tags))
            return NULL;
        temp = PyUnicode_FromStringAndSize(encoded, size);
        if (!text)
            return NULL;
        Py_XDECREF(self->text);
        text = PySequence_Fast(temp, "expected a sequence");
        Py_XDECREF(temp);
        self->text = text;
    }

    self->head = self->global = self->depth = self->cycles = 0;
    self->length = PyList_GET_SIZE(self->text);
    self->skip_style_tags = skip_style_tags;
    tokens = Tokenizer_parse(self, context, 1);

    if (!tokens && !PyErr_Occurred()) {
        if (!ParserError) {
            if (load_exceptions())
                return NULL;
        }
        if (BAD_ROUTE) {
            RESET_ROUTE();
            PyErr_SetString(ParserError, "C tokenizer exited with BAD_ROUTE");
        }
        else
            PyErr_SetString(ParserError, "C tokenizer exited unexpectedly");
        return NULL;
    }
    return tokens;
}

static int load_entities(void)
{
    PyObject *tempmod, *defmap, *deflist;
    unsigned numdefs, i;
#ifdef IS_PY3K
    PyObject *string;
#endif

    tempmod = PyImport_ImportModule(ENTITYDEFS_MODULE);
    if (!tempmod)
        return -1;
    defmap = PyObject_GetAttrString(tempmod, "entitydefs");
    if (!defmap)
        return -1;
    Py_DECREF(tempmod);
    deflist = PyDict_Keys(defmap);
    if (!deflist)
        return -1;
    Py_DECREF(defmap);
    numdefs = (unsigned) PyList_GET_SIZE(defmap);
    entitydefs = calloc(numdefs + 1, sizeof(char*));
    if (!entitydefs)
        return -1;
    for (i = 0; i < numdefs; i++) {
#ifdef IS_PY3K
        string = PyUnicode_AsASCIIString(PyList_GET_ITEM(deflist, i));
        if (!string)
            return -1;
        entitydefs[i] = PyBytes_AsString(string);
#else
        entitydefs[i] = PyBytes_AsString(PyList_GET_ITEM(deflist, i));
#endif
        if (!entitydefs[i])
            return -1;
    }
    Py_DECREF(deflist);
    return 0;
}

static int load_tokens(void)
{
    PyObject *tempmod, *tokens,
             *globals = PyEval_GetGlobals(),
             *locals = PyEval_GetLocals(),
             *fromlist = PyList_New(1),
             *modname = IMPORT_NAME_FUNC("tokens");
    char *name = "mwparserfromhell.parser";

    if (!fromlist || !modname)
        return -1;
    PyList_SET_ITEM(fromlist, 0, modname);
    tempmod = PyImport_ImportModuleLevel(name, globals, locals, fromlist, 0);
    Py_DECREF(fromlist);
    if (!tempmod)
        return -1;
    tokens = PyObject_GetAttrString(tempmod, "tokens");
    Py_DECREF(tempmod);

    Text = PyObject_GetAttrString(tokens, "Text");

    TemplateOpen = PyObject_GetAttrString(tokens, "TemplateOpen");
    TemplateParamSeparator = PyObject_GetAttrString(tokens,
                                                    "TemplateParamSeparator");
    TemplateParamEquals = PyObject_GetAttrString(tokens,
                                                 "TemplateParamEquals");
    TemplateClose = PyObject_GetAttrString(tokens, "TemplateClose");

    ArgumentOpen = PyObject_GetAttrString(tokens, "ArgumentOpen");
    ArgumentSeparator = PyObject_GetAttrString(tokens, "ArgumentSeparator");
    ArgumentClose = PyObject_GetAttrString(tokens, "ArgumentClose");

    WikilinkOpen = PyObject_GetAttrString(tokens, "WikilinkOpen");
    WikilinkSeparator = PyObject_GetAttrString(tokens, "WikilinkSeparator");
    WikilinkClose = PyObject_GetAttrString(tokens, "WikilinkClose");

    ExternalLinkOpen = PyObject_GetAttrString(tokens, "ExternalLinkOpen");
    ExternalLinkSeparator = PyObject_GetAttrString(tokens,
                                                   "ExternalLinkSeparator");
    ExternalLinkClose = PyObject_GetAttrString(tokens, "ExternalLinkClose");

    HTMLEntityStart = PyObject_GetAttrString(tokens, "HTMLEntityStart");
    HTMLEntityNumeric = PyObject_GetAttrString(tokens, "HTMLEntityNumeric");
    HTMLEntityHex = PyObject_GetAttrString(tokens, "HTMLEntityHex");
    HTMLEntityEnd = PyObject_GetAttrString(tokens, "HTMLEntityEnd");

    HeadingStart = PyObject_GetAttrString(tokens, "HeadingStart");
    HeadingEnd = PyObject_GetAttrString(tokens, "HeadingEnd");

    CommentStart = PyObject_GetAttrString(tokens, "CommentStart");
    CommentEnd = PyObject_GetAttrString(tokens, "CommentEnd");

    TagOpenOpen = PyObject_GetAttrString(tokens, "TagOpenOpen");
    TagAttrStart = PyObject_GetAttrString(tokens, "TagAttrStart");
    TagAttrEquals = PyObject_GetAttrString(tokens, "TagAttrEquals");
    TagAttrQuote = PyObject_GetAttrString(tokens, "TagAttrQuote");
    TagCloseOpen = PyObject_GetAttrString(tokens, "TagCloseOpen");
    TagCloseSelfclose = PyObject_GetAttrString(tokens, "TagCloseSelfclose");
    TagOpenClose = PyObject_GetAttrString(tokens, "TagOpenClose");
    TagCloseClose = PyObject_GetAttrString(tokens, "TagCloseClose");

    Py_DECREF(tokens);
    return 0;
}

static int load_defs(void)
{
    PyObject *tempmod,
             *globals = PyEval_GetGlobals(),
             *locals = PyEval_GetLocals(),
             *fromlist = PyList_New(1),
             *modname = IMPORT_NAME_FUNC("definitions");
    char *name = "mwparserfromhell";

    if (!fromlist || !modname)
        return -1;
    PyList_SET_ITEM(fromlist, 0, modname);
    tempmod = PyImport_ImportModuleLevel(name, globals, locals, fromlist, 0);
    Py_DECREF(fromlist);
    if (!tempmod)
        return -1;
    definitions = PyObject_GetAttrString(tempmod, "definitions");
    Py_DECREF(tempmod);
    return 0;
}

static int load_exceptions(void)
{
    PyObject *tempmod, *parsermod,
             *globals = PyEval_GetGlobals(),
             *locals = PyEval_GetLocals(),
             *fromlist = PyList_New(1),
             *modname = IMPORT_NAME_FUNC("parser");
    char *name = "mwparserfromhell";

    if (!fromlist || !modname)
        return -1;
    PyList_SET_ITEM(fromlist, 0, modname);
    tempmod = PyImport_ImportModuleLevel(name, globals, locals, fromlist, 0);
    Py_DECREF(fromlist);
    if (!tempmod)
        return -1;
    parsermod = PyObject_GetAttrString(tempmod, "parser");
    Py_DECREF(tempmod);
    ParserError = PyObject_GetAttrString(parsermod, "ParserError");
    Py_DECREF(parsermod);
    return 0;
}

PyMODINIT_FUNC INIT_FUNC_NAME(void)
{
    PyObject *module;

    TokenizerType.tp_new = PyType_GenericNew;
    if (PyType_Ready(&TokenizerType) < 0)
        INIT_ERROR;
    module = CREATE_MODULE;
    if (!module)
        INIT_ERROR;
    Py_INCREF(&TokenizerType);
    PyModule_AddObject(module, "CTokenizer", (PyObject*) &TokenizerType);
    Py_INCREF(Py_True);
    PyDict_SetItemString(TokenizerType.tp_dict, "USES_C", Py_True);
    EMPTY = PyUnicode_FromString("");
    NOARGS = PyTuple_New(0);
    if (!EMPTY || !NOARGS || load_entities() || load_tokens() || load_defs())
        INIT_ERROR;
#ifdef IS_PY3K
    return module;
#endif
}