diff --git a/mwparserfromhell/parser/tokenizer.c b/mwparserfromhell/parser/tokenizer.c index 88ca3f2..8dace5a 100644 --- a/mwparserfromhell/parser/tokenizer.c +++ b/mwparserfromhell/parser/tokenizer.c @@ -835,11 +835,7 @@ static int Tokenizer_parse_heading(Tokenizer* self) self->global ^= GL_HEADING; return 0; } -#ifdef IS_PY3K - level = PyLong_FromSsize_t(heading->level); -#else - level = PyInt_FromSsize_t(heading->level); -#endif + level = NEW_INT_FUNC(heading->level); if (!level) { Py_DECREF(heading->title); free(heading); @@ -2279,7 +2275,11 @@ static PyObject* Tokenizer_tokenize(Tokenizer* self, PyObject* args) { PyObject *text, *temp; - if (!PyArg_ParseTuple(args, "U", &text)) { + if (PyArg_ParseTuple(args, "U", &text)) { + Py_XDECREF(self->text); + self->text = PySequence_Fast(text, "expected a sequence"); + } + else { const char* encoded; Py_ssize_t size; /* Failed to parse a Unicode object; try a string instead. */ @@ -2294,10 +2294,6 @@ static PyObject* Tokenizer_tokenize(Tokenizer* self, PyObject* args) Py_XDECREF(temp); self->text = text; } - else { - Py_XDECREF(self->text); - self->text = PySequence_Fast(text, "expected a sequence"); - } self->head = self->global = self->depth = self->cycles = 0; self->length = PyList_GET_SIZE(self->text); return Tokenizer_parse(self, 0, 1); diff --git a/mwparserfromhell/parser/tokenizer.h b/mwparserfromhell/parser/tokenizer.h index 1229688..264360e 100644 --- a/mwparserfromhell/parser/tokenizer.h +++ b/mwparserfromhell/parser/tokenizer.h @@ -255,12 +255,14 @@ static PyObject* Tokenizer_tokenize(Tokenizer*, PyObject*); /* Macros for Python 2/3 compatibility: */ #ifdef IS_PY3K + #define NEW_INT_FUNC PyLong_FromSsize_t #define IMPORT_NAME_FUNC PyUnicode_FromString #define CREATE_MODULE PyModule_Create(&module_def); #define ENTITYDEFS_MODULE "html.entities" #define INIT_FUNC_NAME PyInit__tokenizer #define INIT_ERROR return NULL #else + #define NEW_INT_FUNC PyInt_FromSsize_t #define IMPORT_NAME_FUNC PyBytes_FromString #define CREATE_MODULE Py_InitModule("_tokenizer", NULL); #define ENTITYDEFS_MODULE "htmlentitydefs"