From 487dabec9da7756d2cde83fad493099781973797 Mon Sep 17 00:00:00 2001 From: Sean Li <22557931+sean0407@users.noreply.github.com> Date: Mon, 19 Oct 2026 13:54:38 +0800 Subject: [PATCH] Fix C tokenizer handling of embedded NUL --- src/mwparserfromhell/parser/ctokenizer/tok_parse.c | 55 +++++++++++++--------- .../parser/ctokenizer/tok_support.c | 7 ++- tests/tokenizer/embedded_nul.mwtest | 11 +++++ 3 files changed, 49 insertions(+), 24 deletions(-) create mode 100644 tests/tokenizer/embedded_nul.mwtest diff --git a/src/mwparserfromhell/parser/ctokenizer/tok_parse.c b/src/mwparserfromhell/parser/ctokenizer/tok_parse.c index 72c4bed..ed42a21 100644 --- a/src/mwparserfromhell/parser/ctokenizer/tok_parse.c +++ b/src/mwparserfromhell/parser/ctokenizer/tok_parse.c @@ -56,6 +56,14 @@ is_marker(Py_UCS4 this) { int i; + // Out-of-range codepoints are tokenizer boundaries; NUL is input text. + if (this > 0x10FFFF) { + return 1; + } + if (this == '\0') { + return 0; + } + for (i = 0; i < NUM_MARKERS; i++) { if (MARKERS[i] == this) { return 1; @@ -509,7 +517,8 @@ Tokenizer_parse_bracketed_uri_scheme(Tokenizer *self) if (!buffer) { return -1; } - while ((this = Tokenizer_read(self, 0))) { + while (self->head < self->text.length) { + this = Tokenizer_read(self, 0); i = 0; while (1) { if (!valid[i]) { @@ -678,8 +687,8 @@ Tokenizer_is_uri_end(Tokenizer *self, Py_UCS4 this, Py_UCS4 next) Py_UCS4 after = Tokenizer_read(self, 2); uint64_t ctx = self->topstack->context; - return (!this || this == '\n' || this == '[' || this == ']' || this == '<' || - this == '>' || this == '"' || this == ' ' || + return (self->head >= self->text.length || this == '\n' || this == '[' || + this == ']' || this == '<' || this == '>' || this == '"' || this == ' ' || (this == '\'' && next == '\'') || (this == '|' && ctx & LC_TEMPLATE) || (this == '=' && ctx & (LC_TEMPLATE_PARAM_KEY | LC_HEADING)) || (this == '}' && next == '}' && @@ -703,7 +712,8 @@ Tokenizer_really_parse_external_link(Tokenizer *self, int brackets, Textbuffer * return NULL; } this = Tokenizer_read(self, 0); - if (!this || this == '\n' || this == ' ' || this == ']') { + if (self->head >= self->text.length || this == '\n' || this == ' ' || + this == ']') { return Tokenizer_fail_route(self); } if (!brackets && this == '[') { @@ -729,7 +739,7 @@ Tokenizer_really_parse_external_link(Tokenizer *self, int brackets, Textbuffer * return NULL; } } else if (brackets) { - if (!this || this == '\n') { + if (self->head >= self->text.length || this == '\n') { return Tokenizer_fail_route(self); } if (this == ']') { @@ -1040,7 +1050,7 @@ Tokenizer_really_parse_entity(Tokenizer *self) } self->head++; this = Tokenizer_read(self, 0); - if (!this) { + if (self->head >= self->text.length) { Tokenizer_fail_route(self); return 0; } @@ -1051,7 +1061,7 @@ Tokenizer_really_parse_entity(Tokenizer *self) } self->head++; this = Tokenizer_read(self, 0); - if (!this) { + if (self->head >= self->text.length) { Tokenizer_fail_route(self); return 0; } @@ -1233,7 +1243,7 @@ Tokenizer_parse_comment(Tokenizer *self) } while (1) { this = Tokenizer_read(self, 0); - if (!this) { + if (self->head >= self->text.length) { comment = Tokenizer_pop(self); Py_XDECREF(comment); self->head = reset; @@ -1597,7 +1607,7 @@ Tokenizer_handle_blacklisted_tag(Tokenizer *self) while (1) { this = Tokenizer_read(self, 0); next = Tokenizer_read(self, 1); - if (!this) { + if (self->head >= self->text.length) { return Tokenizer_fail_route(self); } else if (this == '<' && next == '/') { self->head += 2; @@ -1639,7 +1649,7 @@ Tokenizer_handle_blacklisted_tag(Tokenizer *self) } return Tokenizer_pop(self); } - if (!this || this == '\n') { + if (self->head >= self->text.length || this == '\n') { no_matching_end: Textbuffer_dealloc(buffer); self->head = reset; @@ -1789,7 +1799,7 @@ Tokenizer_really_parse_tag(Tokenizer *self) next = Tokenizer_read(self, 1); can_exit = (!(data->context & (TAG_QUOTED | TAG_NAME)) || data->context & TAG_NOTE_SPACE); - if (!this) { + if (self->head >= self->text.length) { if (self->topstack->context & LC_TAG_ATTR) { if (data->context & TAG_QUOTED) { // Unclosed attribute quote: reset, don't die @@ -2441,7 +2451,7 @@ Tokenizer_handle_table_style(Tokenizer *self, Py_UCS4 end_token) return NULL; } return padding; - } else if (!this || this == end_token) { + } else if (self->head >= self->text.length || this == end_token) { if (self->topstack->context & LC_TAG_ATTR) { if (data->context & TAG_QUOTED) { // Unclosed attribute quote: reset, don't die @@ -2844,7 +2854,7 @@ Tokenizer_has_leading_whitespace(Tokenizer *self) Py_UCS4 current_character; while (1) { current_character = Tokenizer_read_backwards(self, offset); - if (!current_character || current_character == '\n') { + if (offset > self->head || current_character == '\n') { return 1; } else if (!Py_UNICODE_ISSPACE(current_character)) { return 0; @@ -2876,7 +2886,8 @@ Tokenizer_parse(Tokenizer *self, uint64_t context, int push) this = Tokenizer_read(self, 0); this_context = self->topstack->context; if (this_context & AGG_UNSAFE) { - if (Tokenizer_verify_safe(self, this_context, this) < 0) { + Py_UCS4 data = self->head >= self->text.length ? '\0' : this; + if (Tokenizer_verify_safe(self, this_context, data) < 0) { if (this_context & AGG_DOUBLE) { temp = Tokenizer_pop(self); Py_XDECREF(temp); @@ -2884,6 +2895,9 @@ Tokenizer_parse(Tokenizer *self, uint64_t context, int push) return Tokenizer_fail_route(self); } } + if (self->head >= self->text.length) { + return Tokenizer_handle_end(self, this_context); + } if (!is_marker(this)) { if (Tokenizer_emit_char(self, this)) { return NULL; @@ -2891,9 +2905,6 @@ Tokenizer_parse(Tokenizer *self, uint64_t context, int push) self->head++; continue; } - if (!this) { - return Tokenizer_handle_end(self, this_context); - } if (PyErr_CheckSignals()) { return NULL; } @@ -2912,7 +2923,7 @@ Tokenizer_parse(Tokenizer *self, uint64_t context, int push) return NULL; } } else if (this == '=' && this_context & LC_TEMPLATE_PARAM_KEY) { - if (!(self->global & GL_HEADING) && (!last || last == '\n') && + if (!(self->global & GL_HEADING) && (self->head == 0 || last == '\n') && next == '=') { if (Tokenizer_parse_heading(self)) { return NULL; @@ -2963,7 +2974,7 @@ Tokenizer_parse(Tokenizer *self, uint64_t context, int push) return Tokenizer_pop(self); } else if (this == '=' && !(self->global & GL_HEADING) && !(this_context & LC_TEMPLATE)) { - if (!last || last == '\n') { + if (self->head == 0 || last == '\n') { if (Tokenizer_parse_heading(self)) { return NULL; } @@ -2987,7 +2998,7 @@ Tokenizer_parse(Tokenizer *self, uint64_t context, int push) } else if (Tokenizer_emit_char(self, this)) { return NULL; } - } else if (this == '<' && next == '/' && Tokenizer_read(self, 2)) { + } else if (this == '<' && next == '/' && self->head + 2 < self->text.length) { if (this_context & LC_TAG_BODY ? Tokenizer_handle_tag_open_close(self) : Tokenizer_handle_invalid_tag_start(self)) { return NULL; @@ -3007,12 +3018,12 @@ Tokenizer_parse(Tokenizer *self, uint64_t context, int push) if (temp != Py_None) { return temp; } - } else if ((!last || last == '\n') && + } else if ((self->head == 0 || last == '\n') && (this == '#' || this == '*' || this == ';' || this == ':')) { if (Tokenizer_handle_list(self)) { return NULL; } - } else if ((!last || last == '\n') && + } else if ((self->head == 0 || last == '\n') && (this == '-' && this == next && this == Tokenizer_read(self, 2) && this == Tokenizer_read(self, 3))) { if (Tokenizer_handle_hr(self)) { diff --git a/src/mwparserfromhell/parser/ctokenizer/tok_support.c b/src/mwparserfromhell/parser/ctokenizer/tok_support.c index 4c495ce..15aa53b 100644 --- a/src/mwparserfromhell/parser/ctokenizer/tok_support.c +++ b/src/mwparserfromhell/parser/ctokenizer/tok_support.c @@ -24,6 +24,9 @@ SOFTWARE. #include "textbuffer.h" #include "tokens.h" +/* Outside the Unicode range, so it cannot collide with input text. */ +#define TOKENIZER_BOUNDARY 0x110000 + /* Add a new token stack, context, and textbuffer to the list. */ @@ -442,7 +445,7 @@ Tokenizer_read(Tokenizer *self, Py_ssize_t delta) Py_ssize_t index = self->head + delta; if (index >= self->text.length) { - return '\0'; + return TOKENIZER_BOUNDARY; } return read_codepoint(&self->text, index); } @@ -456,7 +459,7 @@ Tokenizer_read_backwards(Tokenizer *self, Py_ssize_t delta) Py_ssize_t index; if (delta > self->head) { - return '\0'; + return TOKENIZER_BOUNDARY; } index = self->head - delta; return read_codepoint(&self->text, index); diff --git a/tests/tokenizer/embedded_nul.mwtest b/tests/tokenizer/embedded_nul.mwtest new file mode 100644 index 0000000..030568c --- /dev/null +++ b/tests/tokenizer/embedded_nul.mwtest @@ -0,0 +1,11 @@ +name: embedded NUL in plain text +input: "a\x00b" +output: [Text(text="a\x00b")] +--- +name: embedded NUL in a longer string +input: "hello\x00world" +output: [Text(text="hello\x00world")] +--- +name: embedded NUL inside a template +input: "{{a\x00b}}" +output: [TemplateOpen(), Text(text="a\x00b"), TemplateClose()]