From 669edbcefa2c52673a4447f41b1f23e600c946ca Mon Sep 17 00:00:00 2001 From: June Kim Date: Tue, 12 May 2026 01:29:14 -0700 Subject: [PATCH] fix: handle TOKENIZER_EOF in invalid tag start loop and bare colon check MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two bugs introduced by the NUL truncation fix: 1. Tokenizer_handle_invalid_tag_start: the tag name scanning loop checked is_marker() and Py_UNICODE_ISSPACE() to terminate, but TOKENIZER_EOF (0x110000) matches neither, causing an infinite loop when an incomplete closing tag like "". 2. Tokenizer_parse colon handling: the bare external link check "this == ':' && !is_marker(last)" fired at start-of-input because is_marker(TOKENIZER_EOF) returns false (0x110000 not in MARKERS). This intercepted ":" before the list handler could run, breaking definition list items like ":text" →
. --- src/mwparserfromhell/parser/ctokenizer/tok_parse.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/mwparserfromhell/parser/ctokenizer/tok_parse.c b/src/mwparserfromhell/parser/ctokenizer/tok_parse.c index c8f7c0e..7c4c20b 100644 --- a/src/mwparserfromhell/parser/ctokenizer/tok_parse.c +++ b/src/mwparserfromhell/parser/ctokenizer/tok_parse.c @@ -1863,7 +1863,7 @@ Tokenizer_handle_invalid_tag_start(Tokenizer *self) } while (1) { this = Tokenizer_read(self, pos); - if (Py_UNICODE_ISSPACE(this) || is_marker(this)) { + if (this == TOKENIZER_EOF || Py_UNICODE_ISSPACE(this) || is_marker(this)) { name = Textbuffer_render(buf); if (!name) { Textbuffer_dealloc(buf); @@ -2956,7 +2956,7 @@ Tokenizer_parse(Tokenizer *self, uint64_t context, int push) if (Tokenizer_parse_external_link(self, 1)) { return NULL; } - } else if (this == ':' && !is_marker(last)) { + } else if (this == ':' && last != TOKENIZER_EOF && !is_marker(last)) { if (Tokenizer_parse_external_link(self, 0)) { return NULL; }