From 0f5f898bb76c18331d588794c1a84ade74a1afe7 Mon Sep 17 00:00:00 2001 From: June Kim Date: Mon, 11 May 2026 22:27:11 -0700 Subject: [PATCH] fix: replace remaining !last checks with TOKENIZER_EOF comparison Four start-of-input checks in the main parse loop still used !last (falsy NUL) to detect beginning-of-input. After the TOKENIZER_EOF sentinel change, Tokenizer_read_backwards returns 0x110000 instead of '\0', so !last is always false and headings, lists, and horizontal rules at position 0 would silently fail to parse. --- src/mwparserfromhell/parser/ctokenizer/tok_parse.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/mwparserfromhell/parser/ctokenizer/tok_parse.c b/src/mwparserfromhell/parser/ctokenizer/tok_parse.c index 999df04..04d252a 100644 --- a/src/mwparserfromhell/parser/ctokenizer/tok_parse.c +++ b/src/mwparserfromhell/parser/ctokenizer/tok_parse.c @@ -2913,7 +2913,7 @@ Tokenizer_parse(Tokenizer *self, uint64_t context, int push) return NULL; } } else if (this == '=' && this_context & LC_TEMPLATE_PARAM_KEY) { - if (!(self->global & GL_HEADING) && (!last || last == '\n') && + if (!(self->global & GL_HEADING) && (last == TOKENIZER_EOF || last == '\n') && next == '=') { if (Tokenizer_parse_heading(self)) { return NULL; @@ -2964,7 +2964,7 @@ Tokenizer_parse(Tokenizer *self, uint64_t context, int push) return Tokenizer_pop(self); } else if (this == '=' && !(self->global & GL_HEADING) && !(this_context & LC_TEMPLATE)) { - if (!last || last == '\n') { + if (last == TOKENIZER_EOF || last == '\n') { if (Tokenizer_parse_heading(self)) { return NULL; } @@ -3008,12 +3008,12 @@ Tokenizer_parse(Tokenizer *self, uint64_t context, int push) if (temp != Py_None) { return temp; } - } else if ((!last || last == '\n') && + } else if ((last == TOKENIZER_EOF || last == '\n') && (this == '#' || this == '*' || this == ';' || this == ':')) { if (Tokenizer_handle_list(self)) { return NULL; } - } else if ((!last || last == '\n') && + } else if ((last == TOKENIZER_EOF || last == '\n') && (this == '-' && this == next && this == Tokenizer_read(self, 2) && this == Tokenizer_read(self, 3))) { if (Tokenizer_handle_hr(self)) {