Fix C tokenizer NUL byte truncation (issue #355)

The C tokenizer was silently truncating input at the first NUL byte (\x00)
because it used '\0' both as a valid input character and as the EOF sentinel.

Root cause:
- Tokenizer_read() returned '\0' for both:
  1. Real NUL bytes in the input
  2. End-of-input (when index >= text.length)
- This made them indistinguishable, causing real NULs to be treated as EOF

Fix:
1. Define TOKENIZER_EOF as 0x110000 (first invalid Unicode code point)
2. Update Tokenizer_read() and Tokenizer_read_backwards() to return
   TOKENIZER_EOF instead of '\0' for out-of-bounds reads
3. Replace all `!this` and `'\0'` checks with explicit `TOKENIZER_EOF` checks
4. Remove '\0' from the MARKERS array (no longer needed as EOF marker)
5. Move EOF check before is_marker() in main parse loop to ensure
   TOKENIZER_EOF doesn't try to emit as a character
6. Fix Tokenizer_has_leading_whitespace() to recognize TOKENIZER_EOF

The Python tokenizer already preserved NUL bytes correctly; this brings
the C tokenizer into parity.

Regression test added: test_nul_byte_preservation() verifies that both
tokenizers now preserve NUL bytes in plain text, templates, and
multiple-NUL scenarios.

src/mwparserfromhell/parser/ctokenizer/common.h

@@ -47,6 +47,9 @@ SOFTWARE.
 #define PyUnicode_FROM_SINGLE(chr)                                                     \
     PyUnicode_FromKindAndData(PyUnicode_4BYTE_KIND, &(chr), 1)
 
+/* EOF sentinel: use a value beyond valid Unicode range (0x110000 is first invalid) */
+#define TOKENIZER_EOF ((Py_UCS4) 0x110000)
+
 /* Error handling macros */
 
 #define BAD_ROUTE         self->route_state

src/mwparserfromhell/parser/ctokenizer/tok_parse.c

@@ -509,7 +509,7 @@ Tokenizer_parse_bracketed_uri_scheme(Tokenizer *self)
         if (!buffer) {
             return -1;
         }
-        while ((this = Tokenizer_read(self, 0))) {
+        while ((this = Tokenizer_read(self, 0)) != TOKENIZER_EOF) {
             i = 0;
             while (1) {
                 if (!valid[i]) {
@@ -678,7 +678,7 @@ Tokenizer_is_uri_end(Tokenizer *self, Py_UCS4 this, Py_UCS4 next)
     Py_UCS4 after = Tokenizer_read(self, 2);
     uint64_t ctx = self->topstack->context;
 
-    return (!this || this == '\n' || this == '[' || this == ']' || this == '<' ||
+    return (this == TOKENIZER_EOF || this == '\n' || this == '[' || this == ']' || this == '<' ||
             this == '>' || this == '"' || this == ' ' ||
             (this == '\'' && next == '\'') || (this == '|' && ctx & LC_TEMPLATE) ||
             (this == '=' && ctx & (LC_TEMPLATE_PARAM_KEY | LC_HEADING)) ||
@@ -703,7 +703,7 @@ Tokenizer_really_parse_external_link(Tokenizer *self, int brackets, Textbuffer *
         return NULL;
     }
     this = Tokenizer_read(self, 0);
-    if (!this || this == '\n' || this == ' ' || this == ']') {
+    if (this == TOKENIZER_EOF || this == '\n' || this == ' ' || this == ']') {
         return Tokenizer_fail_route(self);
     }
     if (!brackets && this == '[') {
@@ -729,7 +729,7 @@ Tokenizer_really_parse_external_link(Tokenizer *self, int brackets, Textbuffer *
                 return NULL;
             }
         } else if (brackets) {
-            if (!this || this == '\n') {
+            if (this == TOKENIZER_EOF || this == '\n') {
                 return Tokenizer_fail_route(self);
             }
             if (this == ']') {
@@ -1040,7 +1040,7 @@ Tokenizer_really_parse_entity(Tokenizer *self)
     }
     self->head++;
     this = Tokenizer_read(self, 0);
-    if (!this) {
+    if (this == TOKENIZER_EOF) {
         Tokenizer_fail_route(self);
         return 0;
     }
@@ -1051,7 +1051,7 @@ Tokenizer_really_parse_entity(Tokenizer *self)
         }
         self->head++;
         this = Tokenizer_read(self, 0);
-        if (!this) {
+        if (this == TOKENIZER_EOF) {
             Tokenizer_fail_route(self);
             return 0;
         }
@@ -1233,7 +1233,7 @@ Tokenizer_parse_comment(Tokenizer *self)
     }
     while (1) {
         this = Tokenizer_read(self, 0);
-        if (!this) {
+        if (this == TOKENIZER_EOF) {
             comment = Tokenizer_pop(self);
             Py_XDECREF(comment);
             self->head = reset;
@@ -1597,7 +1597,7 @@ Tokenizer_handle_blacklisted_tag(Tokenizer *self)
     while (1) {
         this = Tokenizer_read(self, 0);
         next = Tokenizer_read(self, 1);
-        if (!this) {
+        if (this == TOKENIZER_EOF) {
             return Tokenizer_fail_route(self);
         } else if (this == '<' && next == '/') {
             self->head += 2;
@@ -1639,7 +1639,7 @@ Tokenizer_handle_blacklisted_tag(Tokenizer *self)
                     }
                     return Tokenizer_pop(self);
                 }
-                if (!this || this == '\n') {
+                if (this == TOKENIZER_EOF || this == '\n') {
                 no_matching_end:
                     Textbuffer_dealloc(buffer);
                     self->head = reset;
@@ -1790,7 +1790,7 @@ Tokenizer_really_parse_tag(Tokenizer *self)
         next = Tokenizer_read(self, 1);
         can_exit = (!(data->context & (TAG_QUOTED | TAG_NAME)) ||
                     data->context & TAG_NOTE_SPACE);
-        if (!this) {
+        if (this == TOKENIZER_EOF) {
             if (self->topstack->context & LC_TAG_ATTR) {
                 if (data->context & TAG_QUOTED) {
                     // Unclosed attribute quote: reset, don't die
@@ -2442,7 +2442,7 @@ Tokenizer_handle_table_style(Tokenizer *self, Py_UCS4 end_token)
                 return NULL;
             }
             return padding;
-        } else if (!this || this == end_token) {
+        } else if (this == TOKENIZER_EOF || this == end_token) {
             if (self->topstack->context & LC_TAG_ATTR) {
                 if (data->context & TAG_QUOTED) {
                     // Unclosed attribute quote: reset, don't die
@@ -2845,7 +2845,7 @@ Tokenizer_has_leading_whitespace(Tokenizer *self)
     Py_UCS4 current_character;
     while (1) {
         current_character = Tokenizer_read_backwards(self, offset);
-        if (!current_character || current_character == '\n') {
+        if (current_character == TOKENIZER_EOF || current_character == '\n') {
             return 1;
         } else if (!Py_UNICODE_ISSPACE(current_character)) {
             return 0;
@@ -2876,6 +2876,9 @@ Tokenizer_parse(Tokenizer *self, uint64_t context, int push)
     while (1) {
         this = Tokenizer_read(self, 0);
         this_context = self->topstack->context;
+        if (this == TOKENIZER_EOF) {
+            return Tokenizer_handle_end(self, this_context);
+        }
         if (this_context & AGG_UNSAFE) {
             if (Tokenizer_verify_safe(self, this_context, this) < 0) {
                 if (this_context & AGG_DOUBLE) {
@@ -2892,9 +2895,6 @@ Tokenizer_parse(Tokenizer *self, uint64_t context, int push)
             self->head++;
             continue;
         }
-        if (!this) {
-            return Tokenizer_handle_end(self, this_context);
-        }
         if (PyErr_CheckSignals()) {
             return NULL;
         }

src/mwparserfromhell/parser/ctokenizer/tok_parse.h

@@ -25,11 +25,11 @@ SOFTWARE.
 #include "common.h"
 
 static const Py_UCS4 MARKERS[] = {
-    '{', '}', '[', ']', '<', '>', '|', '=',  '&',  '\'',
-    '#', '*', ';', ':', '/', '-', '!', '\n', '\0',
+    '{', '}', '[', ']', '<', '>', '|', '=', '&', '\'',
+    '#', '*', ';', ':', '/', '-', '!', '\n',
 };
 
-#define NUM_MARKERS 19
+#define NUM_MARKERS 18
 
 /* Functions */
 

src/mwparserfromhell/parser/ctokenizer/tok_support.c

@@ -442,7 +442,7 @@ Tokenizer_read(Tokenizer *self, Py_ssize_t delta)
     Py_ssize_t index = self->head + delta;
 
     if (index >= self->text.length) {
-        return '\0';
+        return TOKENIZER_EOF;
     }
     return read_codepoint(&self->text, index);
 }
@@ -456,7 +456,7 @@ Tokenizer_read_backwards(Tokenizer *self, Py_ssize_t delta)
     Py_ssize_t index;
 
     if (delta > self->head) {
-        return '\0';
+        return TOKENIZER_EOF;
     }
     index = self->head - delta;
     return read_codepoint(&self->text, index);

tests/test_tokenizer.py

@@ -145,3 +145,31 @@ def test_describe_context():
     assert "" == contexts.describe(0)
     ctx = contexts.describe(contexts.TEMPLATE_PARAM_KEY | contexts.HAS_TEXT)
     assert "TEMPLATE_PARAM_KEY|HAS_TEXT" == ctx
+
+
+@pytest.mark.parametrize(
+    "tokenizer",
+    filter(None, (CTokenizer, PyTokenizer)),
+    ids=lambda t: "CTokenizer" if t.USES_C else "PyTokenizer",
+)
+def test_nul_byte_preservation(tokenizer):
+    """Test that NUL bytes (\\x00) are preserved in the input instead of being treated as EOF.
+
+    Regression test for issue #355: C tokenizer was returning '\\0' both for real NUL bytes
+    in the input and for EOF, causing silent truncation at the first NUL.
+    """
+    # Basic NUL preservation
+    result = str(Builder().build(tokenizer().tokenize("a\x00b")))
+    assert result == "a\x00b", f"Expected 'a\\x00b', got {result!r}"
+
+    # NUL in the middle of text
+    result = str(Builder().build(tokenizer().tokenize("hello\x00world")))
+    assert result == "hello\x00world", f"Expected 'hello\\x00world', got {result!r}"
+
+    # NUL inside template
+    result = str(Builder().build(tokenizer().tokenize("{{a\x00b}}")))
+    assert result == "{{a\x00b}}", f"Expected '{{{{a\\x00b}}}}', got {result!r}"
+
+    # Multiple NULs
+    result = str(Builder().build(tokenizer().tokenize("a\x00b\x00c")))
+    assert result == "a\x00b\x00c", f"Expected 'a\\x00b\\x00c', got {result!r}"