Add a USES_C field to the tokenizers; add TestParser.test_use_c()

mwparserfromhell/parser/tokenizer.c

@@ -1387,6 +1387,8 @@ init_tokenizer(void)
     module = Py_InitModule("_tokenizer", module_methods);
     Py_INCREF(&TokenizerType);
     PyModule_AddObject(module, "CTokenizer", (PyObject*) &TokenizerType);
+    Py_INCREF(Py_True);
+    PyDict_SetItemString(TokenizerType.tp_dict, "USES_C", Py_True);
 
     tempmod = PyImport_ImportModule("htmlentitydefs");
     if (!tempmod)

mwparserfromhell/parser/tokenizer.py

@@ -38,6 +38,7 @@ class BadRoute(Exception):
 
 class Tokenizer(object):
     """Creates a list of tokens from a string of wikicode."""
+    USES_C = False
     START = object()
     END = object()
     MARKERS = ["{", "}", "[", "]", "<", ">", "|", "=", "&", "#", "*", ";", ":",

tests/test_parser.py

@@ -23,10 +23,10 @@
 from __future__ import unicode_literals
 import unittest
 
+from mwparserfromhell import parser
 from mwparserfromhell.compat import range
 from mwparserfromhell.nodes import Template, Text, Wikilink
 from mwparserfromhell.nodes.extras import Parameter
-from mwparserfromhell.parser import Parser
 from mwparserfromhell.smart_list import SmartList
 from mwparserfromhell.wikicode import Wikicode
 
@@ -63,7 +63,14 @@ class TestParser(unittest.TestCase):
         for i in range(length):
             self.assertNodesEqual(expected.get(i), actual.get(i))
 
-    def test_parser(self):
+    def test_use_c(self):
+        """make sure the correct tokenizer is used"""
+        if parser.use_c:
+            self.assertTrue(parser.Parser(None)._tokenizer.USES_C)
+            parser.use_c = False
+        self.assertFalse(parser.Parser(None)._tokenizer.USES_C)
+
+    def test_parsing(self):
         """integration test for parsing overall"""
         text = "this is text; {{this|is=a|template={{with|[[links]]|in}}it}}"
         wrap = lambda L: Wikicode(SmartList(L))
@@ -83,7 +90,7 @@ class TestParser(unittest.TestCase):
                 ]))
             ])
         ])
-        actual = Parser(text).parse()
+        actual = parser.Parser(text).parse()
         self.assertWikicodeEqual(expected, actual)
 
 if __name__ == "__main__":