From f8032695146f032108c1b736631f546712689372 Mon Sep 17 00:00:00 2001
From: Ben Kurtovic <ben.kurtovic@verizon.net>
Date: Wed, 27 Mar 2013 17:19:08 -0400
Subject: [PATCH] Add a USES_C field to the tokenizers; add
 TestParser.test_use_c()

---
 mwparserfromhell/parser/tokenizer.c  |  2 ++
 mwparserfromhell/parser/tokenizer.py |  1 +
 tests/test_parser.py                 | 13 ++++++++++---
 3 files changed, 13 insertions(+), 3 deletions(-)

diff --git a/mwparserfromhell/parser/tokenizer.c b/mwparserfromhell/parser/tokenizer.c
index 8c96500..d3abb22 100644
--- a/mwparserfromhell/parser/tokenizer.c
+++ b/mwparserfromhell/parser/tokenizer.c
@@ -1387,6 +1387,8 @@ init_tokenizer(void)
     module = Py_InitModule("_tokenizer", module_methods);
     Py_INCREF(&TokenizerType);
     PyModule_AddObject(module, "CTokenizer", (PyObject*) &TokenizerType);
+    Py_INCREF(Py_True);
+    PyDict_SetItemString(TokenizerType.tp_dict, "USES_C", Py_True);
 
     tempmod = PyImport_ImportModule("htmlentitydefs");
     if (!tempmod)
diff --git a/mwparserfromhell/parser/tokenizer.py b/mwparserfromhell/parser/tokenizer.py
index 67638ca..0bf0322 100644
--- a/mwparserfromhell/parser/tokenizer.py
+++ b/mwparserfromhell/parser/tokenizer.py
@@ -38,6 +38,7 @@ class BadRoute(Exception):
 
 class Tokenizer(object):
     """Creates a list of tokens from a string of wikicode."""
+    USES_C = False
     START = object()
     END = object()
     MARKERS = ["{", "}", "[", "]", "<", ">", "|", "=", "&", "#", "*", ";", ":",
diff --git a/tests/test_parser.py b/tests/test_parser.py
index 6e775ce..4f718c8 100644
--- a/tests/test_parser.py
+++ b/tests/test_parser.py
@@ -23,10 +23,10 @@
 from __future__ import unicode_literals
 import unittest
 
+from mwparserfromhell import parser
 from mwparserfromhell.compat import range
 from mwparserfromhell.nodes import Template, Text, Wikilink
 from mwparserfromhell.nodes.extras import Parameter
-from mwparserfromhell.parser import Parser
 from mwparserfromhell.smart_list import SmartList
 from mwparserfromhell.wikicode import Wikicode
 
@@ -63,7 +63,14 @@ class TestParser(unittest.TestCase):
         for i in range(length):
             self.assertNodesEqual(expected.get(i), actual.get(i))
 
-    def test_parser(self):
+    def test_use_c(self):
+        """make sure the correct tokenizer is used"""
+        if parser.use_c:
+            self.assertTrue(parser.Parser(None)._tokenizer.USES_C)
+            parser.use_c = False
+        self.assertFalse(parser.Parser(None)._tokenizer.USES_C)
+
+    def test_parsing(self):
         """integration test for parsing overall"""
         text = "this is text; {{this|is=a|template={{with|[[links]]|in}}it}}"
         wrap = lambda L: Wikicode(SmartList(L))
@@ -83,7 +90,7 @@ class TestParser(unittest.TestCase):
                 ]))
             ])
         ])
-        actual = Parser(text).parse()
+        actual = parser.Parser(text).parse()
         self.assertWikicodeEqual(expected, actual)
 
 if __name__ == "__main__":