Can specify specific tokenizer tests via the command line; add files

tests/_test_tokenizer.py

@@ -22,6 +22,7 @@
 
 from __future__ import print_function, unicode_literals
 from os import listdir, path
+import sys
 
 from mwparserfromhell.compat import py3k
 from mwparserfromhell.parser import tokens
@@ -107,15 +108,25 @@ class TokenizerTestCase(object):
     @classmethod
     def build(cls):
         """Load and install all tests from the 'tokenizer' directory."""
-        directory = path.join(path.dirname(__file__), "tokenizer")
-        extension = ".mwtest"
-        for filename in listdir(directory):
-            if not filename.endswith(extension):
-                continue
-            with open(path.join(directory, filename), "r") as fp:
+        def load_file(filename):
+            with open(filename, "r") as fp:
                 text = fp.read()
                 if not py3k:
                     text = text.decode("utf8")
                 cls._load_tests(filename[:0-len(extension)], text)
 
+        directory = path.join(path.dirname(__file__), "tokenizer")
+        extension = ".mwtest"
+        if len(sys.argv) > 1:  # Read specific tests from command line
+            for name in sys.argv[1:]:
+                load_file(path.join(directory, name + extension))
+            sys.argv = [sys.argv[0]]  # So unittest doesn't try to load these
+            cls.skip_others = True
+        else:
+            for filename in listdir(directory):
+                if not filename.endswith(extension):
+                    continue
+                load_file(path.join(directory, filename))
+            cls.skip_others = False
+
 TokenizerTestCase.build()

tests/test_ctokenizer.py

@@ -38,10 +38,11 @@ class TestCTokenizer(TokenizerTestCase, unittest.TestCase):
     def setUpClass(cls):
         cls.tokenizer = CTokenizer
 
-    def test_uses_c(self):
-        """make sure the C tokenizer identifies as using a C extension"""
-        self.assertTrue(CTokenizer.USES_C)
-        self.assertTrue(CTokenizer().USES_C)
+    if not TokenizerTestCase.skip_others:
+        def test_uses_c(self):
+            """make sure the C tokenizer identifies as using a C extension"""
+            self.assertTrue(CTokenizer.USES_C)
+            self.assertTrue(CTokenizer().USES_C)
 
 if __name__ == "__main__":
     unittest.main(verbosity=2)

tests/test_pytokenizer.py

@@ -34,10 +34,11 @@ class TestPyTokenizer(TokenizerTestCase, unittest.TestCase):
     def setUpClass(cls):
         cls.tokenizer = Tokenizer
 
-    def test_uses_c(self):
-        """make sure the Python tokenizer identifies as not using C"""
-        self.assertFalse(Tokenizer.USES_C)
-        self.assertFalse(Tokenizer().USES_C)
+    if not TokenizerTestCase.skip_others:
+        def test_uses_c(self):
+            """make sure the Python tokenizer identifies as not using C"""
+            self.assertFalse(Tokenizer.USES_C)
+            self.assertFalse(Tokenizer().USES_C)
 
 if __name__ == "__main__":
     unittest.main(verbosity=2)

tests/tokenizer/arguments.mwtest

@@ -0,0 +1,4 @@
+name:   no_params
+label:  simplest type of argument
+input:  "{{{argument}}}"
+output: [ArgumentOpen(), Text(text="argument"), ArgumentClose()]