Added a metric ton of template tests; adjustments; docstrings.

README.rst

@@ -18,7 +18,8 @@ so you can install the latest release with ``pip install mwparserfromhell``
     cd mwparserfromhell
     python setup.py install
 
-You can run the comprehensive unit testing suite with ``python setup.py test``.
+You can run the comprehensive unit testing suite with
+``python setup.py test -q``.
 
 Usage
 -----

tests/_test_tokenizer.py

@@ -32,8 +32,20 @@ class _TestParseError(Exception):
 
 
 class TokenizerTestCase(object):
+    """A base test case for tokenizers, whose tests are loaded dynamically.
+
+    Subclassed along with unittest.TestCase to form TestPyTokenizer and
+    TestCTokenizer. Tests are loaded dynamically from files in the 'tokenizer'
+    directory.
+    """
     @classmethod
     def _build_test_method(cls, funcname, data):
+        """Create and return a method to be treated as a test case method.
+
+        *data* is a dict containing multiple keys: the *input* text to be
+        tokenized, the expected list of tokens as *output*, and an optional
+        *label* for the method's docstring.
+        """
         def inner(self):
             actual = self.tokenizer().tokenize(data["input"])
             self.assertEqual(actual, data["output"])
@@ -44,8 +56,10 @@ class TokenizerTestCase(object):
 
     @classmethod
     def _load_tests(cls, filename, text):
-        counter = 1
+        """Load all tests in *text* from the file *filename*."""
         tests = text.split("\n---\n")
+        counter = 1
+        digits = len(str(len(tests)))
         for test in tests:
             data = {"name": "", "label": "", "input": "", "output": []}
             try:
@@ -79,16 +93,18 @@ class TokenizerTestCase(object):
                 print(error.format(filename))
                 continue
             if not data["input"] or not data["output"]:
-                error = "Test '{0}'' in '{1}' was ignored because it lacked an input or an output"
+                error = "Test '{0}' in '{1}' was ignored because it lacked an input or an output"
                 print(error.format(data["name"], filename))
                 continue
-            fname = "test_{0}{1}_{2}".format(filename, counter, data["name"])
+            number = str(counter).zfill(digits)
+            fname = "test_{0}{1}_{2}".format(filename, number, data["name"])
             meth = cls._build_test_method(fname, data)
             setattr(cls, fname, meth)
             counter += 1
 
     @classmethod
     def build(cls):
+        """Load and install all tests from the 'tokenizer' directory."""
         directory = path.join(path.dirname(__file__), "tokenizer")
         extension = ".test"
         for filename in listdir(directory):

tests/test_ctokenizer.py

@@ -25,6 +25,7 @@ import unittest
 from _test_tokenizer import TokenizerTestCase
 
 class TestCTokenizer(TokenizerTestCase, unittest.TestCase):
+    """Test cases for the C tokenizer."""
     @classmethod
     def setUpClass(cls):
         from mwparserfromhell.parser._tokenizer import CTokenizer

tests/test_docs.py

@@ -29,6 +29,7 @@ import mwparserfromhell
 from mwparserfromhell.compat import py3k, str, StringIO
 
 class TestDocs(unittest.TestCase):
+    """Integration test cases for mwparserfromhell's documentation."""
     def assertPrint(self, input, output):
         """Assertion check that *input*, when printed, produces *output*."""
         buff = StringIO()
@@ -37,6 +38,7 @@ class TestDocs(unittest.TestCase):
         self.assertEqual(buff.read(), output)
 
     def test_readme_1(self):
+        """test a block of example code in the README"""
         text = "I has a template! {{foo|bar|baz|eggs=spam}} See it?"
         wikicode = mwparserfromhell.parse(text)
         self.assertPrint(wikicode,
@@ -56,6 +58,7 @@ class TestDocs(unittest.TestCase):
         self.assertPrint(template.get("eggs").value, "spam")
 
     def test_readme_2(self):
+        """test a block of example code in the README"""
         code = mwparserfromhell.parse("{{foo|this {{includes a|template}}}}")
         if py3k:
             self.assertPrint(code.filter_templates(),
@@ -71,6 +74,7 @@ class TestDocs(unittest.TestCase):
                          "template")
 
     def test_readme_3(self):
+        """test a block of example code in the README"""
         text = "{{foo|{{bar}}={{baz|{{spam}}}}}}"
         temps = mwparserfromhell.parse(text).filter_templates(recursive=True)
         if py3k:
@@ -80,6 +84,7 @@ class TestDocs(unittest.TestCase):
         self.assertPrint(temps, res)
 
     def test_readme_4(self):
+        """test a block of example code in the README"""
         text = "{{cleanup}} '''Foo''' is a [[bar]]. {{uncategorized}}"
         code = mwparserfromhell.parse(text)
         for template in code.filter_templates():
@@ -101,6 +106,7 @@ class TestDocs(unittest.TestCase):
         self.assertEqual(text, code)
 
     def test_readme_5(self):
+        """test a block of example code in the README; includes a web call"""
         url1 = "http://en.wikipedia.org/w/api.php"
         url2 = "http://en.wikipedia.org/w/index.php?title={0}&action=raw"
         title = "Test"

tests/test_pytokenizer.py

@@ -25,6 +25,7 @@ import unittest
 from _test_tokenizer import TokenizerTestCase
 
 class TestPyTokenizer(TokenizerTestCase, unittest.TestCase):
+    """Test cases for the Python tokenizer."""
     @classmethod
     def setUpClass(cls):
         from mwparserfromhell.parser.tokenizer import Tokenizer

tests/tokenizer/templates.test

@@ -30,3 +30,288 @@ name:   multiple_named_params
 label:  basic template with multiple named parameters
 input:  "{{foo|bar=baz|biz=buzz|buff=baff|usr=bin}}"
 output: [TemplateOpen(), Text(text="foo"), TemplateParamSeparator(), Text(text="bar"), TemplateParamEquals(), Text(text="baz"), TemplateParamSeparator(), Text(text="biz"), TemplateParamEquals(), Text(text="buzz"), TemplateParamSeparator(), Text(text="buff"), TemplateParamEquals(), Text(text="baff"), TemplateParamSeparator(), Text(text="usr"), TemplateParamEquals(), Text(text="bin"), TemplateClose()]
+
+---
+
+name:   multiple_mixed_params
+label:  basic template with multiple unnamed/named parameters
+input:  "{{foo|bar=baz|biz|buzz=buff|usr|bin}}"
+output: [TemplateOpen(), Text(text="foo"), TemplateParamSeparator(), Text(text="bar"), TemplateParamEquals(), Text(text="baz"), TemplateParamSeparator(), Text(text="biz"), TemplateParamSeparator(), Text(text="buzz"), TemplateParamEquals(), Text(text="buff"), TemplateParamSeparator(), Text(text="usr"), TemplateParamSeparator(), Text(text="bin"), TemplateClose()]
+
+---
+
+name:   multiple_mixed_params2
+label:  basic template with multiple unnamed/named parameters in another order
+input:  "{{foo|bar|baz|biz=buzz|buff=baff|usr=bin}}"
+output: [TemplateOpen(), Text(text="foo"), TemplateParamSeparator(), Text(text="bar"), TemplateParamSeparator(), Text(text="baz"), TemplateParamSeparator(), Text(text="biz"), TemplateParamEquals(), Text(text="buzz"), TemplateParamSeparator(), Text(text="buff"), TemplateParamEquals(), Text(text="baff"), TemplateParamSeparator(), Text(text="usr"), TemplateParamEquals(), Text(text="bin"), TemplateClose()]
+
+---
+
+name:   nested_unnamed_param
+label:  nested template as an unnamed parameter
+input:  "{{foo|{{bar}}}}"
+output: [TemplateOpen(), Text(text="foo"), TemplateParamSeparator(), TemplateOpen(), Text(text="bar"), TemplateClose(), TemplateClose()]
+
+---
+
+name:   nested_named_param_value
+label:  nested template as a parameter value with a named parameter
+input:  "{{foo|bar={{baz}}}}"
+output: [TemplateOpen(), Text(text="foo"), TemplateParamSeparator(), Text(text="bar"), TemplateParamEquals(), TemplateOpen(), Text(text="baz"), TemplateClose(), TemplateClose()]
+
+---
+
+name:   nested_named_param_name_and_value
+label:  nested templates as a parameter name and value
+input:  "{{foo|{{bar}}={{baz}}}}"
+output: [TemplateOpen(), Text(text="foo"), TemplateParamSeparator(), TemplateOpen(), Text(text="bar"), TemplateClose(), TemplateParamEquals(), TemplateOpen(), Text(text="baz"), TemplateClose(), TemplateClose()]
+
+---
+
+name:   nested_name_start
+label:  nested template at the beginning of a template name
+input:  "{{{{foo}}bar}}"
+output: [TemplateOpen(), TemplateOpen(), Text(text="foo"), TemplateClose(), Text(text="bar"), TemplateClose()]
+
+---
+
+name:   nested_name_start_unnamed_param
+label:  nested template at the beginning of a template name and as an unnamed parameter
+input:  "{{{{foo}}bar|{{baz}}}}"
+output: [TemplateOpen(), TemplateOpen(), Text(text="foo"), TemplateClose(), Text(text="bar"), TemplateParamSeparator(), TemplateOpen(), Text(text="baz"), TemplateClose(), TemplateClose()]
+
+---
+
+name:   nested_name_start_named_param_value
+label:  nested template at the beginning of a template name and as a parameter value with a named parameter
+input:  "{{{{foo}}bar|baz={{biz}}}}"
+output: [TemplateOpen(), TemplateOpen(), Text(text="foo"), TemplateClose(), Text(text="bar"), TemplateParamSeparator(), Text(text="baz"), TemplateParamEquals(), TemplateOpen(), Text(text="biz"), TemplateClose(), TemplateClose()]
+
+---
+
+name:   nested_name_start_named_param_name_and_value
+label:  nested template at the beginning of a template name and as a parameter name and value
+input:  "{{{{foo}}bar|{{baz}}={{biz}}}}"
+output: [TemplateOpen(), TemplateOpen(), Text(text="foo"), TemplateClose(), Text(text="bar"), TemplateParamSeparator(), TemplateOpen(), Text(text="baz"), TemplateClose(), TemplateParamEquals(), TemplateOpen(), Text(text="biz"), TemplateClose(), TemplateClose()]
+
+---
+
+name:   nested_name_end
+label:  nested template at the end of a template name
+input:  "{{foo{{bar}}}}"
+output: [TemplateOpen(), Text(text="foo"), TemplateOpen(), Text(text="bar"), TemplateClose(), TemplateClose()]
+
+---
+
+name:   nested_name_end_unnamed_param
+label:  nested template at the end of a template name and as an unnamed parameter
+input:  "{{foo{{bar}}|{{baz}}}}"
+output: [TemplateOpen(), Text(text="foo"), TemplateOpen(), Text(text="bar"), TemplateClose(), TemplateParamSeparator(), TemplateOpen(), Text(text="baz"), TemplateClose(), TemplateClose()]
+
+---
+
+name:   nested_name_end_named_param_value
+label:  nested template at the end of a template name and as a parameter value with a named parameter
+input:  "{{foo{{bar}}|baz={{biz}}}}"
+output: [TemplateOpen(), Text(text="foo"), TemplateOpen(), Text(text="bar"), TemplateClose(), TemplateParamSeparator(), Text(text="baz"), TemplateParamEquals(), TemplateOpen(), Text(text="biz"), TemplateClose(), TemplateClose()]
+
+---
+
+name:   nested_name_end_named_param_name_and_value
+label:  nested template at the end of a template name and as a parameter name and value
+input:  "{{foo{{bar}}|{{baz}}={{biz}}}}"
+output: [TemplateOpen(), Text(text="foo"), TemplateOpen(), Text(text="bar"), TemplateClose(), TemplateParamSeparator(), TemplateOpen(), Text(text="baz"), TemplateClose(), TemplateParamEquals(), TemplateOpen(), Text(text="biz"), TemplateClose(), TemplateClose()]
+
+---
+
+name:   nested_name_mid
+label:  nested template in the middle of a template name
+input:  "{{foo{{bar}}baz}}"
+output: [TemplateOpen(), Text(text="foo"), TemplateOpen(), Text(text="bar"), TemplateClose(), Text(text="baz"), TemplateClose()]
+
+---
+
+name:   nested_name_mid_unnamed_param
+label:  nested template in the middle of a template name and as an unnamed parameter
+input:  "{{foo{{bar}}baz|{{biz}}}}"
+output: [TemplateOpen(), Text(text="foo"), TemplateOpen(), Text(text="bar"), TemplateClose(), Text(text="baz"), TemplateParamSeparator(), TemplateOpen(), Text(text="biz"), TemplateClose(), TemplateClose()]
+
+---
+
+name:   nested_name_mid_named_param_value
+label:  nested template in the middle of a template name and as a parameter value with a named parameter
+input:  "{{foo{{bar}}baz|biz={{buzz}}}}"
+output: [TemplateOpen(), Text(text="foo"), TemplateOpen(), Text(text="bar"), TemplateClose(), Text(text="baz"), TemplateParamSeparator(), Text(text="biz"), TemplateParamEquals(), TemplateOpen(), Text(text="buzz"), TemplateClose(), TemplateClose()]
+
+---
+
+name:   nested_name_mid_named_param_name_and_value
+label:  nested template in the middle of a template name and as a parameter name and value
+input:  "{{foo{{bar}}baz|{{biz}}={{buzz}}}}"
+output: [TemplateOpen(), Text(text="foo"), TemplateOpen(), Text(text="bar"), TemplateClose(), Text(text="baz"), TemplateParamSeparator(), TemplateOpen(), Text(text="biz"), TemplateClose(), TemplateParamEquals(), TemplateOpen(), Text(text="buzz"), TemplateClose(), TemplateClose()]
+
+---
+
+name:   nested_name_start_end
+label:  nested template at the beginning and end of a template name
+input:  "{{{{foo}}{{bar}}}}"
+output: [TemplateOpen(), TemplateOpen(), Text(text="foo"), TemplateClose(), TemplateOpen(), Text(text="bar"), TemplateClose(), TemplateClose()]
+
+---
+
+name:   nested_name_start_end_unnamed_param
+label:  nested template at the beginning and end of a template name and as an unnamed parameter
+input:  "{{{{foo}}{{bar}}|{{baz}}}}"
+output: [TemplateOpen(), TemplateOpen(), Text(text="foo"), TemplateClose(), TemplateOpen(), Text(text="bar"), TemplateClose(), TemplateParamSeparator(), TemplateOpen(), Text(text="baz"), TemplateClose(), TemplateClose()]
+
+---
+
+name:   nested_name_start_end_named_param_value
+label:  nested template at the beginning and end of a template name and as a parameter value with a named parameter
+input:  "{{{{foo}}{{bar}}|baz={{biz}}}}"
+output: [TemplateOpen(), TemplateOpen(), Text(text="foo"), TemplateClose(), TemplateOpen(), Text(text="bar"), TemplateClose(), TemplateParamSeparator(), Text(text="baz"), TemplateParamEquals(), TemplateOpen(), Text(text="biz"), TemplateClose(), TemplateClose()]
+
+---
+
+name:   nested_name_start_end_named_param_name_and_value
+label:  nested template at the beginning and end of a template name and as a parameter name and value
+input:  "{{{{foo}}{{bar}}|{{baz}}={{biz}}}}"
+output: [TemplateOpen(), TemplateOpen(), Text(text="foo"), TemplateClose(), TemplateOpen(), Text(text="bar"), TemplateClose(), TemplateParamSeparator(), TemplateOpen(), Text(text="baz"), TemplateClose(), TemplateParamEquals(), TemplateOpen(), Text(text="biz"), TemplateClose(), TemplateClose()]
+
+---
+
+name:   nested_names_multiple
+label:  multiple nested templates within nested templates
+input:  "{{{{{{{{foo}}bar}}baz}}biz}}"
+output: [TemplateOpen(), TemplateOpen(), TemplateOpen(), TemplateOpen(), Text(text="foo"), TemplateClose(), Text(text="bar"), TemplateClose(), Text(text="baz"), TemplateClose(), Text(text="biz"), TemplateClose()]
+
+---
+
+name:   nested_names_multiple_unnamed_param
+label:  multiple nested templates within nested templates with a nested unnamed parameter
+input:  "{{{{{{{{foo}}bar}}baz}}biz|{{buzz}}}}"
+output: [TemplateOpen(), TemplateOpen(), TemplateOpen(), TemplateOpen(), Text(text="foo"), TemplateClose(), Text(text="bar"), TemplateClose(), Text(text="baz"), TemplateClose(), Text(text="biz"), TemplateParamSeparator(), TemplateOpen(), Text(text="buzz"), TemplateClose(), TemplateClose()]
+
+---
+
+name:   nested_names_multiple_named_param_value
+label:  multiple nested templates within nested templates with a nested parameter value in a named parameter
+input:  "{{{{{{{{foo}}bar}}baz}}biz|buzz={{bin}}}}"
+output: [TemplateOpen(), TemplateOpen(), TemplateOpen(), TemplateOpen(), Text(text="foo"), TemplateClose(), Text(text="bar"), TemplateClose(), Text(text="baz"), TemplateClose(), Text(text="biz"), TemplateParamSeparator(), Text(text="buzz"), TemplateParamEquals(), TemplateOpen(), Text(text="bin"), TemplateClose(), TemplateClose()]
+
+---
+
+name:   nested_names_multiple_named_param_name_and_value
+label:  multiple nested templates within nested templates with a nested parameter name and value
+input:  "{{{{{{{{foo}}bar}}baz}}biz|{{buzz}}={{bin}}}}"
+output: [TemplateOpen(), TemplateOpen(), TemplateOpen(), TemplateOpen(), Text(text="foo"), TemplateClose(), Text(text="bar"), TemplateClose(), Text(text="baz"), TemplateClose(), Text(text="biz"), TemplateParamSeparator(), TemplateOpen(), Text(text="buzz"), TemplateClose(), TemplateParamEquals(), TemplateOpen(), Text(text="bin"), TemplateClose(), TemplateClose()]
+
+---
+
+name:   incomplete_tests
+
+"{{{{{{{{foo}}bar|baz=biz}}buzz}}usr|{{bin}}}}"
+
+"{{\nfoobar}}"
+"{{foobar\n}}"
+"{{\nfoobar\n}}"
+"{{foo\nbar}}"
+"{{\nfoo\nbar}}"
+"{{foo\nbar\n}}"
+"{{\nfoo\nbar\n}}"
+
+"{{foo|\nbar}}"
+"{{foo|bar\n}}"
+"{{foo|\nbar\n}}"
+"{{foo|\nb\nar}}"
+"{{foo|b\nar\n}}"
+"{{foo|\nb\nar\n}}"
+"{{\nfoo|\nbar}}"
+"{{\nfoo|bar\n}}"
+"{{\nfoo|\nbar\n}}"
+"{{\nfoo|\nb\nar}}"
+"{{\nfoo|b\nar\n}}"
+"{{\nfoo|\nb\nar\n}}"
+"{{foo\n|\nbar}}"
+"{{foo\n|bar\n}}"
+"{{foo\n|\nbar\n}}"
+"{{foo\n|\nb\nar}}"
+"{{foo\n|b\nar\n}}"
+"{{foo\n|\nb\nar\n}}"
+"{{\nfoo\n|\nbar}}"
+"{{\nfoo\n|bar\n}}"
+"{{\nfoo\n|\nbar\n}}"
+"{{\nfoo\n|\nb\nar}}"
+"{{\nfoo\n|b\nar\n}}"
+"{{\nfoo\n|\nb\nar\n}}"
+"{{f\noo|\nbar}}"
+"{{\nf\noo|\nbar}}"
+"{{f\noo\n|\nbar}}"
+"{{\nf\noo\n|\nbar}}"
+
+"{{foo|1=\nbar}}"
+"{{foo|1=bar\n}}"
+"{{foo|1=\nbar\n}}"
+"{{foo|1=\nb\nar}}"
+"{{foo|1=b\nar\n}}"
+"{{foo|1=\nb\nar\n}}"
+"{{foo|\nbar=baz}}"
+"{{foo|bar\n=baz}}"
+"{{foo|\nbar\n=baz}}"
+"{{foo|\nb\nar=baz}}"
+"{{foo|b\nar\n=baz}}"
+"{{foo|\nb\nar\n=baz}}"
+"{{foo|\nbar=baz\n}}"
+"{{foo|bar\n=baz\n}}"
+"{{foo|\nbar\n=baz\n}}"
+"{{foo|\nb\nar=baz\n}}"
+"{{foo|b\nar\n=baz\n}}"
+"{{foo|\nb\nar\n=baz\n}}"
+"{{foo|\nbar=\nbaz}}"
+"{{foo|bar\n=\nbaz}}"
+"{{foo|\nbar\n=\nbaz}}"
+"{{foo|\nb\nar=\nbaz}}"
+"{{foo|b\nar\n=\nbaz}}"
+"{{foo|\nb\nar\n=\nbaz}}"
+"{{foo|\nbar=\nbaz\n}}"
+"{{foo|bar\n=\nbaz\n}}"
+"{{foo|\nbar\n=\nbaz\n}}"
+"{{foo|\nb\nar=\nbaz\n}}"
+"{{foo|b\nar\n=\nbaz\n}}"
+"{{foo|\nb\nar\n=\nbaz\n}}"
+"{{foo|\nbar=ba\nz}}"
+"{{foo|bar\n=ba\nz}}"
+"{{foo|\nbar\n=ba\nz}}"
+"{{foo|\nb\nar=ba\nz}}"
+"{{foo|b\nar\n=ba\nz}}"
+"{{foo|\nb\nar\n=ba\nz}}"
+
+"{{\nfoo\n|\nbar\n=\nb\naz\n|\nb\nuz\n}}"
+"{{\nfoo\n|\nb\nar\n|\nbaz\n=\nb\nuz\n}}"
+
+"{{\nfoo\n|\n{{\nbar\n|\nbaz\n=\nb\niz\n}}\n=\nb\nuzz\n}}"
+
+"{{foo{bar}}"
+"{{foo}bar}}"
+"{{{foobar}}"
+"{{foo{b{ar}}"
+"{{foo[bar}}"
+"{{foo]bar}}"
+"{{[foobar}}"
+"{{foobar]}}"
+
+"{{foobar"
+"{{foobar}"
+"{{foobar|"
+"{{foo|bar"
+"{{foo|bar|"
+"{{foo|bar="
+"{{foo|bar=|"
+"{{foo|bar=baz"
+"{{foo|bar=baz|"
+"{{foo|bar|baz"
+"{{foo|bar|baz="
+"{{foo|bar|baz=biz"
+"{{foo|bar=baz|biz"
+"{{foo|bar=baz|biz="
+"{{foo|bar=baz|biz=buzz"