From 713b83a4d94e05bf907158aa6a5d98f7132d998c Mon Sep 17 00:00:00 2001 From: Ben Kurtovic Date: Sun, 3 Feb 2013 17:41:55 -0500 Subject: [PATCH] Added a metric ton of template tests; adjustments; docstrings. --- README.rst | 3 +- tests/_test_tokenizer.py | 22 +++- tests/test_ctokenizer.py | 1 + tests/test_docs.py | 6 + tests/test_pytokenizer.py | 1 + tests/tokenizer/templates.test | 285 +++++++++++++++++++++++++++++++++++++++++ 6 files changed, 314 insertions(+), 4 deletions(-) diff --git a/README.rst b/README.rst index 3901103..90e896f 100644 --- a/README.rst +++ b/README.rst @@ -18,7 +18,8 @@ so you can install the latest release with ``pip install mwparserfromhell`` cd mwparserfromhell python setup.py install -You can run the comprehensive unit testing suite with ``python setup.py test``. +You can run the comprehensive unit testing suite with +``python setup.py test -q``. Usage ----- diff --git a/tests/_test_tokenizer.py b/tests/_test_tokenizer.py index 2571692..bef7569 100644 --- a/tests/_test_tokenizer.py +++ b/tests/_test_tokenizer.py @@ -32,8 +32,20 @@ class _TestParseError(Exception): class TokenizerTestCase(object): + """A base test case for tokenizers, whose tests are loaded dynamically. + + Subclassed along with unittest.TestCase to form TestPyTokenizer and + TestCTokenizer. Tests are loaded dynamically from files in the 'tokenizer' + directory. + """ @classmethod def _build_test_method(cls, funcname, data): + """Create and return a method to be treated as a test case method. + + *data* is a dict containing multiple keys: the *input* text to be + tokenized, the expected list of tokens as *output*, and an optional + *label* for the method's docstring. + """ def inner(self): actual = self.tokenizer().tokenize(data["input"]) self.assertEqual(actual, data["output"]) @@ -44,8 +56,10 @@ class TokenizerTestCase(object): @classmethod def _load_tests(cls, filename, text): - counter = 1 + """Load all tests in *text* from the file *filename*.""" tests = text.split("\n---\n") + counter = 1 + digits = len(str(len(tests))) for test in tests: data = {"name": "", "label": "", "input": "", "output": []} try: @@ -79,16 +93,18 @@ class TokenizerTestCase(object): print(error.format(filename)) continue if not data["input"] or not data["output"]: - error = "Test '{0}'' in '{1}' was ignored because it lacked an input or an output" + error = "Test '{0}' in '{1}' was ignored because it lacked an input or an output" print(error.format(data["name"], filename)) continue - fname = "test_{0}{1}_{2}".format(filename, counter, data["name"]) + number = str(counter).zfill(digits) + fname = "test_{0}{1}_{2}".format(filename, number, data["name"]) meth = cls._build_test_method(fname, data) setattr(cls, fname, meth) counter += 1 @classmethod def build(cls): + """Load and install all tests from the 'tokenizer' directory.""" directory = path.join(path.dirname(__file__), "tokenizer") extension = ".test" for filename in listdir(directory): diff --git a/tests/test_ctokenizer.py b/tests/test_ctokenizer.py index 7d3ffd7..86f4787 100644 --- a/tests/test_ctokenizer.py +++ b/tests/test_ctokenizer.py @@ -25,6 +25,7 @@ import unittest from _test_tokenizer import TokenizerTestCase class TestCTokenizer(TokenizerTestCase, unittest.TestCase): + """Test cases for the C tokenizer.""" @classmethod def setUpClass(cls): from mwparserfromhell.parser._tokenizer import CTokenizer diff --git a/tests/test_docs.py b/tests/test_docs.py index 5ec25e1..d99652f 100644 --- a/tests/test_docs.py +++ b/tests/test_docs.py @@ -29,6 +29,7 @@ import mwparserfromhell from mwparserfromhell.compat import py3k, str, StringIO class TestDocs(unittest.TestCase): + """Integration test cases for mwparserfromhell's documentation.""" def assertPrint(self, input, output): """Assertion check that *input*, when printed, produces *output*.""" buff = StringIO() @@ -37,6 +38,7 @@ class TestDocs(unittest.TestCase): self.assertEqual(buff.read(), output) def test_readme_1(self): + """test a block of example code in the README""" text = "I has a template! {{foo|bar|baz|eggs=spam}} See it?" wikicode = mwparserfromhell.parse(text) self.assertPrint(wikicode, @@ -56,6 +58,7 @@ class TestDocs(unittest.TestCase): self.assertPrint(template.get("eggs").value, "spam") def test_readme_2(self): + """test a block of example code in the README""" code = mwparserfromhell.parse("{{foo|this {{includes a|template}}}}") if py3k: self.assertPrint(code.filter_templates(), @@ -71,6 +74,7 @@ class TestDocs(unittest.TestCase): "template") def test_readme_3(self): + """test a block of example code in the README""" text = "{{foo|{{bar}}={{baz|{{spam}}}}}}" temps = mwparserfromhell.parse(text).filter_templates(recursive=True) if py3k: @@ -80,6 +84,7 @@ class TestDocs(unittest.TestCase): self.assertPrint(temps, res) def test_readme_4(self): + """test a block of example code in the README""" text = "{{cleanup}} '''Foo''' is a [[bar]]. {{uncategorized}}" code = mwparserfromhell.parse(text) for template in code.filter_templates(): @@ -101,6 +106,7 @@ class TestDocs(unittest.TestCase): self.assertEqual(text, code) def test_readme_5(self): + """test a block of example code in the README; includes a web call""" url1 = "http://en.wikipedia.org/w/api.php" url2 = "http://en.wikipedia.org/w/index.php?title={0}&action=raw" title = "Test" diff --git a/tests/test_pytokenizer.py b/tests/test_pytokenizer.py index f739726..4254748 100644 --- a/tests/test_pytokenizer.py +++ b/tests/test_pytokenizer.py @@ -25,6 +25,7 @@ import unittest from _test_tokenizer import TokenizerTestCase class TestPyTokenizer(TokenizerTestCase, unittest.TestCase): + """Test cases for the Python tokenizer.""" @classmethod def setUpClass(cls): from mwparserfromhell.parser.tokenizer import Tokenizer diff --git a/tests/tokenizer/templates.test b/tests/tokenizer/templates.test index 23ac38f..7399022 100644 --- a/tests/tokenizer/templates.test +++ b/tests/tokenizer/templates.test @@ -30,3 +30,288 @@ name: multiple_named_params label: basic template with multiple named parameters input: "{{foo|bar=baz|biz=buzz|buff=baff|usr=bin}}" output: [TemplateOpen(), Text(text="foo"), TemplateParamSeparator(), Text(text="bar"), TemplateParamEquals(), Text(text="baz"), TemplateParamSeparator(), Text(text="biz"), TemplateParamEquals(), Text(text="buzz"), TemplateParamSeparator(), Text(text="buff"), TemplateParamEquals(), Text(text="baff"), TemplateParamSeparator(), Text(text="usr"), TemplateParamEquals(), Text(text="bin"), TemplateClose()] + +--- + +name: multiple_mixed_params +label: basic template with multiple unnamed/named parameters +input: "{{foo|bar=baz|biz|buzz=buff|usr|bin}}" +output: [TemplateOpen(), Text(text="foo"), TemplateParamSeparator(), Text(text="bar"), TemplateParamEquals(), Text(text="baz"), TemplateParamSeparator(), Text(text="biz"), TemplateParamSeparator(), Text(text="buzz"), TemplateParamEquals(), Text(text="buff"), TemplateParamSeparator(), Text(text="usr"), TemplateParamSeparator(), Text(text="bin"), TemplateClose()] + +--- + +name: multiple_mixed_params2 +label: basic template with multiple unnamed/named parameters in another order +input: "{{foo|bar|baz|biz=buzz|buff=baff|usr=bin}}" +output: [TemplateOpen(), Text(text="foo"), TemplateParamSeparator(), Text(text="bar"), TemplateParamSeparator(), Text(text="baz"), TemplateParamSeparator(), Text(text="biz"), TemplateParamEquals(), Text(text="buzz"), TemplateParamSeparator(), Text(text="buff"), TemplateParamEquals(), Text(text="baff"), TemplateParamSeparator(), Text(text="usr"), TemplateParamEquals(), Text(text="bin"), TemplateClose()] + +--- + +name: nested_unnamed_param +label: nested template as an unnamed parameter +input: "{{foo|{{bar}}}}" +output: [TemplateOpen(), Text(text="foo"), TemplateParamSeparator(), TemplateOpen(), Text(text="bar"), TemplateClose(), TemplateClose()] + +--- + +name: nested_named_param_value +label: nested template as a parameter value with a named parameter +input: "{{foo|bar={{baz}}}}" +output: [TemplateOpen(), Text(text="foo"), TemplateParamSeparator(), Text(text="bar"), TemplateParamEquals(), TemplateOpen(), Text(text="baz"), TemplateClose(), TemplateClose()] + +--- + +name: nested_named_param_name_and_value +label: nested templates as a parameter name and value +input: "{{foo|{{bar}}={{baz}}}}" +output: [TemplateOpen(), Text(text="foo"), TemplateParamSeparator(), TemplateOpen(), Text(text="bar"), TemplateClose(), TemplateParamEquals(), TemplateOpen(), Text(text="baz"), TemplateClose(), TemplateClose()] + +--- + +name: nested_name_start +label: nested template at the beginning of a template name +input: "{{{{foo}}bar}}" +output: [TemplateOpen(), TemplateOpen(), Text(text="foo"), TemplateClose(), Text(text="bar"), TemplateClose()] + +--- + +name: nested_name_start_unnamed_param +label: nested template at the beginning of a template name and as an unnamed parameter +input: "{{{{foo}}bar|{{baz}}}}" +output: [TemplateOpen(), TemplateOpen(), Text(text="foo"), TemplateClose(), Text(text="bar"), TemplateParamSeparator(), TemplateOpen(), Text(text="baz"), TemplateClose(), TemplateClose()] + +--- + +name: nested_name_start_named_param_value +label: nested template at the beginning of a template name and as a parameter value with a named parameter +input: "{{{{foo}}bar|baz={{biz}}}}" +output: [TemplateOpen(), TemplateOpen(), Text(text="foo"), TemplateClose(), Text(text="bar"), TemplateParamSeparator(), Text(text="baz"), TemplateParamEquals(), TemplateOpen(), Text(text="biz"), TemplateClose(), TemplateClose()] + +--- + +name: nested_name_start_named_param_name_and_value +label: nested template at the beginning of a template name and as a parameter name and value +input: "{{{{foo}}bar|{{baz}}={{biz}}}}" +output: [TemplateOpen(), TemplateOpen(), Text(text="foo"), TemplateClose(), Text(text="bar"), TemplateParamSeparator(), TemplateOpen(), Text(text="baz"), TemplateClose(), TemplateParamEquals(), TemplateOpen(), Text(text="biz"), TemplateClose(), TemplateClose()] + +--- + +name: nested_name_end +label: nested template at the end of a template name +input: "{{foo{{bar}}}}" +output: [TemplateOpen(), Text(text="foo"), TemplateOpen(), Text(text="bar"), TemplateClose(), TemplateClose()] + +--- + +name: nested_name_end_unnamed_param +label: nested template at the end of a template name and as an unnamed parameter +input: "{{foo{{bar}}|{{baz}}}}" +output: [TemplateOpen(), Text(text="foo"), TemplateOpen(), Text(text="bar"), TemplateClose(), TemplateParamSeparator(), TemplateOpen(), Text(text="baz"), TemplateClose(), TemplateClose()] + +--- + +name: nested_name_end_named_param_value +label: nested template at the end of a template name and as a parameter value with a named parameter +input: "{{foo{{bar}}|baz={{biz}}}}" +output: [TemplateOpen(), Text(text="foo"), TemplateOpen(), Text(text="bar"), TemplateClose(), TemplateParamSeparator(), Text(text="baz"), TemplateParamEquals(), TemplateOpen(), Text(text="biz"), TemplateClose(), TemplateClose()] + +--- + +name: nested_name_end_named_param_name_and_value +label: nested template at the end of a template name and as a parameter name and value +input: "{{foo{{bar}}|{{baz}}={{biz}}}}" +output: [TemplateOpen(), Text(text="foo"), TemplateOpen(), Text(text="bar"), TemplateClose(), TemplateParamSeparator(), TemplateOpen(), Text(text="baz"), TemplateClose(), TemplateParamEquals(), TemplateOpen(), Text(text="biz"), TemplateClose(), TemplateClose()] + +--- + +name: nested_name_mid +label: nested template in the middle of a template name +input: "{{foo{{bar}}baz}}" +output: [TemplateOpen(), Text(text="foo"), TemplateOpen(), Text(text="bar"), TemplateClose(), Text(text="baz"), TemplateClose()] + +--- + +name: nested_name_mid_unnamed_param +label: nested template in the middle of a template name and as an unnamed parameter +input: "{{foo{{bar}}baz|{{biz}}}}" +output: [TemplateOpen(), Text(text="foo"), TemplateOpen(), Text(text="bar"), TemplateClose(), Text(text="baz"), TemplateParamSeparator(), TemplateOpen(), Text(text="biz"), TemplateClose(), TemplateClose()] + +--- + +name: nested_name_mid_named_param_value +label: nested template in the middle of a template name and as a parameter value with a named parameter +input: "{{foo{{bar}}baz|biz={{buzz}}}}" +output: [TemplateOpen(), Text(text="foo"), TemplateOpen(), Text(text="bar"), TemplateClose(), Text(text="baz"), TemplateParamSeparator(), Text(text="biz"), TemplateParamEquals(), TemplateOpen(), Text(text="buzz"), TemplateClose(), TemplateClose()] + +--- + +name: nested_name_mid_named_param_name_and_value +label: nested template in the middle of a template name and as a parameter name and value +input: "{{foo{{bar}}baz|{{biz}}={{buzz}}}}" +output: [TemplateOpen(), Text(text="foo"), TemplateOpen(), Text(text="bar"), TemplateClose(), Text(text="baz"), TemplateParamSeparator(), TemplateOpen(), Text(text="biz"), TemplateClose(), TemplateParamEquals(), TemplateOpen(), Text(text="buzz"), TemplateClose(), TemplateClose()] + +--- + +name: nested_name_start_end +label: nested template at the beginning and end of a template name +input: "{{{{foo}}{{bar}}}}" +output: [TemplateOpen(), TemplateOpen(), Text(text="foo"), TemplateClose(), TemplateOpen(), Text(text="bar"), TemplateClose(), TemplateClose()] + +--- + +name: nested_name_start_end_unnamed_param +label: nested template at the beginning and end of a template name and as an unnamed parameter +input: "{{{{foo}}{{bar}}|{{baz}}}}" +output: [TemplateOpen(), TemplateOpen(), Text(text="foo"), TemplateClose(), TemplateOpen(), Text(text="bar"), TemplateClose(), TemplateParamSeparator(), TemplateOpen(), Text(text="baz"), TemplateClose(), TemplateClose()] + +--- + +name: nested_name_start_end_named_param_value +label: nested template at the beginning and end of a template name and as a parameter value with a named parameter +input: "{{{{foo}}{{bar}}|baz={{biz}}}}" +output: [TemplateOpen(), TemplateOpen(), Text(text="foo"), TemplateClose(), TemplateOpen(), Text(text="bar"), TemplateClose(), TemplateParamSeparator(), Text(text="baz"), TemplateParamEquals(), TemplateOpen(), Text(text="biz"), TemplateClose(), TemplateClose()] + +--- + +name: nested_name_start_end_named_param_name_and_value +label: nested template at the beginning and end of a template name and as a parameter name and value +input: "{{{{foo}}{{bar}}|{{baz}}={{biz}}}}" +output: [TemplateOpen(), TemplateOpen(), Text(text="foo"), TemplateClose(), TemplateOpen(), Text(text="bar"), TemplateClose(), TemplateParamSeparator(), TemplateOpen(), Text(text="baz"), TemplateClose(), TemplateParamEquals(), TemplateOpen(), Text(text="biz"), TemplateClose(), TemplateClose()] + +--- + +name: nested_names_multiple +label: multiple nested templates within nested templates +input: "{{{{{{{{foo}}bar}}baz}}biz}}" +output: [TemplateOpen(), TemplateOpen(), TemplateOpen(), TemplateOpen(), Text(text="foo"), TemplateClose(), Text(text="bar"), TemplateClose(), Text(text="baz"), TemplateClose(), Text(text="biz"), TemplateClose()] + +--- + +name: nested_names_multiple_unnamed_param +label: multiple nested templates within nested templates with a nested unnamed parameter +input: "{{{{{{{{foo}}bar}}baz}}biz|{{buzz}}}}" +output: [TemplateOpen(), TemplateOpen(), TemplateOpen(), TemplateOpen(), Text(text="foo"), TemplateClose(), Text(text="bar"), TemplateClose(), Text(text="baz"), TemplateClose(), Text(text="biz"), TemplateParamSeparator(), TemplateOpen(), Text(text="buzz"), TemplateClose(), TemplateClose()] + +--- + +name: nested_names_multiple_named_param_value +label: multiple nested templates within nested templates with a nested parameter value in a named parameter +input: "{{{{{{{{foo}}bar}}baz}}biz|buzz={{bin}}}}" +output: [TemplateOpen(), TemplateOpen(), TemplateOpen(), TemplateOpen(), Text(text="foo"), TemplateClose(), Text(text="bar"), TemplateClose(), Text(text="baz"), TemplateClose(), Text(text="biz"), TemplateParamSeparator(), Text(text="buzz"), TemplateParamEquals(), TemplateOpen(), Text(text="bin"), TemplateClose(), TemplateClose()] + +--- + +name: nested_names_multiple_named_param_name_and_value +label: multiple nested templates within nested templates with a nested parameter name and value +input: "{{{{{{{{foo}}bar}}baz}}biz|{{buzz}}={{bin}}}}" +output: [TemplateOpen(), TemplateOpen(), TemplateOpen(), TemplateOpen(), Text(text="foo"), TemplateClose(), Text(text="bar"), TemplateClose(), Text(text="baz"), TemplateClose(), Text(text="biz"), TemplateParamSeparator(), TemplateOpen(), Text(text="buzz"), TemplateClose(), TemplateParamEquals(), TemplateOpen(), Text(text="bin"), TemplateClose(), TemplateClose()] + +--- + +name: incomplete_tests + +"{{{{{{{{foo}}bar|baz=biz}}buzz}}usr|{{bin}}}}" + +"{{\nfoobar}}" +"{{foobar\n}}" +"{{\nfoobar\n}}" +"{{foo\nbar}}" +"{{\nfoo\nbar}}" +"{{foo\nbar\n}}" +"{{\nfoo\nbar\n}}" + +"{{foo|\nbar}}" +"{{foo|bar\n}}" +"{{foo|\nbar\n}}" +"{{foo|\nb\nar}}" +"{{foo|b\nar\n}}" +"{{foo|\nb\nar\n}}" +"{{\nfoo|\nbar}}" +"{{\nfoo|bar\n}}" +"{{\nfoo|\nbar\n}}" +"{{\nfoo|\nb\nar}}" +"{{\nfoo|b\nar\n}}" +"{{\nfoo|\nb\nar\n}}" +"{{foo\n|\nbar}}" +"{{foo\n|bar\n}}" +"{{foo\n|\nbar\n}}" +"{{foo\n|\nb\nar}}" +"{{foo\n|b\nar\n}}" +"{{foo\n|\nb\nar\n}}" +"{{\nfoo\n|\nbar}}" +"{{\nfoo\n|bar\n}}" +"{{\nfoo\n|\nbar\n}}" +"{{\nfoo\n|\nb\nar}}" +"{{\nfoo\n|b\nar\n}}" +"{{\nfoo\n|\nb\nar\n}}" +"{{f\noo|\nbar}}" +"{{\nf\noo|\nbar}}" +"{{f\noo\n|\nbar}}" +"{{\nf\noo\n|\nbar}}" + +"{{foo|1=\nbar}}" +"{{foo|1=bar\n}}" +"{{foo|1=\nbar\n}}" +"{{foo|1=\nb\nar}}" +"{{foo|1=b\nar\n}}" +"{{foo|1=\nb\nar\n}}" +"{{foo|\nbar=baz}}" +"{{foo|bar\n=baz}}" +"{{foo|\nbar\n=baz}}" +"{{foo|\nb\nar=baz}}" +"{{foo|b\nar\n=baz}}" +"{{foo|\nb\nar\n=baz}}" +"{{foo|\nbar=baz\n}}" +"{{foo|bar\n=baz\n}}" +"{{foo|\nbar\n=baz\n}}" +"{{foo|\nb\nar=baz\n}}" +"{{foo|b\nar\n=baz\n}}" +"{{foo|\nb\nar\n=baz\n}}" +"{{foo|\nbar=\nbaz}}" +"{{foo|bar\n=\nbaz}}" +"{{foo|\nbar\n=\nbaz}}" +"{{foo|\nb\nar=\nbaz}}" +"{{foo|b\nar\n=\nbaz}}" +"{{foo|\nb\nar\n=\nbaz}}" +"{{foo|\nbar=\nbaz\n}}" +"{{foo|bar\n=\nbaz\n}}" +"{{foo|\nbar\n=\nbaz\n}}" +"{{foo|\nb\nar=\nbaz\n}}" +"{{foo|b\nar\n=\nbaz\n}}" +"{{foo|\nb\nar\n=\nbaz\n}}" +"{{foo|\nbar=ba\nz}}" +"{{foo|bar\n=ba\nz}}" +"{{foo|\nbar\n=ba\nz}}" +"{{foo|\nb\nar=ba\nz}}" +"{{foo|b\nar\n=ba\nz}}" +"{{foo|\nb\nar\n=ba\nz}}" + +"{{\nfoo\n|\nbar\n=\nb\naz\n|\nb\nuz\n}}" +"{{\nfoo\n|\nb\nar\n|\nbaz\n=\nb\nuz\n}}" + +"{{\nfoo\n|\n{{\nbar\n|\nbaz\n=\nb\niz\n}}\n=\nb\nuzz\n}}" + +"{{foo{bar}}" +"{{foo}bar}}" +"{{{foobar}}" +"{{foo{b{ar}}" +"{{foo[bar}}" +"{{foo]bar}}" +"{{[foobar}}" +"{{foobar]}}" + +"{{foobar" +"{{foobar}" +"{{foobar|" +"{{foo|bar" +"{{foo|bar|" +"{{foo|bar=" +"{{foo|bar=|" +"{{foo|bar=baz" +"{{foo|bar=baz|" +"{{foo|bar|baz" +"{{foo|bar|baz=" +"{{foo|bar|baz=biz" +"{{foo|bar=baz|biz" +"{{foo|bar=baz|biz=" +"{{foo|bar=baz|biz=buzz"