Explorar el Código

Added a metric ton of template tests; adjustments; docstrings.

tags/v0.2
Ben Kurtovic hace 11 años
padre
commit
713b83a4d9
Se han modificado 6 ficheros con 314 adiciones y 4 borrados
  1. +2
    -1
      README.rst
  2. +19
    -3
      tests/_test_tokenizer.py
  3. +1
    -0
      tests/test_ctokenizer.py
  4. +6
    -0
      tests/test_docs.py
  5. +1
    -0
      tests/test_pytokenizer.py
  6. +285
    -0
      tests/tokenizer/templates.test

+ 2
- 1
README.rst Ver fichero

@@ -18,7 +18,8 @@ so you can install the latest release with ``pip install mwparserfromhell``
cd mwparserfromhell
python setup.py install

You can run the comprehensive unit testing suite with ``python setup.py test``.
You can run the comprehensive unit testing suite with
``python setup.py test -q``.

Usage
-----


+ 19
- 3
tests/_test_tokenizer.py Ver fichero

@@ -32,8 +32,20 @@ class _TestParseError(Exception):


class TokenizerTestCase(object):
"""A base test case for tokenizers, whose tests are loaded dynamically.

Subclassed along with unittest.TestCase to form TestPyTokenizer and
TestCTokenizer. Tests are loaded dynamically from files in the 'tokenizer'
directory.
"""
@classmethod
def _build_test_method(cls, funcname, data):
"""Create and return a method to be treated as a test case method.

*data* is a dict containing multiple keys: the *input* text to be
tokenized, the expected list of tokens as *output*, and an optional
*label* for the method's docstring.
"""
def inner(self):
actual = self.tokenizer().tokenize(data["input"])
self.assertEqual(actual, data["output"])
@@ -44,8 +56,10 @@ class TokenizerTestCase(object):

@classmethod
def _load_tests(cls, filename, text):
counter = 1
"""Load all tests in *text* from the file *filename*."""
tests = text.split("\n---\n")
counter = 1
digits = len(str(len(tests)))
for test in tests:
data = {"name": "", "label": "", "input": "", "output": []}
try:
@@ -79,16 +93,18 @@ class TokenizerTestCase(object):
print(error.format(filename))
continue
if not data["input"] or not data["output"]:
error = "Test '{0}'' in '{1}' was ignored because it lacked an input or an output"
error = "Test '{0}' in '{1}' was ignored because it lacked an input or an output"
print(error.format(data["name"], filename))
continue
fname = "test_{0}{1}_{2}".format(filename, counter, data["name"])
number = str(counter).zfill(digits)
fname = "test_{0}{1}_{2}".format(filename, number, data["name"])
meth = cls._build_test_method(fname, data)
setattr(cls, fname, meth)
counter += 1

@classmethod
def build(cls):
"""Load and install all tests from the 'tokenizer' directory."""
directory = path.join(path.dirname(__file__), "tokenizer")
extension = ".test"
for filename in listdir(directory):


+ 1
- 0
tests/test_ctokenizer.py Ver fichero

@@ -25,6 +25,7 @@ import unittest
from _test_tokenizer import TokenizerTestCase

class TestCTokenizer(TokenizerTestCase, unittest.TestCase):
"""Test cases for the C tokenizer."""
@classmethod
def setUpClass(cls):
from mwparserfromhell.parser._tokenizer import CTokenizer


+ 6
- 0
tests/test_docs.py Ver fichero

@@ -29,6 +29,7 @@ import mwparserfromhell
from mwparserfromhell.compat import py3k, str, StringIO

class TestDocs(unittest.TestCase):
"""Integration test cases for mwparserfromhell's documentation."""
def assertPrint(self, input, output):
"""Assertion check that *input*, when printed, produces *output*."""
buff = StringIO()
@@ -37,6 +38,7 @@ class TestDocs(unittest.TestCase):
self.assertEqual(buff.read(), output)

def test_readme_1(self):
"""test a block of example code in the README"""
text = "I has a template! {{foo|bar|baz|eggs=spam}} See it?"
wikicode = mwparserfromhell.parse(text)
self.assertPrint(wikicode,
@@ -56,6 +58,7 @@ class TestDocs(unittest.TestCase):
self.assertPrint(template.get("eggs").value, "spam")

def test_readme_2(self):
"""test a block of example code in the README"""
code = mwparserfromhell.parse("{{foo|this {{includes a|template}}}}")
if py3k:
self.assertPrint(code.filter_templates(),
@@ -71,6 +74,7 @@ class TestDocs(unittest.TestCase):
"template")

def test_readme_3(self):
"""test a block of example code in the README"""
text = "{{foo|{{bar}}={{baz|{{spam}}}}}}"
temps = mwparserfromhell.parse(text).filter_templates(recursive=True)
if py3k:
@@ -80,6 +84,7 @@ class TestDocs(unittest.TestCase):
self.assertPrint(temps, res)

def test_readme_4(self):
"""test a block of example code in the README"""
text = "{{cleanup}} '''Foo''' is a [[bar]]. {{uncategorized}}"
code = mwparserfromhell.parse(text)
for template in code.filter_templates():
@@ -101,6 +106,7 @@ class TestDocs(unittest.TestCase):
self.assertEqual(text, code)

def test_readme_5(self):
"""test a block of example code in the README; includes a web call"""
url1 = "http://en.wikipedia.org/w/api.php"
url2 = "http://en.wikipedia.org/w/index.php?title={0}&action=raw"
title = "Test"


+ 1
- 0
tests/test_pytokenizer.py Ver fichero

@@ -25,6 +25,7 @@ import unittest
from _test_tokenizer import TokenizerTestCase

class TestPyTokenizer(TokenizerTestCase, unittest.TestCase):
"""Test cases for the Python tokenizer."""
@classmethod
def setUpClass(cls):
from mwparserfromhell.parser.tokenizer import Tokenizer


+ 285
- 0
tests/tokenizer/templates.test Ver fichero

@@ -30,3 +30,288 @@ name: multiple_named_params
label: basic template with multiple named parameters
input: "{{foo|bar=baz|biz=buzz|buff=baff|usr=bin}}"
output: [TemplateOpen(), Text(text="foo"), TemplateParamSeparator(), Text(text="bar"), TemplateParamEquals(), Text(text="baz"), TemplateParamSeparator(), Text(text="biz"), TemplateParamEquals(), Text(text="buzz"), TemplateParamSeparator(), Text(text="buff"), TemplateParamEquals(), Text(text="baff"), TemplateParamSeparator(), Text(text="usr"), TemplateParamEquals(), Text(text="bin"), TemplateClose()]

---

name: multiple_mixed_params
label: basic template with multiple unnamed/named parameters
input: "{{foo|bar=baz|biz|buzz=buff|usr|bin}}"
output: [TemplateOpen(), Text(text="foo"), TemplateParamSeparator(), Text(text="bar"), TemplateParamEquals(), Text(text="baz"), TemplateParamSeparator(), Text(text="biz"), TemplateParamSeparator(), Text(text="buzz"), TemplateParamEquals(), Text(text="buff"), TemplateParamSeparator(), Text(text="usr"), TemplateParamSeparator(), Text(text="bin"), TemplateClose()]

---

name: multiple_mixed_params2
label: basic template with multiple unnamed/named parameters in another order
input: "{{foo|bar|baz|biz=buzz|buff=baff|usr=bin}}"
output: [TemplateOpen(), Text(text="foo"), TemplateParamSeparator(), Text(text="bar"), TemplateParamSeparator(), Text(text="baz"), TemplateParamSeparator(), Text(text="biz"), TemplateParamEquals(), Text(text="buzz"), TemplateParamSeparator(), Text(text="buff"), TemplateParamEquals(), Text(text="baff"), TemplateParamSeparator(), Text(text="usr"), TemplateParamEquals(), Text(text="bin"), TemplateClose()]

---

name: nested_unnamed_param
label: nested template as an unnamed parameter
input: "{{foo|{{bar}}}}"
output: [TemplateOpen(), Text(text="foo"), TemplateParamSeparator(), TemplateOpen(), Text(text="bar"), TemplateClose(), TemplateClose()]

---

name: nested_named_param_value
label: nested template as a parameter value with a named parameter
input: "{{foo|bar={{baz}}}}"
output: [TemplateOpen(), Text(text="foo"), TemplateParamSeparator(), Text(text="bar"), TemplateParamEquals(), TemplateOpen(), Text(text="baz"), TemplateClose(), TemplateClose()]

---

name: nested_named_param_name_and_value
label: nested templates as a parameter name and value
input: "{{foo|{{bar}}={{baz}}}}"
output: [TemplateOpen(), Text(text="foo"), TemplateParamSeparator(), TemplateOpen(), Text(text="bar"), TemplateClose(), TemplateParamEquals(), TemplateOpen(), Text(text="baz"), TemplateClose(), TemplateClose()]

---

name: nested_name_start
label: nested template at the beginning of a template name
input: "{{{{foo}}bar}}"
output: [TemplateOpen(), TemplateOpen(), Text(text="foo"), TemplateClose(), Text(text="bar"), TemplateClose()]

---

name: nested_name_start_unnamed_param
label: nested template at the beginning of a template name and as an unnamed parameter
input: "{{{{foo}}bar|{{baz}}}}"
output: [TemplateOpen(), TemplateOpen(), Text(text="foo"), TemplateClose(), Text(text="bar"), TemplateParamSeparator(), TemplateOpen(), Text(text="baz"), TemplateClose(), TemplateClose()]

---

name: nested_name_start_named_param_value
label: nested template at the beginning of a template name and as a parameter value with a named parameter
input: "{{{{foo}}bar|baz={{biz}}}}"
output: [TemplateOpen(), TemplateOpen(), Text(text="foo"), TemplateClose(), Text(text="bar"), TemplateParamSeparator(), Text(text="baz"), TemplateParamEquals(), TemplateOpen(), Text(text="biz"), TemplateClose(), TemplateClose()]

---

name: nested_name_start_named_param_name_and_value
label: nested template at the beginning of a template name and as a parameter name and value
input: "{{{{foo}}bar|{{baz}}={{biz}}}}"
output: [TemplateOpen(), TemplateOpen(), Text(text="foo"), TemplateClose(), Text(text="bar"), TemplateParamSeparator(), TemplateOpen(), Text(text="baz"), TemplateClose(), TemplateParamEquals(), TemplateOpen(), Text(text="biz"), TemplateClose(), TemplateClose()]

---

name: nested_name_end
label: nested template at the end of a template name
input: "{{foo{{bar}}}}"
output: [TemplateOpen(), Text(text="foo"), TemplateOpen(), Text(text="bar"), TemplateClose(), TemplateClose()]

---

name: nested_name_end_unnamed_param
label: nested template at the end of a template name and as an unnamed parameter
input: "{{foo{{bar}}|{{baz}}}}"
output: [TemplateOpen(), Text(text="foo"), TemplateOpen(), Text(text="bar"), TemplateClose(), TemplateParamSeparator(), TemplateOpen(), Text(text="baz"), TemplateClose(), TemplateClose()]

---

name: nested_name_end_named_param_value
label: nested template at the end of a template name and as a parameter value with a named parameter
input: "{{foo{{bar}}|baz={{biz}}}}"
output: [TemplateOpen(), Text(text="foo"), TemplateOpen(), Text(text="bar"), TemplateClose(), TemplateParamSeparator(), Text(text="baz"), TemplateParamEquals(), TemplateOpen(), Text(text="biz"), TemplateClose(), TemplateClose()]

---

name: nested_name_end_named_param_name_and_value
label: nested template at the end of a template name and as a parameter name and value
input: "{{foo{{bar}}|{{baz}}={{biz}}}}"
output: [TemplateOpen(), Text(text="foo"), TemplateOpen(), Text(text="bar"), TemplateClose(), TemplateParamSeparator(), TemplateOpen(), Text(text="baz"), TemplateClose(), TemplateParamEquals(), TemplateOpen(), Text(text="biz"), TemplateClose(), TemplateClose()]

---

name: nested_name_mid
label: nested template in the middle of a template name
input: "{{foo{{bar}}baz}}"
output: [TemplateOpen(), Text(text="foo"), TemplateOpen(), Text(text="bar"), TemplateClose(), Text(text="baz"), TemplateClose()]

---

name: nested_name_mid_unnamed_param
label: nested template in the middle of a template name and as an unnamed parameter
input: "{{foo{{bar}}baz|{{biz}}}}"
output: [TemplateOpen(), Text(text="foo"), TemplateOpen(), Text(text="bar"), TemplateClose(), Text(text="baz"), TemplateParamSeparator(), TemplateOpen(), Text(text="biz"), TemplateClose(), TemplateClose()]

---

name: nested_name_mid_named_param_value
label: nested template in the middle of a template name and as a parameter value with a named parameter
input: "{{foo{{bar}}baz|biz={{buzz}}}}"
output: [TemplateOpen(), Text(text="foo"), TemplateOpen(), Text(text="bar"), TemplateClose(), Text(text="baz"), TemplateParamSeparator(), Text(text="biz"), TemplateParamEquals(), TemplateOpen(), Text(text="buzz"), TemplateClose(), TemplateClose()]

---

name: nested_name_mid_named_param_name_and_value
label: nested template in the middle of a template name and as a parameter name and value
input: "{{foo{{bar}}baz|{{biz}}={{buzz}}}}"
output: [TemplateOpen(), Text(text="foo"), TemplateOpen(), Text(text="bar"), TemplateClose(), Text(text="baz"), TemplateParamSeparator(), TemplateOpen(), Text(text="biz"), TemplateClose(), TemplateParamEquals(), TemplateOpen(), Text(text="buzz"), TemplateClose(), TemplateClose()]

---

name: nested_name_start_end
label: nested template at the beginning and end of a template name
input: "{{{{foo}}{{bar}}}}"
output: [TemplateOpen(), TemplateOpen(), Text(text="foo"), TemplateClose(), TemplateOpen(), Text(text="bar"), TemplateClose(), TemplateClose()]

---

name: nested_name_start_end_unnamed_param
label: nested template at the beginning and end of a template name and as an unnamed parameter
input: "{{{{foo}}{{bar}}|{{baz}}}}"
output: [TemplateOpen(), TemplateOpen(), Text(text="foo"), TemplateClose(), TemplateOpen(), Text(text="bar"), TemplateClose(), TemplateParamSeparator(), TemplateOpen(), Text(text="baz"), TemplateClose(), TemplateClose()]

---

name: nested_name_start_end_named_param_value
label: nested template at the beginning and end of a template name and as a parameter value with a named parameter
input: "{{{{foo}}{{bar}}|baz={{biz}}}}"
output: [TemplateOpen(), TemplateOpen(), Text(text="foo"), TemplateClose(), TemplateOpen(), Text(text="bar"), TemplateClose(), TemplateParamSeparator(), Text(text="baz"), TemplateParamEquals(), TemplateOpen(), Text(text="biz"), TemplateClose(), TemplateClose()]

---

name: nested_name_start_end_named_param_name_and_value
label: nested template at the beginning and end of a template name and as a parameter name and value
input: "{{{{foo}}{{bar}}|{{baz}}={{biz}}}}"
output: [TemplateOpen(), TemplateOpen(), Text(text="foo"), TemplateClose(), TemplateOpen(), Text(text="bar"), TemplateClose(), TemplateParamSeparator(), TemplateOpen(), Text(text="baz"), TemplateClose(), TemplateParamEquals(), TemplateOpen(), Text(text="biz"), TemplateClose(), TemplateClose()]

---

name: nested_names_multiple
label: multiple nested templates within nested templates
input: "{{{{{{{{foo}}bar}}baz}}biz}}"
output: [TemplateOpen(), TemplateOpen(), TemplateOpen(), TemplateOpen(), Text(text="foo"), TemplateClose(), Text(text="bar"), TemplateClose(), Text(text="baz"), TemplateClose(), Text(text="biz"), TemplateClose()]

---

name: nested_names_multiple_unnamed_param
label: multiple nested templates within nested templates with a nested unnamed parameter
input: "{{{{{{{{foo}}bar}}baz}}biz|{{buzz}}}}"
output: [TemplateOpen(), TemplateOpen(), TemplateOpen(), TemplateOpen(), Text(text="foo"), TemplateClose(), Text(text="bar"), TemplateClose(), Text(text="baz"), TemplateClose(), Text(text="biz"), TemplateParamSeparator(), TemplateOpen(), Text(text="buzz"), TemplateClose(), TemplateClose()]

---

name: nested_names_multiple_named_param_value
label: multiple nested templates within nested templates with a nested parameter value in a named parameter
input: "{{{{{{{{foo}}bar}}baz}}biz|buzz={{bin}}}}"
output: [TemplateOpen(), TemplateOpen(), TemplateOpen(), TemplateOpen(), Text(text="foo"), TemplateClose(), Text(text="bar"), TemplateClose(), Text(text="baz"), TemplateClose(), Text(text="biz"), TemplateParamSeparator(), Text(text="buzz"), TemplateParamEquals(), TemplateOpen(), Text(text="bin"), TemplateClose(), TemplateClose()]

---

name: nested_names_multiple_named_param_name_and_value
label: multiple nested templates within nested templates with a nested parameter name and value
input: "{{{{{{{{foo}}bar}}baz}}biz|{{buzz}}={{bin}}}}"
output: [TemplateOpen(), TemplateOpen(), TemplateOpen(), TemplateOpen(), Text(text="foo"), TemplateClose(), Text(text="bar"), TemplateClose(), Text(text="baz"), TemplateClose(), Text(text="biz"), TemplateParamSeparator(), TemplateOpen(), Text(text="buzz"), TemplateClose(), TemplateParamEquals(), TemplateOpen(), Text(text="bin"), TemplateClose(), TemplateClose()]

---

name: incomplete_tests

"{{{{{{{{foo}}bar|baz=biz}}buzz}}usr|{{bin}}}}"

"{{\nfoobar}}"
"{{foobar\n}}"
"{{\nfoobar\n}}"
"{{foo\nbar}}"
"{{\nfoo\nbar}}"
"{{foo\nbar\n}}"
"{{\nfoo\nbar\n}}"

"{{foo|\nbar}}"
"{{foo|bar\n}}"
"{{foo|\nbar\n}}"
"{{foo|\nb\nar}}"
"{{foo|b\nar\n}}"
"{{foo|\nb\nar\n}}"
"{{\nfoo|\nbar}}"
"{{\nfoo|bar\n}}"
"{{\nfoo|\nbar\n}}"
"{{\nfoo|\nb\nar}}"
"{{\nfoo|b\nar\n}}"
"{{\nfoo|\nb\nar\n}}"
"{{foo\n|\nbar}}"
"{{foo\n|bar\n}}"
"{{foo\n|\nbar\n}}"
"{{foo\n|\nb\nar}}"
"{{foo\n|b\nar\n}}"
"{{foo\n|\nb\nar\n}}"
"{{\nfoo\n|\nbar}}"
"{{\nfoo\n|bar\n}}"
"{{\nfoo\n|\nbar\n}}"
"{{\nfoo\n|\nb\nar}}"
"{{\nfoo\n|b\nar\n}}"
"{{\nfoo\n|\nb\nar\n}}"
"{{f\noo|\nbar}}"
"{{\nf\noo|\nbar}}"
"{{f\noo\n|\nbar}}"
"{{\nf\noo\n|\nbar}}"

"{{foo|1=\nbar}}"
"{{foo|1=bar\n}}"
"{{foo|1=\nbar\n}}"
"{{foo|1=\nb\nar}}"
"{{foo|1=b\nar\n}}"
"{{foo|1=\nb\nar\n}}"
"{{foo|\nbar=baz}}"
"{{foo|bar\n=baz}}"
"{{foo|\nbar\n=baz}}"
"{{foo|\nb\nar=baz}}"
"{{foo|b\nar\n=baz}}"
"{{foo|\nb\nar\n=baz}}"
"{{foo|\nbar=baz\n}}"
"{{foo|bar\n=baz\n}}"
"{{foo|\nbar\n=baz\n}}"
"{{foo|\nb\nar=baz\n}}"
"{{foo|b\nar\n=baz\n}}"
"{{foo|\nb\nar\n=baz\n}}"
"{{foo|\nbar=\nbaz}}"
"{{foo|bar\n=\nbaz}}"
"{{foo|\nbar\n=\nbaz}}"
"{{foo|\nb\nar=\nbaz}}"
"{{foo|b\nar\n=\nbaz}}"
"{{foo|\nb\nar\n=\nbaz}}"
"{{foo|\nbar=\nbaz\n}}"
"{{foo|bar\n=\nbaz\n}}"
"{{foo|\nbar\n=\nbaz\n}}"
"{{foo|\nb\nar=\nbaz\n}}"
"{{foo|b\nar\n=\nbaz\n}}"
"{{foo|\nb\nar\n=\nbaz\n}}"
"{{foo|\nbar=ba\nz}}"
"{{foo|bar\n=ba\nz}}"
"{{foo|\nbar\n=ba\nz}}"
"{{foo|\nb\nar=ba\nz}}"
"{{foo|b\nar\n=ba\nz}}"
"{{foo|\nb\nar\n=ba\nz}}"

"{{\nfoo\n|\nbar\n=\nb\naz\n|\nb\nuz\n}}"
"{{\nfoo\n|\nb\nar\n|\nbaz\n=\nb\nuz\n}}"

"{{\nfoo\n|\n{{\nbar\n|\nbaz\n=\nb\niz\n}}\n=\nb\nuzz\n}}"

"{{foo{bar}}"
"{{foo}bar}}"
"{{{foobar}}"
"{{foo{b{ar}}"
"{{foo[bar}}"
"{{foo]bar}}"
"{{[foobar}}"
"{{foobar]}}"

"{{foobar"
"{{foobar}"
"{{foobar|"
"{{foo|bar"
"{{foo|bar|"
"{{foo|bar="
"{{foo|bar=|"
"{{foo|bar=baz"
"{{foo|bar=baz|"
"{{foo|bar|baz"
"{{foo|bar|baz="
"{{foo|bar|baz=biz"
"{{foo|bar=baz|biz"
"{{foo|bar=baz|biz="
"{{foo|bar=baz|biz=buzz"

Cargando…
Cancelar
Guardar