diff --git a/src/mwparserfromhell/nodes/tag.py b/src/mwparserfromhell/nodes/tag.py
index a314847..8ade3f8 100644
--- a/src/mwparserfromhell/nodes/tag.py
+++ b/src/mwparserfromhell/nodes/tag.py
@@ -227,7 +227,7 @@ class Tag(Node):
return self._padding
@padding.setter
- def padding(self, value: str) -> None:
+ def padding(self, value: str | None) -> None:
if not value:
self._padding = ""
else:
@@ -278,7 +278,7 @@ class Tag(Node):
def closing_wiki_markup(self, value: str | None) -> None:
self._closing_wiki_markup = str(value) if value else None
- def has(self, name: str) -> bool:
+ def has(self, name: str | Attribute | Wikicode) -> bool:
"""Return whether any attribute in the tag has the given *name*.
Note that a tag may have multiple attributes with the same name, but
@@ -289,7 +289,7 @@ class Tag(Node):
return True
return False
- def get(self, name: str) -> Attribute:
+ def get(self, name: str | Attribute | Wikicode) -> Attribute:
"""Get the attribute with the given *name*.
The returned object is a :class:`.Attribute` instance. Raises
diff --git a/src/mwparserfromhell/nodes/template.py b/src/mwparserfromhell/nodes/template.py
index 2fd35cd..fb598ec 100644
--- a/src/mwparserfromhell/nodes/template.py
+++ b/src/mwparserfromhell/nodes/template.py
@@ -42,7 +42,6 @@ if TYPE_CHECKING:
__all__ = ["Template"]
-FLAGS = re.DOTALL | re.UNICODE
# Used to allow None as a valid fallback value
_UNSET = object()
@@ -134,7 +133,7 @@ class Template(Node):
if sval.isspace():
before, after = "", sval
else:
- match = re.search(r"^(\s*).*?(\s*)$", sval, FLAGS)
+ match = re.search(r"^(\s*).*?(\s*)$", sval, re.DOTALL)
assert match, sval
before, after = match.group(1), match.group(2)
value.nodes = [Text(before), Text(after)]
@@ -157,7 +156,7 @@ class Template(Node):
component = str(param.name)
else:
component = str(param.value)
- match = re.search(r"^(\s*).*?(\s*)$", component, FLAGS)
+ match = re.search(r"^(\s*).*?(\s*)$", component, re.DOTALL)
assert match, component
before, after = match.group(1), match.group(2)
if not use_names and component.isspace() and "\n" in before:
@@ -377,7 +376,7 @@ class Template(Node):
def __setitem__(self, name: Any, value: Any) -> Parameter:
return self.add(name, value)
- def remove(self, param: Parameter | str, keep_field: bool = False) -> None:
+ def remove(self, param: Parameter | str | int, keep_field: bool = False) -> None:
"""Remove a parameter from the template, identified by *param*.
If *param* is a :class:`.Parameter` object, it will be matched exactly,
diff --git a/src/mwparserfromhell/wikicode.py b/src/mwparserfromhell/wikicode.py
index 616433c..160fd61 100644
--- a/src/mwparserfromhell/wikicode.py
+++ b/src/mwparserfromhell/wikicode.py
@@ -43,7 +43,7 @@ from .utils import parse_anything
__all__ = ["Wikicode"]
-FLAGS = re.IGNORECASE | re.DOTALL | re.UNICODE
+FLAGS = re.IGNORECASE | re.DOTALL
N = TypeVar("N", bound=Node)
diff --git a/tests/test_comment.py b/tests/test_comment.py
index 9055036..3c0e300 100644
--- a/tests/test_comment.py
+++ b/tests/test_comment.py
@@ -53,7 +53,7 @@ def test_showtree():
"""test Comment.__showtree__()"""
output = []
node = Comment("foobar")
- node.__showtree__(output.append, None, None)
+ node.__showtree__(output.append, lambda _code: None, lambda: None)
assert [""] == output
diff --git a/tests/test_heading.py b/tests/test_heading.py
index 5e9a5e2..ed65aec 100644
--- a/tests/test_heading.py
+++ b/tests/test_heading.py
@@ -64,8 +64,8 @@ def test_showtree():
node1 = Heading(wraptext("foobar"), 3)
node2 = Heading(wraptext(" baz "), 4)
- node1.__showtree__(output.append, get, None)
- node2.__showtree__(output.append, get, None)
+ node1.__showtree__(output.append, get, lambda: None)
+ node2.__showtree__(output.append, get, lambda: None)
valid = ["===", (getter, node1.title), "===", "====", (getter, node2.title), "===="]
assert valid == output
diff --git a/tests/test_html_entity.py b/tests/test_html_entity.py
index d96ae56..38007c3 100644
--- a/tests/test_html_entity.py
+++ b/tests/test_html_entity.py
@@ -69,9 +69,9 @@ def test_showtree():
node1 = HTMLEntity("nbsp", named=True, hexadecimal=False)
node2 = HTMLEntity("107", named=False, hexadecimal=False)
node3 = HTMLEntity("e9", named=False, hexadecimal=True)
- node1.__showtree__(output.append, None, None)
- node2.__showtree__(output.append, None, None)
- node3.__showtree__(output.append, None, None)
+ node1.__showtree__(output.append, lambda _code: None, lambda: None)
+ node2.__showtree__(output.append, lambda _code: None, lambda: None)
+ node3.__showtree__(output.append, lambda _code: None, lambda: None)
res = [" ", "k", "é"]
assert res == output
@@ -125,9 +125,9 @@ def test_named():
assert node1.named is True
assert node2.named is False
assert node3.named is False
- node1.named = 1
- node2.named = 0
- node3.named = 0
+ node1.named = True
+ node2.named = False
+ node3.named = False
assert node1.named is True
assert node2.named is False
assert node3.named is False
diff --git a/tests/test_tag.py b/tests/test_tag.py
index a149575..3a3ef42 100644
--- a/tests/test_tag.py
+++ b/tests/test_tag.py
@@ -208,7 +208,7 @@ def test_wiki_markup():
node.wiki_markup = "''"
assert "''" == node.wiki_markup
assert "''italic text''" == node
- node.wiki_markup = False
+ node.wiki_markup = None
assert node.wiki_markup is None
assert "italic text" == node
@@ -220,7 +220,7 @@ def test_self_closing():
node.self_closing = True
assert node.self_closing is True
assert "[foobar]" == node
@@ -232,7 +232,7 @@ def test_invalid():
node.invalid = True
assert node.invalid is True
assert "" == node
- node.invalid = 0
+ node.invalid = False
assert node.invalid is False
assert "
" == node
@@ -244,7 +244,7 @@ def test_implicit():
node.implicit = True
assert node.implicit is True
assert "
" == node
- node.implicit = 0
+ node.implicit = False
assert node.implicit is False
assert "
" == node
@@ -298,7 +298,7 @@ def test_closing_wiki_markup():
node.wiki_markup = "!!"
assert "|}" == node.closing_wiki_markup
assert "!!\n|}" == node
- node.wiki_markup = False
+ node.wiki_markup = None
assert node.closing_wiki_markup is None
assert "" == node
node2 = Tag(
diff --git a/tests/test_text.py b/tests/test_text.py
index 011d779..cccd53e 100644
--- a/tests/test_text.py
+++ b/tests/test_text.py
@@ -57,9 +57,9 @@ def test_showtree():
node1 = Text("foobar")
node2 = Text("fóóbar")
node3 = Text("𐌲𐌿𐍄")
- node1.__showtree__(output.append, None, None)
- node2.__showtree__(output.append, None, None)
- node3.__showtree__(output.append, None, None)
+ node1.__showtree__(output.append, lambda _code: None, lambda: None)
+ node2.__showtree__(output.append, lambda _code: None, lambda: None)
+ node3.__showtree__(output.append, lambda _code: None, lambda: None)
res = ["foobar", r"f\xf3\xf3bar", "\\U00010332\\U0001033f\\U00010344"]
assert res == output
diff --git a/tests/test_tokenizer.py b/tests/test_tokenizer.py
index cfa9af6..8cab57c 100644
--- a/tests/test_tokenizer.py
+++ b/tests/test_tokenizer.py
@@ -20,9 +20,9 @@
from __future__ import annotations
-import codecs
-import warnings
-from os import listdir, path
+import os
+from collections.abc import Generator
+from dataclasses import dataclass
import pytest
@@ -40,71 +40,79 @@ class _TestParseError(Exception):
"""Raised internally when a test could not be parsed."""
-def _parse_test(test, data):
- """Parse an individual *test*, storing its info in *data*."""
- for line in test.strip().splitlines():
- if line.startswith("name:"):
- data["name"] = line[len("name:") :].strip()
- elif line.startswith("label:"):
- data["label"] = line[len("label:") :].strip()
- elif line.startswith("input:"):
- raw = line[len("input:") :].strip()
- if raw[0] == '"' and raw[-1] == '"':
- raw = raw[1:-1]
- raw = raw.encode("raw_unicode_escape")
- data["input"] = raw.decode("unicode_escape")
- elif line.startswith("output:"):
- raw = line[len("output:") :].strip()
- try:
- data["output"] = eval(raw, vars(tokens))
- except Exception as err:
- raise _TestParseError(err) from err
-
-
-def _load_tests(filename, name, text):
- """Load all tests in *text* from the file *filename*."""
+@dataclass
+class _TestCase:
+ name: str
+ label: str | None
+ input: str
+ output: list[tokens.Token]
+
+
+def _load_tests(file_path, filename, text) -> Generator[_TestCase]:
tests = text.split("\n---\n")
for test in tests:
- data = {"name": None, "label": None, "input": None, "output": None}
- try:
- _parse_test(test, data)
- except _TestParseError as err:
- if data["name"]:
- error = "Could not parse test '{0}' in '{1}':\n\t{2}"
- warnings.warn(error.format(data["name"], filename, err))
- else:
- error = "Could not parse a test in '{0}':\n\t{1}"
- warnings.warn(error.format(filename, err))
- continue
-
- if not data["name"]:
- error = "A test in '{0}' was ignored because it lacked a name"
- warnings.warn(error.format(filename))
+ name = None
+ label = None
+ input = None
+ output = None
+
+ for line in test.strip().splitlines():
+ if line.startswith("name:"):
+ name = line[len("name:") :].strip()
+ elif line.startswith("label:"):
+ label = line[len("label:") :].strip()
+ elif line.startswith("input:"):
+ raw = line[len("input:") :].strip()
+ if raw[0] == '"' and raw[-1] == '"':
+ raw = raw[1:-1]
+ raw = raw.encode("raw_unicode_escape")
+ input = raw.decode("unicode_escape")
+ elif line.startswith("output:"):
+ raw = line[len("output:") :].strip()
+ try:
+ output = eval(raw, dict(vars(tokens)))
+ except Exception as err:
+ raise _TestParseError(
+ f"Could not parse a test in {file_path!r}:\n\t{err}\n\n"
+ f"Raw test:\n{test}"
+ )
+
+ if not name:
+ raise _TestParseError(
+ f"Test in {filename!r} lacks a name\nRaw test:\n{test}"
+ )
continue
- if data["input"] is None or data["output"] is None:
- error = (
- "Test '{}' in '{}' was ignored because it lacked an input or an output"
+ if input is None or output is None:
+ raise _TestParseError(
+ f"Test {name!r} in {file_path!r} lacks an input or an output"
)
- warnings.warn(error.format(data["name"], filename))
continue
+ if not isinstance(output, list) or not all(
+ isinstance(tok, tokens.Token) for tok in output
+ ):
+ raise _TestParseError(
+ f"Test {name!r} in {file_path!r} should have a list of tokens as its output"
+ )
- # Include test filename in name
- data["name"] = "{}:{}".format(name, data["name"])
-
- yield data
+ yield _TestCase(
+ name=f"{filename}:{name}",
+ label=label,
+ input=input,
+ output=output,
+ )
def build():
"""Load and install all tests from the 'tokenizer' directory."""
- directory = path.join(path.dirname(__file__), "tokenizer")
+ directory = os.path.join(os.path.dirname(__file__), "tokenizer")
extension = ".mwtest"
- for filename in listdir(directory):
+ for filename in os.listdir(directory):
if not filename.endswith(extension):
continue
- fullname = path.join(directory, filename)
- with codecs.open(fullname, "r", encoding="utf8") as fp:
+ fullname = os.path.join(directory, filename)
+ with open(fullname) as fp:
text = fp.read()
- name = path.split(fullname)[1][: -len(extension)]
+ name = os.path.split(fullname)[1][: -len(extension)]
yield from _load_tests(fullname, name, text)
@@ -113,23 +121,22 @@ def build():
filter(None, (CTokenizer, PyTokenizer)),
ids=lambda t: "CTokenizer" if t.USES_C else "PyTokenizer",
)
-@pytest.mark.parametrize("data", build(), ids=lambda data: data["name"])
-def test_tokenizer(tokenizer, data):
- expected = data["output"]
- actual = tokenizer().tokenize(data["input"])
- assert expected == actual
+@pytest.mark.parametrize("test_case", build(), ids=lambda test_case: test_case.name)
+def test_tokenizer(tokenizer, test_case: _TestCase):
+ actual = tokenizer().tokenize(test_case.input)
+ assert test_case.output == actual
-@pytest.mark.parametrize("data", build(), ids=lambda data: data["name"])
-def test_roundtrip(data):
- expected = data["input"]
- actual = str(Builder().build(data["output"][:]))
- assert expected == actual
+@pytest.mark.parametrize("test_case", build(), ids=lambda test_case: test_case.name)
+def test_roundtrip(test_case: _TestCase):
+ actual = str(Builder().build(test_case.output[:]))
+ assert test_case.input == actual
@pytest.mark.skipif(CTokenizer is None, reason="CTokenizer not available")
def test_c_tokenizer_uses_c():
"""make sure the C tokenizer identifies as using a C extension"""
+ assert CTokenizer is not None
assert CTokenizer.USES_C is True
assert CTokenizer().USES_C is True
diff --git a/tests/test_tokens.py b/tests/test_tokens.py
index d425205..6239ae1 100644
--- a/tests/test_tokens.py
+++ b/tests/test_tokens.py
@@ -96,4 +96,4 @@ def test_equality():
)
def test_repr_equality(token):
"""check that eval(repr(token)) == token"""
- assert token == eval(repr(token), vars(tokens))
+ assert token == eval(repr(token), dict(vars(tokens)))