From 5a0a00ba98f0edde985239cc4717e70c0d37c618 Mon Sep 17 00:00:00 2001 From: Ben Kurtovic Date: Sun, 3 Mar 2013 20:29:34 -0500 Subject: [PATCH] Change the way verify_safe() handles template params (#25). - Newlines are now allowed in template param names. - Changes also affect handling of arguments like {{{foo}}}. - Update unit tests: remove some unnecessary ones, and add some to cover the changes. - Update StringMixIn tests to actually work for some of the methods. - Update copyright notices for the C extensions. --- mwparserfromhell/parser/tokenizer.c | 2 +- mwparserfromhell/parser/tokenizer.h | 2 +- mwparserfromhell/parser/tokenizer.py | 70 ++--- tests/test_string_mixin.py | 166 ++++++----- tests/tokenizer/templates.mwtest | 540 +++++++---------------------------- 5 files changed, 235 insertions(+), 545 deletions(-) diff --git a/mwparserfromhell/parser/tokenizer.c b/mwparserfromhell/parser/tokenizer.c index d82b080..6716698 100644 --- a/mwparserfromhell/parser/tokenizer.c +++ b/mwparserfromhell/parser/tokenizer.c @@ -1,6 +1,6 @@ /* Tokenizer for MWParserFromHell -Copyright (C) 2012 Ben Kurtovic +Copyright (C) 2012-2013 Ben Kurtovic Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in diff --git a/mwparserfromhell/parser/tokenizer.h b/mwparserfromhell/parser/tokenizer.h index af86321..8d51013 100644 --- a/mwparserfromhell/parser/tokenizer.h +++ b/mwparserfromhell/parser/tokenizer.h @@ -1,6 +1,6 @@ /* Tokenizer Header File for MWParserFromHell -Copyright (C) 2012 Ben Kurtovic +Copyright (C) 2012-2013 Ben Kurtovic Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in diff --git a/mwparserfromhell/parser/tokenizer.py b/mwparserfromhell/parser/tokenizer.py index a365db8..67638ca 100644 --- a/mwparserfromhell/parser/tokenizer.py +++ b/mwparserfromhell/parser/tokenizer.py @@ -396,34 +396,42 @@ class Tokenizer(object): """Make sure we are not trying to write an invalid character.""" context = self._context if context & contexts.FAIL_NEXT: - self._fail_route() + return False if context & contexts.WIKILINK_TITLE: if this == "]" or this == "{": self._context |= contexts.FAIL_NEXT elif this == "\n" or this == "[" or this == "}": - self._fail_route() - return + return False + return True if context & contexts.TEMPLATE_NAME: if this == "{" or this == "}" or this == "[": self._context |= contexts.FAIL_NEXT - return + return True if this == "]": - self._fail_route() - return + return False if this == "|": - return - elif context & (contexts.TEMPLATE_PARAM_KEY | contexts.ARGUMENT_NAME): + return True + if context & contexts.HAS_TEXT: + if context & contexts.FAIL_ON_TEXT: + if this is self.END or not this.isspace(): + return False + else: + if this == "\n": + self._context |= contexts.FAIL_ON_TEXT + elif this is not self.END or not this.isspace(): + self._context |= contexts.HAS_TEXT + return True + else: if context & contexts.FAIL_ON_EQUALS: if this == "=": - self._fail_route() - return + return False elif context & contexts.FAIL_ON_LBRACE: - if this == "{": + if this == "{" or (self._read(-1) == self._read(-2) == "{"): if context & contexts.TEMPLATE: self._context |= contexts.FAIL_ON_EQUALS else: self._context |= contexts.FAIL_NEXT - return + return True self._context ^= contexts.FAIL_ON_LBRACE elif context & contexts.FAIL_ON_RBRACE: if this == "}": @@ -431,40 +439,13 @@ class Tokenizer(object): self._context |= contexts.FAIL_ON_EQUALS else: self._context |= contexts.FAIL_NEXT - return + return True self._context ^= contexts.FAIL_ON_RBRACE elif this == "{": self._context |= contexts.FAIL_ON_LBRACE elif this == "}": self._context |= contexts.FAIL_ON_RBRACE - if context & contexts.HAS_TEXT: - if context & contexts.FAIL_ON_TEXT: - if this is self.END or not this.isspace(): - if context & contexts.TEMPLATE_PARAM_KEY: - self._context ^= contexts.FAIL_ON_TEXT - self._context |= contexts.FAIL_ON_EQUALS - else: - self._fail_route() - return - else: - if this == "\n": - self._context |= contexts.FAIL_ON_TEXT - elif this is self.END or not this.isspace(): - self._context |= contexts.HAS_TEXT - - def _reset_safety_checks(self): - """Unset any safety-checking contexts set by Tokenizer_verify_safe(). - - Used when we preserve a context but previous data becomes invalid, like - when moving between template parameters. - """ - context = self._context - checks = (contexts.HAS_TEXT, contexts.FAIL_ON_TEXT, contexts.FAIL_NEXT, - contexts.FAIL_ON_LBRACE, contexts.FAIL_ON_RBRACE, - contexts.FAIL_ON_EQUALS) - for check in checks: - if context & check: - self._context ^= check; + return True def _parse(self, context=0): """Parse the wikicode string, using *context* for when to stop.""" @@ -474,12 +455,10 @@ class Tokenizer(object): unsafe = (contexts.TEMPLATE_NAME | contexts.WIKILINK_TITLE | contexts.TEMPLATE_PARAM_KEY | contexts.ARGUMENT_NAME) if self._context & unsafe: - try: - self._verify_safe(this) - except BadRoute: + if not self._verify_safe(this): if self._context & contexts.TEMPLATE_PARAM_KEY: self._pop() - raise + self._fail_route() if this not in self.MARKERS: self._write_text(this) self._head += 1 @@ -504,7 +483,6 @@ class Tokenizer(object): if self._context & contexts.FAIL_NEXT: self._context ^= contexts.FAIL_NEXT elif this == "|" and self._context & contexts.TEMPLATE: - self._reset_safety_checks() self._handle_template_param() elif this == "=" and self._context & contexts.TEMPLATE_PARAM_KEY: self._handle_template_param_value() diff --git a/tests/test_string_mixin.py b/tests/test_string_mixin.py index 4e4fa68..43a9e9a 100644 --- a/tests/test_string_mixin.py +++ b/tests/test_string_mixin.py @@ -42,12 +42,14 @@ class TestStringMixIn(unittest.TestCase): methods = [ "capitalize", "center", "count", "encode", "endswith", "expandtabs", "find", "format", "index", "isalnum", "isalpha", - "isdecimal", "isdigit", "islower", "isnumeric", "isspace", - "istitle", "isupper", "join", "ljust", "lstrip", "partition", - "replace", "rfind", "rindex", "rjust", "rpartition", "rsplit", - "rstrip", "split", "splitlines", "startswith", "strip", "swapcase", - "title", "translate", "upper", "zfill"] - if not py3k: + "isdigit", "islower", "isnumeric", "isspace", "istitle", "isupper", + "join", "ljust", "lstrip", "partition", "replace", "rfind", + "rindex", "rjust", "rpartition", "rsplit", "rstrip", "split", + "splitlines", "startswith", "strip", "swapcase", "title", + "translate", "upper", "zfill"] + if py3k: + methods.append("isdecimal") + else: methods.append("decode") for meth in methods: expected = getattr(str, meth).__doc__ @@ -157,75 +159,107 @@ class TestStringMixIn(unittest.TestCase): def test_other_methods(self): """test the remaining non-magic methods of StringMixIn""" - fstr = _FakeString("fake string") - - self.assertEquals("Fake string", fstr.capitalize()) + str1 = _FakeString("fake string") + self.assertEquals("Fake string", str1.capitalize()) - self.assertEquals(" fake string ", fstr.center(15)) - self.assertEquals(" fake string ", fstr.center(16)) - self.assertEquals("qqfake stringqq", fstr.center(15, "q")) + self.assertEquals(" fake string ", str1.center(15)) + self.assertEquals(" fake string ", str1.center(16)) + self.assertEquals("qqfake stringqq", str1.center(15, "q")) - self.assertEquals(1, fstr.count("e")) - self.assertEquals(0, fstr.count("z")) - self.assertEquals(1, fstr.count("r", 7)) - self.assertEquals(0, fstr.count("r", 8)) - self.assertEquals(1, fstr.count("r", 5, 9)) - self.assertEquals(0, fstr.count("r", 5, 7)) + self.assertEquals(1, str1.count("e")) + self.assertEquals(0, str1.count("z")) + self.assertEquals(1, str1.count("r", 7)) + self.assertEquals(0, str1.count("r", 8)) + self.assertEquals(1, str1.count("r", 5, 9)) + self.assertEquals(0, str1.count("r", 5, 7)) if not py3k: - self.assertEquals(fstr, fstr.decode()) - actual = '\\U00010332\\U0001033f\\U00010344' + str2 = _FakeString("fo") + self.assertEquals(str1, str1.decode()) + actual = _FakeString("\\U00010332\\U0001033f\\U00010344") self.assertEquals("πŒ²πŒΏπ„", actual.decode("unicode_escape")) - self.assertEquals("𐌲", '\\U00010332'.decode("unicode_escape")) - self.assertRaises(UnicodeError, "fo".decode, "punycode") - self.assertEquals("", "fo".decode("punycode", "ignore")) + self.assertRaises(UnicodeError, str2.decode, "punycode") + self.assertEquals("", str2.decode("punycode", "ignore")) - self.assertEquals(b"fake string", fstr.encode()) + str3 = _FakeString("πŒ²πŒΏπ„") + self.assertEquals(b"fake string", str1.encode()) self.assertEquals(b"\xF0\x90\x8C\xB2\xF0\x90\x8C\xBF\xF0\x90\x8D\x84", - "πŒ²πŒΏπ„".encode("utf8")) - self.assertRaises(UnicodeEncodeError, "πŒ²πŒΏπ„".encode) - self.assertRaises(UnicodeEncodeError, "πŒ²πŒΏπ„".encode, "ascii") - self.assertRaises(UnicodeEncodeError, "πŒ²πŒΏπ„".encode, "ascii", "strict") - self.assertEquals("", "πŒ²πŒΏπ„".encode("ascii", "ignore")) - - self.assertTrue(fstr.endswith("ing")) - self.assertFalse(fstr.endswith("ingh")) - - self.assertEquals("fake string", fstr) - self.assertEquals(" foobar", "\tfoobar".expandtabs()) - self.assertEquals(" foobar", "\tfoobar".expandtabs(4)) - - self.assertEquals(3, fstr.find("e")) - self.assertEquals(-1, fstr.find("z")) - self.assertEquals(7, fstr.find("r", 7)) - self.assertEquals(-1, fstr.find("r", 8)) - self.assertEquals(7, fstr.find("r", 5, 9)) - self.assertEquals(-1, fstr.find("r", 5, 7)) - - self.assertEquals("fake string", fstr.format()) - self.assertEquals("foobarbaz", "foo{0}baz".format("bar")) - self.assertEquals("foobarbaz", "foo{abc}baz".format(abc="bar")) - self.assertEquals("foobarbazbuzz", - "foo{0}{abc}buzz".format("bar", abc="baz")) - self.assertRaises(IndexError, "{0}{1}".format, "abc") - - self.assertEquals(3, fstr.index("e")) - self.assertRaises(ValueError, fstr.index, "z") - self.assertEquals(7, fstr.index("r", 7)) - self.assertRaises(ValueError, fstr.index, "r", 8) - self.assertEquals(7, fstr.index("r", 5, 9)) - self.assertRaises(ValueError, fstr.index, "r", 5, 7) - - self.assertTrue("foobar".isalnum()) - self.assertTrue("foobar123".isalnum()) - self.assertFalse("foo bar".isalnum()) - - self.assertTrue("foobar".isalpha()) - self.assertFalse("foobar123".isalpha()) - self.assertFalse("foo bar".isalpha()) + str3.encode("utf8")) + self.assertRaises(UnicodeEncodeError, str3.encode) + self.assertRaises(UnicodeEncodeError, str3.encode, "ascii") + self.assertRaises(UnicodeEncodeError, str3.encode, "ascii", "strict") + self.assertEquals("", str3.encode("ascii", "ignore")) + + self.assertTrue(str1.endswith("ing")) + self.assertFalse(str1.endswith("ingh")) + + str4 = _FakeString("\tfoobar") + self.assertEquals("fake string", str1) + self.assertEquals(" foobar", str4.expandtabs()) + self.assertEquals(" foobar", str4.expandtabs(4)) + + self.assertEquals(3, str1.find("e")) + self.assertEquals(-1, str1.find("z")) + self.assertEquals(7, str1.find("r", 7)) + self.assertEquals(-1, str1.find("r", 8)) + self.assertEquals(7, str1.find("r", 5, 9)) + self.assertEquals(-1, str1.find("r", 5, 7)) + + str5 = _FakeString("foo{0}baz") + str6 = _FakeString("foo{abc}baz") + str7 = _FakeString("foo{0}{abc}buzz") + str8 = _FakeString("{0}{1}") + self.assertEquals("fake string", str1.format()) + self.assertEquals("foobarbaz", str5.format("bar")) + self.assertEquals("foobarbaz", str6.format(abc="bar")) + self.assertEquals("foobarbazbuzz", str7.format("bar", abc="baz")) + self.assertRaises(IndexError, str8.format, "abc") + + self.assertEquals(3, str1.index("e")) + self.assertRaises(ValueError, str1.index, "z") + self.assertEquals(7, str1.index("r", 7)) + self.assertRaises(ValueError, str1.index, "r", 8) + self.assertEquals(7, str1.index("r", 5, 9)) + self.assertRaises(ValueError, str1.index, "r", 5, 7) + + str9 = _FakeString("foobar") + str10 = _FakeString("foobar123") + str11 = _FakeString("foo bar") + self.assertTrue(str9.isalnum()) + self.assertTrue(str10.isalnum()) + self.assertFalse(str11.isalnum()) + + self.assertTrue(str9.isalpha()) + self.assertFalse(str10.isalpha()) + self.assertFalse(str11.isalpha()) + + str12 = _FakeString("123") + str13 = _FakeString("\u2155") + str14 = _FakeString("\u00B2") + if py3k: + self.assertFalse(str9.isdecimal()) + self.assertTrue(str12.isdecimal()) + self.assertFalse(str13.isdecimal()) + self.assertFalse(str14.isdecimal()) + + self.assertFalse(str9.isdigit()) + self.assertTrue(str12.isdigit()) + self.assertFalse(str13.isdigit()) + self.assertTrue(str14.isdigit()) + + str15 = _FakeString("") + str16 = _FakeString("FooBar") + self.assertTrue(str9.islower()) + self.assertFalse(str15.islower()) + self.assertFalse(str16.islower()) + + self.assertFalse(str9.isnumeric()) + self.assertTrue(str12.isnumeric()) + self.assertTrue(str13.isnumeric()) + self.assertTrue(str14.isnumeric()) methods = [ - "isdecimal", "isdigit", "islower", "isnumeric", "isspace", + "isspace", "istitle", "isupper", "join", "ljust", "lstrip", "partition", "replace", "rfind", "rindex", "rjust", "rpartition", "rsplit", "rstrip", "split", "splitlines", "startswith", "strip", "swapcase", diff --git a/tests/tokenizer/templates.mwtest b/tests/tokenizer/templates.mwtest index d699ef2..fa3c0a4 100644 --- a/tests/tokenizer/templates.mwtest +++ b/tests/tokenizer/templates.mwtest @@ -215,521 +215,150 @@ output: [TemplateOpen(), TemplateOpen(), TemplateOpen(), TemplateOpen(), Text(te --- -name: newline_start +name: newlines_start label: a newline at the start of a template name input: "{{\nfoobar}}" output: [TemplateOpen(), Text(text="\nfoobar"), TemplateClose()] --- -name: newline_end +name: newlines_end label: a newline at the end of a template name input: "{{foobar\n}}" output: [TemplateOpen(), Text(text="foobar\n"), TemplateClose()] --- -name: newline_start_end +name: newlines_start_end label: a newline at the start and end of a template name input: "{{\nfoobar\n}}" output: [TemplateOpen(), Text(text="\nfoobar\n"), TemplateClose()] --- -name: newline_mid +name: newlines_mid label: a newline at the middle of a template name input: "{{foo\nbar}}" output: [Text(text="{{foo\nbar}}")] --- -name: newline_start_mid +name: newlines_start_mid label: a newline at the start and middle of a template name input: "{{\nfoo\nbar}}" output: [Text(text="{{\nfoo\nbar}}")] --- -name: newline_mid_end +name: newlines_mid_end label: a newline at the middle and end of a template name input: "{{foo\nbar\n}}" output: [Text(text="{{foo\nbar\n}}")] --- -name: newline_start_mid_end +name: newlines_start_mid_end label: a newline at the start, middle, and end of a template name input: "{{\nfoo\nbar\n}}" output: [Text(text="{{\nfoo\nbar\n}}")] --- -name: newline_unnamed_param_start -label: a newline at the start of an unnamed template parameter -input: "{{foo|\nbar}}" -output: [TemplateOpen(), Text(text="foo"), TemplateParamSeparator(), Text(text="\nbar"), TemplateClose()] - ---- - -name: newline_unnamed_param_end -label: a newline at the end of an unnamed template parameter -input: "{{foo|bar\n}}" -output: [TemplateOpen(), Text(text="foo"), TemplateParamSeparator(), Text(text="bar\n"), TemplateClose()] - ---- - -name: newline_unnamed_param_start_end -label: a newline at the start and end of an unnamed template parameter -input: "{{foo|\nbar\n}}" -output: [TemplateOpen(), Text(text="foo"), TemplateParamSeparator(), Text(text="\nbar\n"), TemplateClose()] - ---- - -name: newline_unnamed_param_start_mid -label: a newline at the start and middle of an unnamed template parameter -input: "{{foo|\nb\nar}}" -output: [TemplateOpen(), Text(text="foo"), TemplateParamSeparator(), Text(text="\nb\nar"), TemplateClose()] - ---- - -name: newline_unnamed_param_mid_end -label: a newline at the middle and end of an unnamed template parameter -input: "{{foo|b\nar\n}}" -output: [TemplateOpen(), Text(text="foo"), TemplateParamSeparator(), Text(text="b\nar\n"), TemplateClose()] - ---- - -name: newline_unnamed_param_start_mid_end -label: a newline at the start, middle, and end of an unnamed template parameter +name: newlines_unnamed_param +label: newlines within an unnamed template parameter input: "{{foo|\nb\nar\n}}" output: [TemplateOpen(), Text(text="foo"), TemplateParamSeparator(), Text(text="\nb\nar\n"), TemplateClose()] --- -name: newline_start_unnamed_param_start -label: a newline at the start of a template name and at the start of an unnamed template parameter -input: "{{\nfoo|\nbar}}" -output: [TemplateOpen(), Text(text="\nfoo"), TemplateParamSeparator(), Text(text="\nbar"), TemplateClose()] - ---- - -name: newline_start_unnamed_param_end -label: a newline at the start of a template name and at the end of an unnamed template parameter -input: "{{\nfoo|bar\n}}" -output: [TemplateOpen(), Text(text="\nfoo"), TemplateParamSeparator(), Text(text="bar\n"), TemplateClose()] - ---- - -name: newline_start_unnamed_param_start_end -label: a newline at the start of a template name and at the start and end of an unnamed template parameter -input: "{{\nfoo|\nbar\n}}" -output: [TemplateOpen(), Text(text="\nfoo"), TemplateParamSeparator(), Text(text="\nbar\n"), TemplateClose()] - ---- - -name: newline_start_unnamed_param_start_mid -label: a newline at the start of a template name and at the start and middle of an unnamed template parameter -input: "{{\nfoo|\nb\nar}}" -output: [TemplateOpen(), Text(text="\nfoo"), TemplateParamSeparator(), Text(text="\nb\nar"), TemplateClose()] - ---- - -name: newline_start_unnamed_param_mid_end -label: a newline at the start of a template name and at the middle and end of an unnamed template parameter -input: "{{\nfoo|b\nar\n}}" -output: [TemplateOpen(), Text(text="\nfoo"), TemplateParamSeparator(), Text(text="b\nar\n"), TemplateClose()] - ---- - -name: newline_start_unnamed_param_start_mid_end -label: a newline at the start of a template name and at the start, middle, and end of an unnamed template parameter -input: "{{\nfoo|\nb\nar\n}}" -output: [TemplateOpen(), Text(text="\nfoo"), TemplateParamSeparator(), Text(text="\nb\nar\n"), TemplateClose()] - ---- - -name: newline_end_unnamed_param_start -label: a newline at the end of a template name and at the start of an unnamed template parameter -input: "{{foo\n|\nbar}}" -output: [TemplateOpen(), Text(text="foo\n"), TemplateParamSeparator(), Text(text="\nbar"), TemplateClose()] - ---- - -name: newline_end_unnamed_param_end -label: a newline at the end of a template name and at the end of an unnamed template parameter -input: "{{foo\n|bar\n}}" -output: [TemplateOpen(), Text(text="foo\n"), TemplateParamSeparator(), Text(text="bar\n"), TemplateClose()] - ---- - -name: newline_end_unnamed_param_start_end -label: a newline at the end of a template name and at the start and end of an unnamed template parameter -input: "{{foo\n|\nbar\n}}" -output: [TemplateOpen(), Text(text="foo\n"), TemplateParamSeparator(), Text(text="\nbar\n"), TemplateClose()] - ---- - -name: newline_end_unnamed_param_start_mid -label: a newline at the end of a template name and at the start and middle of an unnamed template parameter -input: "{{foo\n|\nb\nar}}" -output: [TemplateOpen(), Text(text="foo\n"), TemplateParamSeparator(), Text(text="\nb\nar"), TemplateClose()] - ---- - -name: newline_end_unnamed_param_mid_end -label: a newline at the end of a template name and at the middle and end of an unnamed template parameter -input: "{{foo\n|b\nar\n}}" -output: [TemplateOpen(), Text(text="foo\n"), TemplateParamSeparator(), Text(text="b\nar\n"), TemplateClose()] - ---- - -name: newline_end_unnamed_param_start_mid_end -label: a newline at the end of a template name and at the start, middle, and end of an unnamed template parameter -input: "{{foo\n|\nb\nar\n}}" -output: [TemplateOpen(), Text(text="foo\n"), TemplateParamSeparator(), Text(text="\nb\nar\n"), TemplateClose()] - ---- - -name: newline_start_end_unnamed_param_end -label: a newline at the start and end of a template name and the start of an unnamed template parameter -input: "{{\nfoo\n|\nbar}}" -output: [TemplateOpen(), Text(text="\nfoo\n"), TemplateParamSeparator(), Text(text="\nbar"), TemplateClose()] - ---- - -name: newline_start_end_unnamed_param_end -label: a newline at the start and end of a template name and the end of an unnamed template parameter -input: "{{\nfoo\n|bar\n}}" -output: [TemplateOpen(), Text(text="\nfoo\n"), TemplateParamSeparator(), Text(text="bar\n"), TemplateClose()] - ---- - -name: newline_start_end_unnamed_param_start_end -label: a newline at the start and end of a template name and the start and end of an unnamed template parameter -input: "{{\nfoo\n|\nbar\n}}" -output: [TemplateOpen(), Text(text="\nfoo\n"), TemplateParamSeparator(), Text(text="\nbar\n"), TemplateClose()] - ---- - -name: newline_start_end_unnamed_param_start_mid -label: a newline at the start and end of a template name and the start and middle of an unnamed template parameter -input: "{{\nfoo\n|\nb\nar}}" -output: [TemplateOpen(), Text(text="\nfoo\n"), TemplateParamSeparator(), Text(text="\nb\nar"), TemplateClose()] - ---- - -name: newline_start_end_unnamed_param_mid_end -label: a newline at the start and end of a template name and the middle and end of an unnamed template parameter -input: "{{\nfoo\n|b\nar\n}}" -output: [TemplateOpen(), Text(text="\nfoo\n"), TemplateParamSeparator(), Text(text="b\nar\n"), TemplateClose()] - ---- - -name: newline_start_end_unnamed_param_start_mid_end -label: a newline at the start and end of a template name and the start, middle, and end of an unnamed template parameter +name: newlines_enclose_template_name_unnamed_param +label: newlines enclosing a template name and within an unnamed template parameter input: "{{\nfoo\n|\nb\nar\n}}" output: [TemplateOpen(), Text(text="\nfoo\n"), TemplateParamSeparator(), Text(text="\nb\nar\n"), TemplateClose()] --- -name: newline_mid_unnamed_param_start -label: a newline at the middle of a template name and at the start of an unnamed template parameter -input: "{{f\noo|\nbar}}" -output: [Text(text="{{f\noo|\nbar}}")] - ---- - -name: newline_start_mid_unnamed_param_start -label: a newline at the start and middle of a template name and at the start of an unnamed template parameter -input: "{{\nf\noo|\nbar}}" -output: [Text(text="{{\nf\noo|\nbar}}")] - ---- - -name: newline_start_end_unnamed_param_start -label: a newline at the middle and of a template name and at the start of an unnamed template parameter -input: "{{f\noo\n|\nbar}}" -output: [Text(text="{{f\noo\n|\nbar}}")] - ---- - -name: newline_start_mid_end_unnamed_param_start -label: a newline at the start, middle, and end of a template name and at the start of an unnamed template parameter -input: "{{\nf\noo\n|\nbar}}" -output: [Text(text="{{\nf\noo\n|\nbar}}")] - ---- - -name: newline_named_param_value_start -label: a newline at the start of a named parameter value -input: "{{foo|1=\nbar}}" -output: [TemplateOpen(), Text(text="foo"), TemplateParamSeparator(), Text(text="1"), TemplateParamEquals(), Text(text="\nbar"), TemplateClose()] - ---- - -name: newline_named_param_value_end -label: a newline at the end of a named parameter value -input: "{{foo|1=bar\n}}" -output: [TemplateOpen(), Text(text="foo"), TemplateParamSeparator(), Text(text="1"), TemplateParamEquals(), Text(text="bar\n"), TemplateClose()] - ---- - -name: newline_named_param_value_start_end -label: a newline at the start and end of a named parameter value -input: "{{foo|1=\nbar\n}}" -output: [TemplateOpen(), Text(text="foo"), TemplateParamSeparator(), Text(text="1"), TemplateParamEquals(), Text(text="\nbar\n"), TemplateClose()] - ---- - -name: newline_named_param_value_start_mid -label: a newline at the start and middle of a named parameter value -input: "{{foo|1=\nb\nar}}" -output: [TemplateOpen(), Text(text="foo"), TemplateParamSeparator(), Text(text="1"), TemplateParamEquals(), Text(text="\nb\nar"), TemplateClose()] - ---- - -name: newline_named_param_value_mid_end -label: a newline at the middle and end of a named parameter value -input: "{{foo|1=b\nar\n}}" -output: [TemplateOpen(), Text(text="foo"), TemplateParamSeparator(), Text(text="1"), TemplateParamEquals(), Text(text="b\nar\n"), TemplateClose()] - ---- - -name: newline_named_param_value_start_mid_end -label: a newline at the start, middle, and end of a named parameter value -input: "{{foo|1=\nb\nar\n}}" -output: [TemplateOpen(), Text(text="foo"), TemplateParamSeparator(), Text(text="1"), TemplateParamEquals(), Text(text="\nb\nar\n"), TemplateClose()] - ---- - -name: newline_named_param_name_start -label: a newline at the start of a parameter name -input: "{{foo|\nbar=baz}}" -output: [TemplateOpen(), Text(text="foo"), TemplateParamSeparator(), Text(text="\nbar"), TemplateParamEquals(), Text(text="baz"), TemplateClose()] - ---- - -name: newline_named_param_name_end -label: a newline at the end of a parameter name -input: "{{foo|bar\n=baz}}" -output: [TemplateOpen(), Text(text="foo"), TemplateParamSeparator(), Text(text="bar\n"), TemplateParamEquals(), Text(text="baz"), TemplateClose()] - ---- - -name: newline_named_param_name_start_end -label: a newline at the start and end of a parameter name -input: "{{foo|\nbar\n=baz}}" -output: [TemplateOpen(), Text(text="foo"), TemplateParamSeparator(), Text(text="\nbar\n"), TemplateParamEquals(), Text(text="baz"), TemplateClose()] +name: newlines_within_template_name_unnamed_param +label: newlines within a template name and within an unnamed template parameter +input: "{{\nfo\no\n|\nb\nar\n}}" +output: [Text(text="{{\nfo\no\n|\nb\nar\n}}")] --- -name: newline_named_param_name_mid -label: a newline at the middle of a parameter name -input: "{{foo|b\nar=baz}}" -output: [Text(text="{{foo|b\nar=baz}}")] +name: newlines_enclose_template_name_named_param_value +label: newlines enclosing a template name and within a named parameter value +input: "{{\nfoo\n|1=\nb\nar\n}}" +output: [TemplateOpen(), Text(text="\nfoo\n"), TemplateParamSeparator(), Text(text="1"), TemplateParamEquals(), Text(text="\nb\nar\n"), TemplateClose()] --- -name: newline_named_param_name_start_mid -label: a newline at the start and middle of a parameter name -input: "{{foo|\nb\nar=baz}}" -output: [Text(text="{{foo|\nb\nar=baz}}")] +name: newlines_within_template_name_named_param_value +label: newlines within a template name and within a named parameter value +input: "{{\nf\noo\n|1=\nb\nar\n}}" +output: [Text(text="{{\nf\noo\n|1=\nb\nar\n}}")] --- -name: newline_named_param_name_mid_end -label: a newline at the middle and end of a parameter name -input: "{{foo|b\nar\n=baz}}" -output: [Text(text="{{foo|b\nar\n=baz}}")] - ---- - -name: newline_named_param_name_start_mid_end -label: a newline at the start, middle, and end of a parameter name +name: newlines_named_param_name +label: newlines within a parameter name input: "{{foo|\nb\nar\n=baz}}" -output: [Text(text="{{foo|\nb\nar\n=baz}}")] - ---- - -name: newline_named_param_name_start_param_value_end -label: a newline at the start of a parameter name and the end of a parameter value -input: "{{foo|\nbar=baz\n}}" -output: [TemplateOpen(), Text(text="foo"), TemplateParamSeparator(), Text(text="\nbar"), TemplateParamEquals(), Text(text="baz\n"), TemplateClose()] - ---- - -name: newline_named_param_name_end_param_value_end -label: a newline at the end of a parameter name and the end of a parameter value -input: "{{foo|bar\n=baz\n}}" -output: [TemplateOpen(), Text(text="foo"), TemplateParamSeparator(), Text(text="bar\n"), TemplateParamEquals(), Text(text="baz\n"), TemplateClose()] - ---- - -name: newline_named_param_name_start_end_param_value_end -label: a newline at the start and end of a parameter name and the end of a parameter value -input: "{{foo|\nbar\n=baz\n}}" -output: [TemplateOpen(), Text(text="foo"), TemplateParamSeparator(), Text(text="\nbar\n"), TemplateParamEquals(), Text(text="baz\n"), TemplateClose()] - ---- - -name: newline_named_param_name_start_mid_param_value_end -label: a newline at the start and middle of a parameter name and the end of a parameter value -input: "{{foo|\nb\nar=baz\n}}" -output: [Text(text="{{foo|\nb\nar=baz\n}}")] - ---- - -name: newline_named_param_name_mid_end_param_value_end -label: a newline at the middle and end of a parameter name and the end of a parameter value -input: "{{foo|b\nar\n=baz\n}}" -output: [Text(text="{{foo|b\nar\n=baz\n}}")] - ---- - -name: newline_named_param_name_start_mid_end_param_value_end -label: a newline at the start, middle, and end of a parameter name and at the end of a parameter value -input: "{{foo|\nb\nar\n=baz\n}}" -output: [Text(text="{{foo|\nb\nar\n=baz\n}}")] - ---- - -name: newline_named_param_name_start_param_value_start -label: a newline at the start of a parameter name and at the start of a parameter value -input: "{{foo|\nbar=\nbaz}}" -output: [TemplateOpen(), Text(text="foo"), TemplateParamSeparator(), Text(text="\nbar"), TemplateParamEquals(), Text(text="\nbaz"), TemplateClose()] - ---- - -name: newline_named_param_name_end_param_value_start -label: a newline at the end of a parameter name and at the start of a parameter value -input: "{{foo|bar\n=\nbaz}}" -output: [TemplateOpen(), Text(text="foo"), TemplateParamSeparator(), Text(text="bar\n"), TemplateParamEquals(), Text(text="\nbaz"), TemplateClose()] - ---- - -name: newline_named_param_name_start_end_param_value_start -label: a newline at the start and end of a parameter name and at the start of a parameter value -input: "{{foo|\nbar\n=\nbaz}}" -output: [TemplateOpen(), Text(text="foo"), TemplateParamSeparator(), Text(text="\nbar\n"), TemplateParamEquals(), Text(text="\nbaz"), TemplateClose()] - ---- - -name: newline_named_param_name_start_mid_param_value_start -label: a newline at the start and middle of a parameter name and at the start of a parameter value -input: "{{foo|\nb\nar=\nbaz}}" -output: [Text(text="{{foo|\nb\nar=\nbaz}}")] - ---- - -name: newline_named_param_name_mid_end_param_value_start -label: a newline at the middle and end of a parameter name and at the start of a parameter value -input: "{{foo|b\nar\n=\nbaz}}" -output: [Text(text="{{foo|b\nar\n=\nbaz}}")] +output: [TemplateOpen(), Text(text="foo"), TemplateParamSeparator(), Text(text="\nb\nar\n"), TemplateParamEquals(), Text(text="baz"), TemplateClose()] --- -name: newline_named_param_name_start_mid_end_param_value_start -label: a newline at the start, middle, and end of a parameter name and at the start of a parameter value -input: "{{foo|\nb\nar\n=\nbaz}}" -output: [Text(text="{{foo|\nb\nar\n=\nbaz}}")] +name: newlines_named_param_name_param_value +label: newlines within a parameter name and within a parameter value +input: "{{foo|\nb\nar\n=\nba\nz\n}}" +output: [TemplateOpen(), Text(text="foo"), TemplateParamSeparator(), Text(text="\nb\nar\n"), TemplateParamEquals(), Text(text="\nba\nz\n"), TemplateClose()] --- -name: newline_named_param_name_start_param_value_start_end -label: a newline at the start of a parameter name and at the start and end of a parameter value -input: "{{foo|\nbar=\nbaz\n}}" -output: [TemplateOpen(), Text(text="foo"), TemplateParamSeparator(), Text(text="\nbar"), TemplateParamEquals(), Text(text="\nbaz\n"), TemplateClose()] +name: newlines_enclose_template_name_named_param_name +label: newlines enclosing a template name and within a parameter name +input: "{{\nfoo\n|\nb\nar\n=baz}}" +output: [TemplateOpen(), Text(text="\nfoo\n"), TemplateParamSeparator(), Text(text="\nb\nar\n"), TemplateParamEquals(), Text(text="baz"), TemplateClose()] --- -name: newline_named_param_name_end_param_value_start_end -label: a newline at the end of a parameter name and at the start and end of a parameter value -input: "{{foo|bar\n=\nbaz\n}}" -output: [TemplateOpen(), Text(text="foo"), TemplateParamSeparator(), Text(text="bar\n"), TemplateParamEquals(), Text(text="\nbaz\n"), TemplateClose()] +name: newlines_enclose_template_name_named_param_name_param_value +label: newlines enclosing a template name and within a parameter name and within a parameter value +input: "{{\nfoo\n|\nb\nar\n=\nba\nz\n}}" +output: [TemplateOpen(), Text(text="\nfoo\n"), TemplateParamSeparator(), Text(text="\nb\nar\n"), TemplateParamEquals(), Text(text="\nba\nz\n"), TemplateClose()] --- -name: newline_named_param_name_start_end_param_value_start_end -label: a newline at the start and end of a parameter name and at the start and end of a parameter value -input: "{{foo|\nbar\n=\nbaz\n}}" -output: [TemplateOpen(), Text(text="foo"), TemplateParamSeparator(), Text(text="\nbar\n"), TemplateParamEquals(), Text(text="\nbaz\n"), TemplateClose()] +name: newlines_within_template_name_named_param_name +label: newlines within a template name and within a parameter name +input: "{{\nfo\no\n|\nb\nar\n=baz}}" +output: [Text(text="{{\nfo\no\n|\nb\nar\n=baz}}")] --- -name: newline_named_param_name_start_mid_param_value_start_end -label: a newline at the start and middle of a parameter name and at the start and end of a parameter value -input: "{{foo|\nb\nar=\nbaz\n}}" -output: [Text(text="{{foo|\nb\nar=\nbaz\n}}")] +name: newlines_within_template_name_named_param_name_param_value +label: newlines within a template name and within a parameter name and within a parameter value +input: "{{\nf\noo\n|\nb\nar\n=\nba\nz\n}}" +output: [Text(text="{{\nf\noo\n|\nb\nar\n=\nba\nz\n}}")] --- -name: newline_named_param_name_mid_end_param_value_start_end -label: a newline at the middle and end of a parameter name and at the start and end of a parameter value -input: "{{foo|b\nar\n=\nbaz\n}}" -output: [Text(text="{{foo|b\nar\n=\nbaz\n}}")] - ---- - -name: newline_named_param_name_start_mid_end_param_value_start_end -label: a newline at the start, middle, and end of a parameter name and at the start and end of a parameter value -input: "{{foo|\nb\nar\n=\nbaz\n}}" -output: [Text(text="{{foo|\nb\nar\n=\nbaz\n}}")] - ---- - -name: newline_named_param_name_start_param_value_mid -label: a newline at the start of a parameter name and at the middle of a parameter value -input: "{{foo|\nbar=ba\nz}}" -output: [TemplateOpen(), Text(text="foo"), TemplateParamSeparator(), Text(text="\nbar"), TemplateParamEquals(), Text(text="ba\nz"), TemplateClose()] - ---- - -name: newline_named_param_name_end_param_value_mid -label: a newline at the end of a parameter name and at the middle of a parameter value -input: "{{foo|bar\n=ba\nz}}" -output: [TemplateOpen(), Text(text="foo"), TemplateParamSeparator(), Text(text="bar\n"), TemplateParamEquals(), Text(text="ba\nz"), TemplateClose()] - ---- - -name: newline_named_param_name_start_end_param_value_mid -label: a newline at the start and end of a parameter name and at the middle of a parameter value -input: "{{foo|\nbar\n=ba\nz}}" -output: [TemplateOpen(), Text(text="foo"), TemplateParamSeparator(), Text(text="\nbar\n"), TemplateParamEquals(), Text(text="ba\nz"), TemplateClose()] - ---- - -name: newline_named_param_name_start_mid_param_value_mid -label: a newline at the start and middle of a parameter name and at the middle of a parameter value -input: "{{foo|\nb\nar=ba\nz}}" -output: [Text(text="{{foo|\nb\nar=ba\nz}}")] - ---- - -name: newline_named_param_name_mid_end_param_value_mid -label: a newline at the middle and end of a parameter name and at the middle of a parameter value -input: "{{foo|b\nar\n=ba\nz}}" -output: [Text(text="{{foo|b\nar\n=ba\nz}}")] - ---- - -name: newline_named_param_start_mid_end_param_value_mid -label: a newline at the start, middle, and end of a parameter name and at the middle of a parameter value -input: "{{foo|\nb\nar\n=ba\nz}}" -output: [Text(text="{{foo|\nb\nar\n=ba\nz}}")] +name: newlines_wildcard +label: a random, complex assortment of templates and newlines +input: "{{\nfoo\n|\nb\nar\n=\nb\naz\n|\nb\nuz\n}}" +output: [TemplateOpen(), Text(text="\nfoo\n"), TemplateParamSeparator(), Text(text="\nb\nar\n"), TemplateParamEquals(), Text(text="\nb\naz\n"), TemplateParamSeparator(), Text(text="\nb\nuz\n"), TemplateClose()] --- -name: newline_wildcard -label: a random, complex assortment of templates and newlines -input: "{{\nfoo\n|\nbar\n=\nb\naz\n|\nb\nuz\n}}" -output: [TemplateOpen(), Text(text="\nfoo\n"), TemplateParamSeparator(), Text(text="\nbar\n"), TemplateParamEquals(), Text(text="\nb\naz\n"), TemplateParamSeparator(), Text(text="\nb\nuz\n"), TemplateClose()] +name: newlines_wildcard_redux +label: an even more random and complex assortment of templates and newlines +input: "{{\nfoo\n|\n{{\nbar\n|\nb\naz\n=\nb\niz\n}}\n=\nb\nuzz\n}}" +output: [TemplateOpen(), Text(text="\nfoo\n"), TemplateParamSeparator(), Text(text="\n"), TemplateOpen(), Text(text="\nbar\n"), TemplateParamSeparator(), Text(text="\nb\naz\n"), TemplateParamEquals(), Text(text="\nb\niz\n"), TemplateClose(), Text(text="\n"), TemplateParamEquals(), Text(text="\nb\nuzz\n"), TemplateClose()] --- -name: newline_wildcard_redux -label: an even more random and complex assortment of templates and newlines -input: "{{\nfoo\n|\n{{\nbar\n|\nbaz\n=\nb\niz\n}}\n=\nb\nuzz\n}}" -output: [TemplateOpen(), Text(text="\nfoo\n"), TemplateParamSeparator(), Text(text="\n"), TemplateOpen(), Text(text="\nbar\n"), TemplateParamSeparator(), Text(text="\nbaz\n"), TemplateParamEquals(), Text(text="\nb\niz\n"), TemplateClose(), Text(text="\n"), TemplateParamEquals(), Text(text="\nb\nuzz\n"), TemplateClose()] +name: newlines_wildcard_redux_invalid +label: a variation of the newlines_wildcard_redux test that is invalid +input: "{{\nfoo\n|\n{{\nb\nar\n|\nb\naz\n=\nb\niz\n}}\n=\nb\nuzz\n}}" +output: [Text(text="{{\nfoo\n|\n{{\nb\nar\n|\nb\naz\n=\nb\niz\n}}\n=\nb\nuzz\n}}")] --- @@ -812,8 +441,43 @@ output: [TemplateOpen(), Text(text="foo"), TemplateParamSeparator(), Text(text=" name: valid_param_name_brackets label: valid characters in unnamed template parameter: left and right brackets -input: "{{foo|ba[r]}}" -output: [TemplateOpen(), Text(text="foo"), TemplateParamSeparator(), Text(text="ba[r]"), TemplateClose()] +input: "{{foo|ba[r]=baz}}" +output: [TemplateOpen(), Text(text="foo"), TemplateParamSeparator(), Text(text="ba[r]"), TemplateParamEquals(), Text(text="baz"), TemplateClose()] + +--- + +name: valid_param_name_double_left_brackets +label: valid characters in unnamed template parameter: double left brackets +input: "{{foo|bar[[in\nvalid=baz}}" +output: [TemplateOpen(), Text(text="foo"), TemplateParamSeparator(), Text(text="bar[[in\nvalid"), TemplateParamEquals(), Text(text="baz"), TemplateClose()] + +--- + +name: valid_param_name_double_right_brackets +label: valid characters in unnamed template parameter: double right brackets +input: "{{foo|bar]]=baz}}" +output: [TemplateOpen(), Text(text="foo"), TemplateParamSeparator(), Text(text="bar]]"), TemplateParamEquals(), Text(text="baz"), TemplateClose()] + +--- + +name: valid_param_name_double_brackets +label: valid characters in unnamed template parameter: double left and right brackets +input: "{{foo|bar[[in\nvalid]]=baz}}" +output: [TemplateOpen(), Text(text="foo"), TemplateParamSeparator(), Text(text="bar[[in\nvalid]]"), TemplateParamEquals(), Text(text="baz"), TemplateClose()] + +--- + +name: invalid_param_name_double_left_braces +label: invalid characters in template parameter name: double left braces +input: "{{foo|bar{{in\nvalid=baz}}" +output: [Text(text="{{foo|bar{{in\nvalid=baz}}")] + +--- + +name: invalid_param_name_double_braces +label: invalid characters in template parameter name: double left and right braces +input: "{{foo|bar{{in\nvalid}}=baz}}" +output: [TemplateOpen(), Text(text="foo"), TemplateParamSeparator(), Text(text="bar{{in\nvalid"), TemplateClose(), Text(text="=baz}}")] --- @@ -919,3 +583,17 @@ name: incomplete_two_named_params label: incomplete templates that should fail gracefully: two named parameters with values input: "{{stuff}} {{foo|bar=baz|biz=buzz" output: [TemplateOpen(), Text(text="stuff"), TemplateClose(), Text(text=" {{foo|bar=baz|biz=buzz")] + +--- + +name: incomplete_nested_template_as_unnamed_param +label: incomplete templates that should fail gracefully: a valid nested template as an unnamed parameter +input: "{{stuff}} {{foo|{{bar}}" +output: [TemplateOpen(), Text(text="stuff"), TemplateClose(), Text(text=" {{foo|"), TemplateOpen(), Text(text="bar"), TemplateClose()] + +--- + +name: incomplete_nested_template_as_param_value +label: incomplete templates that should fail gracefully: a valid nested template as a parameter value +input: "{{stuff}} {{foo|bar={{baz}}" +output: [TemplateOpen(), Text(text="stuff"), TemplateClose(), Text(text=" {{foo|bar="), TemplateOpen(), Text(text="baz"), TemplateClose()]