Change the way verify_safe() handles template params (#25).

- Newlines are now allowed in template param names.
- Changes also affect handling of arguments like {{{foo}}}.
- Update unit tests: remove some unnecessary ones, and add some to cover the changes.
- Update StringMixIn tests to actually work for some of the methods.
- Update copyright notices for the C extensions.

mwparserfromhell/parser/tokenizer.c

@@ -1,6 +1,6 @@
 /*
 Tokenizer for MWParserFromHell
-Copyright (C) 2012 Ben Kurtovic <ben.kurtovic@verizon.net>
+Copyright (C) 2012-2013 Ben Kurtovic <ben.kurtovic@verizon.net>
 
 Permission is hereby granted, free of charge, to any person obtaining a copy of
 this software and associated documentation files (the "Software"), to deal in

mwparserfromhell/parser/tokenizer.h

@@ -1,6 +1,6 @@
 /*
 Tokenizer Header File for MWParserFromHell
-Copyright (C) 2012 Ben Kurtovic <ben.kurtovic@verizon.net>
+Copyright (C) 2012-2013 Ben Kurtovic <ben.kurtovic@verizon.net>
 
 Permission is hereby granted, free of charge, to any person obtaining a copy of
 this software and associated documentation files (the "Software"), to deal in

mwparserfromhell/parser/tokenizer.py

@@ -396,34 +396,42 @@ class Tokenizer(object):
         """Make sure we are not trying to write an invalid character."""
         context = self._context
         if context & contexts.FAIL_NEXT:
-            self._fail_route()
+            return False
         if context & contexts.WIKILINK_TITLE:
             if this == "]" or this == "{":
                 self._context |= contexts.FAIL_NEXT
             elif this == "\n" or this == "[" or this == "}":
-                self._fail_route()
-            return
+                return False
+            return True
         if context & contexts.TEMPLATE_NAME:
             if this == "{" or this == "}" or this == "[":
                 self._context |= contexts.FAIL_NEXT
-                return
+                return True
             if this == "]":
-                self._fail_route()
-                return
+                return False
             if this == "|":
-                return
-        elif context & (contexts.TEMPLATE_PARAM_KEY | contexts.ARGUMENT_NAME):
+                return True
+            if context & contexts.HAS_TEXT:
+                if context & contexts.FAIL_ON_TEXT:
+                    if this is self.END or not this.isspace():
+                        return False
+                else:
+                    if this == "\n":
+                        self._context |= contexts.FAIL_ON_TEXT
+            elif this is not self.END or not this.isspace():
+                self._context |= contexts.HAS_TEXT
+            return True
+        else:
             if context & contexts.FAIL_ON_EQUALS:
                 if this == "=":
-                    self._fail_route()
-                    return
+                    return False
             elif context & contexts.FAIL_ON_LBRACE:
-                if this == "{":
+                if this == "{" or (self._read(-1) == self._read(-2) == "{"):
                     if context & contexts.TEMPLATE:
                         self._context |= contexts.FAIL_ON_EQUALS
                     else:
                         self._context |= contexts.FAIL_NEXT
-                    return
+                    return True
                 self._context ^= contexts.FAIL_ON_LBRACE
             elif context & contexts.FAIL_ON_RBRACE:
                 if this == "}":
@@ -431,40 +439,13 @@ class Tokenizer(object):
                         self._context |= contexts.FAIL_ON_EQUALS
                     else:
                         self._context |= contexts.FAIL_NEXT
-                    return
+                    return True
                 self._context ^= contexts.FAIL_ON_RBRACE
             elif this == "{":
                 self._context |= contexts.FAIL_ON_LBRACE
             elif this == "}":
                 self._context |= contexts.FAIL_ON_RBRACE
-        if context & contexts.HAS_TEXT:
-            if context & contexts.FAIL_ON_TEXT:
-                if this is self.END or not this.isspace():
-                    if context & contexts.TEMPLATE_PARAM_KEY:
-                        self._context ^= contexts.FAIL_ON_TEXT
-                        self._context |= contexts.FAIL_ON_EQUALS
-                    else:
-                        self._fail_route()
-                    return
-            else:
-                if this == "\n":
-                    self._context |= contexts.FAIL_ON_TEXT
-        elif this is self.END or not this.isspace():
-            self._context |= contexts.HAS_TEXT
-
-    def _reset_safety_checks(self):
-        """Unset any safety-checking contexts set by Tokenizer_verify_safe().
-
-        Used when we preserve a context but previous data becomes invalid, like
-        when moving between template parameters.
-        """
-        context = self._context
-        checks = (contexts.HAS_TEXT, contexts.FAIL_ON_TEXT, contexts.FAIL_NEXT,
-                  contexts.FAIL_ON_LBRACE, contexts.FAIL_ON_RBRACE,
-                  contexts.FAIL_ON_EQUALS)
-        for check in checks:
-            if context & check:
-                self._context ^= check;
+            return True
 
     def _parse(self, context=0):
         """Parse the wikicode string, using *context* for when to stop."""
@@ -474,12 +455,10 @@ class Tokenizer(object):
             unsafe = (contexts.TEMPLATE_NAME | contexts.WIKILINK_TITLE |
                       contexts.TEMPLATE_PARAM_KEY | contexts.ARGUMENT_NAME)
             if self._context & unsafe:
-                try:
-                    self._verify_safe(this)
-                except BadRoute:
+                if not self._verify_safe(this):
                     if self._context & contexts.TEMPLATE_PARAM_KEY:
                         self._pop()
-                    raise
+                    self._fail_route()
             if this not in self.MARKERS:
                 self._write_text(this)
                 self._head += 1
@@ -504,7 +483,6 @@ class Tokenizer(object):
                 if self._context & contexts.FAIL_NEXT:
                     self._context ^= contexts.FAIL_NEXT
             elif this == "|" and self._context & contexts.TEMPLATE:
-                self._reset_safety_checks()
                 self._handle_template_param()
             elif this == "=" and self._context & contexts.TEMPLATE_PARAM_KEY:
                 self._handle_template_param_value()

tests/test_string_mixin.py

@@ -42,12 +42,14 @@ class TestStringMixIn(unittest.TestCase):
         methods = [
             "capitalize", "center", "count", "encode", "endswith",
             "expandtabs", "find", "format", "index", "isalnum", "isalpha",
-            "isdecimal", "isdigit", "islower", "isnumeric", "isspace",
-            "istitle", "isupper", "join", "ljust", "lstrip", "partition",
-            "replace", "rfind", "rindex", "rjust", "rpartition", "rsplit",
-            "rstrip", "split", "splitlines", "startswith", "strip", "swapcase",
-            "title", "translate", "upper", "zfill"]
-        if not py3k:
+            "isdigit", "islower", "isnumeric", "isspace", "istitle", "isupper",
+            "join", "ljust", "lstrip", "partition", "replace", "rfind",
+            "rindex", "rjust", "rpartition", "rsplit", "rstrip", "split",
+            "splitlines", "startswith", "strip", "swapcase", "title",
+            "translate", "upper", "zfill"]
+        if py3k:
+            methods.append("isdecimal")
+        else:
             methods.append("decode")
         for meth in methods:
             expected = getattr(str, meth).__doc__
@@ -157,75 +159,107 @@ class TestStringMixIn(unittest.TestCase):
 
     def test_other_methods(self):
         """test the remaining non-magic methods of StringMixIn"""
-        fstr = _FakeString("fake string")
-
-        self.assertEquals("Fake string", fstr.capitalize())
+        str1 = _FakeString("fake string")
+        self.assertEquals("Fake string", str1.capitalize())
 
-        self.assertEquals("  fake string  ", fstr.center(15))
-        self.assertEquals("  fake string   ", fstr.center(16))
-        self.assertEquals("qqfake stringqq", fstr.center(15, "q"))
+        self.assertEquals("  fake string  ", str1.center(15))
+        self.assertEquals("  fake string   ", str1.center(16))
+        self.assertEquals("qqfake stringqq", str1.center(15, "q"))
 
-        self.assertEquals(1, fstr.count("e"))
-        self.assertEquals(0, fstr.count("z"))
-        self.assertEquals(1, fstr.count("r", 7))
-        self.assertEquals(0, fstr.count("r", 8))
-        self.assertEquals(1, fstr.count("r", 5, 9))
-        self.assertEquals(0, fstr.count("r", 5, 7))
+        self.assertEquals(1, str1.count("e"))
+        self.assertEquals(0, str1.count("z"))
+        self.assertEquals(1, str1.count("r", 7))
+        self.assertEquals(0, str1.count("r", 8))
+        self.assertEquals(1, str1.count("r", 5, 9))
+        self.assertEquals(0, str1.count("r", 5, 7))
 
         if not py3k:
-            self.assertEquals(fstr, fstr.decode())
-            actual = '\\U00010332\\U0001033f\\U00010344'
+            str2 = _FakeString("fo")
+            self.assertEquals(str1, str1.decode())
+            actual = _FakeString("\\U00010332\\U0001033f\\U00010344")
             self.assertEquals("𐌲𐌿𐍄", actual.decode("unicode_escape"))
-            self.assertEquals("𐌲", '\\U00010332'.decode("unicode_escape"))
-            self.assertRaises(UnicodeError, "fo".decode, "punycode")
-            self.assertEquals("", "fo".decode("punycode", "ignore"))
+            self.assertRaises(UnicodeError, str2.decode, "punycode")
+            self.assertEquals("", str2.decode("punycode", "ignore"))
 
-        self.assertEquals(b"fake string", fstr.encode())
+        str3 = _FakeString("𐌲𐌿𐍄")
+        self.assertEquals(b"fake string", str1.encode())
         self.assertEquals(b"\xF0\x90\x8C\xB2\xF0\x90\x8C\xBF\xF0\x90\x8D\x84",
-                          "𐌲𐌿𐍄".encode("utf8"))
-        self.assertRaises(UnicodeEncodeError, "𐌲𐌿𐍄".encode)
-        self.assertRaises(UnicodeEncodeError, "𐌲𐌿𐍄".encode, "ascii")
-        self.assertRaises(UnicodeEncodeError, "𐌲𐌿𐍄".encode, "ascii", "strict")
-        self.assertEquals("", "𐌲𐌿𐍄".encode("ascii", "ignore"))
-
-        self.assertTrue(fstr.endswith("ing"))
-        self.assertFalse(fstr.endswith("ingh"))
-
-        self.assertEquals("fake string", fstr)
-        self.assertEquals("        foobar", "\tfoobar".expandtabs())
-        self.assertEquals("    foobar", "\tfoobar".expandtabs(4))
-
-        self.assertEquals(3, fstr.find("e"))
-        self.assertEquals(-1, fstr.find("z"))
-        self.assertEquals(7, fstr.find("r", 7))
-        self.assertEquals(-1, fstr.find("r", 8))
-        self.assertEquals(7, fstr.find("r", 5, 9))
-        self.assertEquals(-1, fstr.find("r", 5, 7))
-
-        self.assertEquals("fake string", fstr.format())
-        self.assertEquals("foobarbaz", "foo{0}baz".format("bar"))
-        self.assertEquals("foobarbaz", "foo{abc}baz".format(abc="bar"))
-        self.assertEquals("foobarbazbuzz",
-                          "foo{0}{abc}buzz".format("bar", abc="baz"))
-        self.assertRaises(IndexError, "{0}{1}".format, "abc")
-
-        self.assertEquals(3, fstr.index("e"))
-        self.assertRaises(ValueError, fstr.index, "z")
-        self.assertEquals(7, fstr.index("r", 7))
-        self.assertRaises(ValueError, fstr.index, "r", 8)
-        self.assertEquals(7, fstr.index("r", 5, 9))
-        self.assertRaises(ValueError, fstr.index, "r", 5, 7)
-
-        self.assertTrue("foobar".isalnum())
-        self.assertTrue("foobar123".isalnum())
-        self.assertFalse("foo bar".isalnum())
-
-        self.assertTrue("foobar".isalpha())
-        self.assertFalse("foobar123".isalpha())
-        self.assertFalse("foo bar".isalpha())
+                          str3.encode("utf8"))
+        self.assertRaises(UnicodeEncodeError, str3.encode)
+        self.assertRaises(UnicodeEncodeError, str3.encode, "ascii")
+        self.assertRaises(UnicodeEncodeError, str3.encode, "ascii", "strict")
+        self.assertEquals("", str3.encode("ascii", "ignore"))
+
+        self.assertTrue(str1.endswith("ing"))
+        self.assertFalse(str1.endswith("ingh"))
+
+        str4 = _FakeString("\tfoobar")
+        self.assertEquals("fake string", str1)
+        self.assertEquals("        foobar", str4.expandtabs())
+        self.assertEquals("    foobar", str4.expandtabs(4))
+
+        self.assertEquals(3, str1.find("e"))
+        self.assertEquals(-1, str1.find("z"))
+        self.assertEquals(7, str1.find("r", 7))
+        self.assertEquals(-1, str1.find("r", 8))
+        self.assertEquals(7, str1.find("r", 5, 9))
+        self.assertEquals(-1, str1.find("r", 5, 7))
+
+        str5 = _FakeString("foo{0}baz")
+        str6 = _FakeString("foo{abc}baz")
+        str7 = _FakeString("foo{0}{abc}buzz")
+        str8 = _FakeString("{0}{1}")
+        self.assertEquals("fake string", str1.format())
+        self.assertEquals("foobarbaz", str5.format("bar"))
+        self.assertEquals("foobarbaz", str6.format(abc="bar"))
+        self.assertEquals("foobarbazbuzz", str7.format("bar", abc="baz"))
+        self.assertRaises(IndexError, str8.format, "abc")
+
+        self.assertEquals(3, str1.index("e"))
+        self.assertRaises(ValueError, str1.index, "z")
+        self.assertEquals(7, str1.index("r", 7))
+        self.assertRaises(ValueError, str1.index, "r", 8)
+        self.assertEquals(7, str1.index("r", 5, 9))
+        self.assertRaises(ValueError, str1.index, "r", 5, 7)
+
+        str9 = _FakeString("foobar")
+        str10 = _FakeString("foobar123")
+        str11 = _FakeString("foo bar")
+        self.assertTrue(str9.isalnum())
+        self.assertTrue(str10.isalnum())
+        self.assertFalse(str11.isalnum())
+
+        self.assertTrue(str9.isalpha())
+        self.assertFalse(str10.isalpha())
+        self.assertFalse(str11.isalpha())
+
+        str12 = _FakeString("123")
+        str13 = _FakeString("\u2155")
+        str14 = _FakeString("\u00B2")
+        if py3k:
+            self.assertFalse(str9.isdecimal())
+            self.assertTrue(str12.isdecimal())
+            self.assertFalse(str13.isdecimal())
+            self.assertFalse(str14.isdecimal())
+
+        self.assertFalse(str9.isdigit())
+        self.assertTrue(str12.isdigit())
+        self.assertFalse(str13.isdigit())
+        self.assertTrue(str14.isdigit())
+
+        str15 = _FakeString("")
+        str16 = _FakeString("FooBar")
+        self.assertTrue(str9.islower())
+        self.assertFalse(str15.islower())
+        self.assertFalse(str16.islower())
+
+        self.assertFalse(str9.isnumeric())
+        self.assertTrue(str12.isnumeric())
+        self.assertTrue(str13.isnumeric())
+        self.assertTrue(str14.isnumeric())
 
         methods = [
-            "isdecimal", "isdigit", "islower", "isnumeric", "isspace",
+            "isspace",
             "istitle", "isupper", "join", "ljust", "lstrip", "partition",
             "replace", "rfind", "rindex", "rjust", "rpartition", "rsplit",
             "rstrip", "split", "splitlines", "startswith", "strip", "swapcase",

tests/tokenizer/templates.mwtest

@@ -215,521 +215,150 @@ output: [TemplateOpen(), TemplateOpen(), TemplateOpen(), TemplateOpen(), Text(te
 
 ---
 
-name:   newline_start
+name:   newlines_start
 label:  a newline at the start of a template name
 input:  "{{\nfoobar}}"
 output: [TemplateOpen(), Text(text="\nfoobar"), TemplateClose()]
 
 ---
 
-name:   newline_end
+name:   newlines_end
 label:  a newline at the end of a template name
 input:  "{{foobar\n}}"
 output: [TemplateOpen(), Text(text="foobar\n"), TemplateClose()]
 
 ---
 
-name:   newline_start_end
+name:   newlines_start_end
 label:  a newline at the start and end of a template name
 input:  "{{\nfoobar\n}}"
 output: [TemplateOpen(), Text(text="\nfoobar\n"), TemplateClose()]
 
 ---
 
-name:   newline_mid
+name:   newlines_mid
 label:  a newline at the middle of a template name
 input:  "{{foo\nbar}}"
 output: [Text(text="{{foo\nbar}}")]
 
 ---
 
-name:   newline_start_mid
+name:   newlines_start_mid
 label:  a newline at the start and middle of a template name
 input:  "{{\nfoo\nbar}}"
 output: [Text(text="{{\nfoo\nbar}}")]
 
 ---
 
-name:   newline_mid_end
+name:   newlines_mid_end
 label:  a newline at the middle and end of a template name
 input:  "{{foo\nbar\n}}"
 output: [Text(text="{{foo\nbar\n}}")]
 
 ---
 
-name:   newline_start_mid_end
+name:   newlines_start_mid_end
 label:  a newline at the start, middle, and end of a template name
 input:  "{{\nfoo\nbar\n}}"
 output: [Text(text="{{\nfoo\nbar\n}}")]
 
 ---
 
-name:   newline_unnamed_param_start
-label:  a newline at the start of an unnamed template parameter
-input:  "{{foo|\nbar}}"
-output: [TemplateOpen(), Text(text="foo"), TemplateParamSeparator(), Text(text="\nbar"), TemplateClose()]
-
-
-name:   newline_unnamed_param_end
-label:  a newline at the end of an unnamed template parameter
-input:  "{{foo|bar\n}}"
-output: [TemplateOpen(), Text(text="foo"), TemplateParamSeparator(), Text(text="bar\n"), TemplateClose()]
-
-
-name:   newline_unnamed_param_start_end
-label:  a newline at the start and end of an unnamed template parameter
-input:  "{{foo|\nbar\n}}"
-output: [TemplateOpen(), Text(text="foo"), TemplateParamSeparator(), Text(text="\nbar\n"), TemplateClose()]
-
-
-name:   newline_unnamed_param_start_mid
-label:  a newline at the start and middle of an unnamed template parameter
-input:  "{{foo|\nb\nar}}"
-output: [TemplateOpen(), Text(text="foo"), TemplateParamSeparator(), Text(text="\nb\nar"), TemplateClose()]
-
-
-name:   newline_unnamed_param_mid_end
-label:  a newline at the middle and end of an unnamed template parameter
-input:  "{{foo|b\nar\n}}"
-output: [TemplateOpen(), Text(text="foo"), TemplateParamSeparator(), Text(text="b\nar\n"), TemplateClose()]
-
-
-name:   newline_unnamed_param_start_mid_end
-label:  a newline at the start, middle, and end of an unnamed template parameter
+name:   newlines_unnamed_param
+label:  newlines within an unnamed template parameter
 input:  "{{foo|\nb\nar\n}}"
 output: [TemplateOpen(), Text(text="foo"), TemplateParamSeparator(), Text(text="\nb\nar\n"), TemplateClose()]
 
 ---
 
-name:   newline_start_unnamed_param_start
-label:  a newline at the start of a template name and at the start of an unnamed template parameter
-input:  "{{\nfoo|\nbar}}"
-output: [TemplateOpen(), Text(text="\nfoo"), TemplateParamSeparator(), Text(text="\nbar"), TemplateClose()]
-
-
-name:   newline_start_unnamed_param_end
-label:  a newline at the start of a template name and at the end of an unnamed template parameter
-input:  "{{\nfoo|bar\n}}"
-output: [TemplateOpen(), Text(text="\nfoo"), TemplateParamSeparator(), Text(text="bar\n"), TemplateClose()]
-
-
-name:   newline_start_unnamed_param_start_end
-label:  a newline at the start of a template name and at the start and end of an unnamed template parameter
-input:  "{{\nfoo|\nbar\n}}"
-output: [TemplateOpen(), Text(text="\nfoo"), TemplateParamSeparator(), Text(text="\nbar\n"), TemplateClose()]
-
-
-name:   newline_start_unnamed_param_start_mid
-label:  a newline at the start of a template name and at the start and middle of an unnamed template parameter
-input:  "{{\nfoo|\nb\nar}}"
-output: [TemplateOpen(), Text(text="\nfoo"), TemplateParamSeparator(), Text(text="\nb\nar"), TemplateClose()]
-
-
-name:   newline_start_unnamed_param_mid_end
-label:  a newline at the start of a template name and at the middle and end of an unnamed template parameter
-input:  "{{\nfoo|b\nar\n}}"
-output: [TemplateOpen(), Text(text="\nfoo"), TemplateParamSeparator(), Text(text="b\nar\n"), TemplateClose()]
-
-
-name:   newline_start_unnamed_param_start_mid_end
-label:  a newline at the start of a template name and at the start, middle, and end of an unnamed template parameter
-input:  "{{\nfoo|\nb\nar\n}}"
-output: [TemplateOpen(), Text(text="\nfoo"), TemplateParamSeparator(), Text(text="\nb\nar\n"), TemplateClose()]
-
-
-name:   newline_end_unnamed_param_start
-label:  a newline at the end of a template name and at the start of an unnamed template parameter
-input:  "{{foo\n|\nbar}}"
-output: [TemplateOpen(), Text(text="foo\n"), TemplateParamSeparator(), Text(text="\nbar"), TemplateClose()]
-
-
-name:   newline_end_unnamed_param_end
-label:  a newline at the end of a template name and at the end of an unnamed template parameter
-input:  "{{foo\n|bar\n}}"
-output: [TemplateOpen(), Text(text="foo\n"), TemplateParamSeparator(), Text(text="bar\n"), TemplateClose()]
-
-
-name:   newline_end_unnamed_param_start_end
-label:  a newline at the end of a template name and at the start and end of an unnamed template parameter
-input:  "{{foo\n|\nbar\n}}"
-output: [TemplateOpen(), Text(text="foo\n"), TemplateParamSeparator(), Text(text="\nbar\n"), TemplateClose()]
-
-
-name:   newline_end_unnamed_param_start_mid
-label:  a newline at the end of a template name and at the start and middle of an unnamed template parameter
-input:  "{{foo\n|\nb\nar}}"
-output: [TemplateOpen(), Text(text="foo\n"), TemplateParamSeparator(), Text(text="\nb\nar"), TemplateClose()]
-
-
-name:   newline_end_unnamed_param_mid_end
-label:  a newline at the end of a template name and at the middle and end of an unnamed template parameter
-input:  "{{foo\n|b\nar\n}}"
-output: [TemplateOpen(), Text(text="foo\n"), TemplateParamSeparator(), Text(text="b\nar\n"), TemplateClose()]
-
-
-name:   newline_end_unnamed_param_start_mid_end
-label:  a newline at the end of a template name and at the start, middle, and end of an unnamed template parameter
-input:  "{{foo\n|\nb\nar\n}}"
-output: [TemplateOpen(), Text(text="foo\n"), TemplateParamSeparator(), Text(text="\nb\nar\n"), TemplateClose()]
-
-
-name:   newline_start_end_unnamed_param_end
-label:  a newline at the start and end of a template name and the start of an unnamed template parameter
-input:  "{{\nfoo\n|\nbar}}"
-output: [TemplateOpen(), Text(text="\nfoo\n"), TemplateParamSeparator(), Text(text="\nbar"), TemplateClose()]
-
-
-name:   newline_start_end_unnamed_param_end
-label:  a newline at the start and end of a template name and the end of an unnamed template parameter
-input:  "{{\nfoo\n|bar\n}}"
-output: [TemplateOpen(), Text(text="\nfoo\n"), TemplateParamSeparator(), Text(text="bar\n"), TemplateClose()]
-
-
-name:   newline_start_end_unnamed_param_start_end
-label:  a newline at the start and end of a template name and the start and end of an unnamed template parameter
-input:  "{{\nfoo\n|\nbar\n}}"
-output: [TemplateOpen(), Text(text="\nfoo\n"), TemplateParamSeparator(), Text(text="\nbar\n"), TemplateClose()]
-
-
-name:   newline_start_end_unnamed_param_start_mid
-label:  a newline at the start and end of a template name and the start and middle of an unnamed template parameter
-input:  "{{\nfoo\n|\nb\nar}}"
-output: [TemplateOpen(), Text(text="\nfoo\n"), TemplateParamSeparator(), Text(text="\nb\nar"), TemplateClose()]
-
-
-name:   newline_start_end_unnamed_param_mid_end
-label:  a newline at the start and end of a template name and the middle and end of an unnamed template parameter
-input:  "{{\nfoo\n|b\nar\n}}"
-output: [TemplateOpen(), Text(text="\nfoo\n"), TemplateParamSeparator(), Text(text="b\nar\n"), TemplateClose()]
-
-
-name:   newline_start_end_unnamed_param_start_mid_end
-label:  a newline at the start and end of a template name and the start, middle, and end of an unnamed template parameter
+name:   newlines_enclose_template_name_unnamed_param
+label:  newlines enclosing a template name and within an unnamed template parameter
 input:  "{{\nfoo\n|\nb\nar\n}}"
 output: [TemplateOpen(), Text(text="\nfoo\n"), TemplateParamSeparator(), Text(text="\nb\nar\n"), TemplateClose()]
 
 ---
 
-name:   newline_mid_unnamed_param_start
-label:  a newline at the middle of a template name and at the start of an unnamed template parameter
-input:  "{{f\noo|\nbar}}"
-output: [Text(text="{{f\noo|\nbar}}")]
-
-
-name:   newline_start_mid_unnamed_param_start
-label:  a newline at the start and middle of a template name and at the start of an unnamed template parameter
-input:  "{{\nf\noo|\nbar}}"
-output: [Text(text="{{\nf\noo|\nbar}}")]
-
-
-name:   newline_start_end_unnamed_param_start
-label:  a newline at the middle and of a template name and at the start of an unnamed template parameter
-input:  "{{f\noo\n|\nbar}}"
-output: [Text(text="{{f\noo\n|\nbar}}")]
-
-
-name:   newline_start_mid_end_unnamed_param_start
-label:  a newline at the start, middle, and end of a template name and at the start of an unnamed template parameter
-input:  "{{\nf\noo\n|\nbar}}"
-output: [Text(text="{{\nf\noo\n|\nbar}}")]
-
-
-name:   newline_named_param_value_start
-label:  a newline at the start of a named parameter value
-input:  "{{foo|1=\nbar}}"
-output: [TemplateOpen(), Text(text="foo"), TemplateParamSeparator(), Text(text="1"), TemplateParamEquals(), Text(text="\nbar"), TemplateClose()]
-
-
-name:   newline_named_param_value_end
-label:  a newline at the end of a named parameter value
-input:  "{{foo|1=bar\n}}"
-output: [TemplateOpen(), Text(text="foo"), TemplateParamSeparator(), Text(text="1"), TemplateParamEquals(), Text(text="bar\n"), TemplateClose()]
-
-
-name:   newline_named_param_value_start_end
-label:  a newline at the start and end of a named parameter value
-input:  "{{foo|1=\nbar\n}}"
-output: [TemplateOpen(), Text(text="foo"), TemplateParamSeparator(), Text(text="1"), TemplateParamEquals(), Text(text="\nbar\n"), TemplateClose()]
-
-
-name:   newline_named_param_value_start_mid
-label:  a newline at the start and middle of a named parameter value
-input:  "{{foo|1=\nb\nar}}"
-output: [TemplateOpen(), Text(text="foo"), TemplateParamSeparator(), Text(text="1"), TemplateParamEquals(), Text(text="\nb\nar"), TemplateClose()]
-
-
-name:   newline_named_param_value_mid_end
-label:  a newline at the middle and end of a named parameter value
-input:  "{{foo|1=b\nar\n}}"
-output: [TemplateOpen(), Text(text="foo"), TemplateParamSeparator(), Text(text="1"), TemplateParamEquals(), Text(text="b\nar\n"), TemplateClose()]
-
-
-name:   newline_named_param_value_start_mid_end
-label:  a newline at the start, middle, and end of a named parameter value
-input:  "{{foo|1=\nb\nar\n}}"
-output: [TemplateOpen(), Text(text="foo"), TemplateParamSeparator(), Text(text="1"), TemplateParamEquals(), Text(text="\nb\nar\n"), TemplateClose()]
-
-
-name:   newline_named_param_name_start
-label:  a newline at the start of a parameter name
-input:  "{{foo|\nbar=baz}}"
-output: [TemplateOpen(), Text(text="foo"), TemplateParamSeparator(), Text(text="\nbar"), TemplateParamEquals(), Text(text="baz"), TemplateClose()]
-
-
-name:   newline_named_param_name_end
-label:  a newline at the end of a parameter name
-input:  "{{foo|bar\n=baz}}"
-output: [TemplateOpen(), Text(text="foo"), TemplateParamSeparator(), Text(text="bar\n"), TemplateParamEquals(), Text(text="baz"), TemplateClose()]
-
-
-name:   newline_named_param_name_start_end
-label:  a newline at the start and end of a parameter name
-input:  "{{foo|\nbar\n=baz}}"
-output: [TemplateOpen(), Text(text="foo"), TemplateParamSeparator(), Text(text="\nbar\n"), TemplateParamEquals(), Text(text="baz"), TemplateClose()]
+name:   newlines_within_template_name_unnamed_param
+label:  newlines within a template name and within an unnamed template parameter
+input:  "{{\nfo\no\n|\nb\nar\n}}"
+output: [Text(text="{{\nfo\no\n|\nb\nar\n}}")]
 
 ---
 
-name:   newline_named_param_name_mid
-label:  a newline at the middle of a parameter name
-input:  "{{foo|b\nar=baz}}"
-output: [Text(text="{{foo|b\nar=baz}}")]
+name:   newlines_enclose_template_name_named_param_value
+label:  newlines enclosing a template name and within a named parameter value
+input:  "{{\nfoo\n|1=\nb\nar\n}}"
+output: [TemplateOpen(), Text(text="\nfoo\n"), TemplateParamSeparator(), Text(text="1"), TemplateParamEquals(), Text(text="\nb\nar\n"), TemplateClose()]
 
 ---
 
-name:   newline_named_param_name_start_mid
-label:  a newline at the start and middle of a parameter name
-input:  "{{foo|\nb\nar=baz}}"
-output: [Text(text="{{foo|\nb\nar=baz}}")]
+name:   newlines_within_template_name_named_param_value
+label:  newlines within a template name and within a named parameter value
+input:  "{{\nf\noo\n|1=\nb\nar\n}}"
+output: [Text(text="{{\nf\noo\n|1=\nb\nar\n}}")]
 
 ---
 
-name:   newline_named_param_name_mid_end
-label:  a newline at the middle and end of a parameter name
-input:  "{{foo|b\nar\n=baz}}"
-output: [Text(text="{{foo|b\nar\n=baz}}")]
-
-
-name:   newline_named_param_name_start_mid_end
-label:  a newline at the start, middle, and end of a parameter name
+name:   newlines_named_param_name
+label:  newlines within a parameter name
 input:  "{{foo|\nb\nar\n=baz}}"
-output: [Text(text="{{foo|\nb\nar\n=baz}}")]
-
-
-name:   newline_named_param_name_start_param_value_end
-label:  a newline at the start of a parameter name and the end of a parameter value
-input:  "{{foo|\nbar=baz\n}}"
-output: [TemplateOpen(), Text(text="foo"), TemplateParamSeparator(), Text(text="\nbar"), TemplateParamEquals(), Text(text="baz\n"), TemplateClose()]
-
-
-name:   newline_named_param_name_end_param_value_end
-label:  a newline at the end of a parameter name and the end of a parameter value
-input:  "{{foo|bar\n=baz\n}}"
-output: [TemplateOpen(), Text(text="foo"), TemplateParamSeparator(), Text(text="bar\n"), TemplateParamEquals(), Text(text="baz\n"), TemplateClose()]
-
-
-name:   newline_named_param_name_start_end_param_value_end
-label:  a newline at the start and end of a parameter name and the end of a parameter value
-input:  "{{foo|\nbar\n=baz\n}}"
-output: [TemplateOpen(), Text(text="foo"), TemplateParamSeparator(), Text(text="\nbar\n"), TemplateParamEquals(), Text(text="baz\n"), TemplateClose()]
-
-
-name:   newline_named_param_name_start_mid_param_value_end
-label:  a newline at the start and middle of a parameter name and the end of a parameter value
-input:  "{{foo|\nb\nar=baz\n}}"
-output: [Text(text="{{foo|\nb\nar=baz\n}}")]
-
-
-name:   newline_named_param_name_mid_end_param_value_end
-label:  a newline at the middle and end of a parameter name and the end of a parameter value
-input:  "{{foo|b\nar\n=baz\n}}"
-output: [Text(text="{{foo|b\nar\n=baz\n}}")]
-
-
-name:   newline_named_param_name_start_mid_end_param_value_end
-label:  a newline at the start, middle, and end of a parameter name and at the end of a parameter value
-input:  "{{foo|\nb\nar\n=baz\n}}"
-output: [Text(text="{{foo|\nb\nar\n=baz\n}}")]
-
-
-name:   newline_named_param_name_start_param_value_start
-label:  a newline at the start of a parameter name and at the start of a parameter value
-input:  "{{foo|\nbar=\nbaz}}"
-output: [TemplateOpen(), Text(text="foo"), TemplateParamSeparator(), Text(text="\nbar"), TemplateParamEquals(), Text(text="\nbaz"), TemplateClose()]
-
-
-name:   newline_named_param_name_end_param_value_start
-label:  a newline at the end of a parameter name and at the start of a parameter value
-input:  "{{foo|bar\n=\nbaz}}"
-output: [TemplateOpen(), Text(text="foo"), TemplateParamSeparator(), Text(text="bar\n"), TemplateParamEquals(), Text(text="\nbaz"), TemplateClose()]
-
-
-name:   newline_named_param_name_start_end_param_value_start
-label:  a newline at the start and end of a parameter name and at the start of a parameter value
-input:  "{{foo|\nbar\n=\nbaz}}"
-output: [TemplateOpen(), Text(text="foo"), TemplateParamSeparator(), Text(text="\nbar\n"), TemplateParamEquals(), Text(text="\nbaz"), TemplateClose()]
-
-
-name:   newline_named_param_name_start_mid_param_value_start
-label:  a newline at the start and middle of a parameter name and at the start of a parameter value
-input:  "{{foo|\nb\nar=\nbaz}}"
-output: [Text(text="{{foo|\nb\nar=\nbaz}}")]
-
-
-name:   newline_named_param_name_mid_end_param_value_start
-label:  a newline at the middle and end of a parameter name and at the start of a parameter value
-input:  "{{foo|b\nar\n=\nbaz}}"
-output: [Text(text="{{foo|b\nar\n=\nbaz}}")]
+output: [TemplateOpen(), Text(text="foo"), TemplateParamSeparator(), Text(text="\nb\nar\n"), TemplateParamEquals(), Text(text="baz"), TemplateClose()]
 
 ---
 
-name:   newline_named_param_name_start_mid_end_param_value_start
-label:  a newline at the start, middle, and end of a parameter name and at the start of a parameter value
-input:  "{{foo|\nb\nar\n=\nbaz}}"
-output: [Text(text="{{foo|\nb\nar\n=\nbaz}}")]
+name:   newlines_named_param_name_param_value
+label:  newlines within a parameter name and within a parameter value
+input:  "{{foo|\nb\nar\n=\nba\nz\n}}"
+output: [TemplateOpen(), Text(text="foo"), TemplateParamSeparator(), Text(text="\nb\nar\n"), TemplateParamEquals(), Text(text="\nba\nz\n"), TemplateClose()]
 
 ---
 
-name:   newline_named_param_name_start_param_value_start_end
-label:  a newline at the start of a parameter name and at the start and end of a parameter value
-input:  "{{foo|\nbar=\nbaz\n}}"
-output: [TemplateOpen(), Text(text="foo"), TemplateParamSeparator(), Text(text="\nbar"), TemplateParamEquals(), Text(text="\nbaz\n"), TemplateClose()]
+name:   newlines_enclose_template_name_named_param_name
+label:  newlines enclosing a template name and within a parameter name
+input:  "{{\nfoo\n|\nb\nar\n=baz}}"
+output: [TemplateOpen(), Text(text="\nfoo\n"), TemplateParamSeparator(), Text(text="\nb\nar\n"), TemplateParamEquals(), Text(text="baz"), TemplateClose()]
 
 ---
 
-name:   newline_named_param_name_end_param_value_start_end
-label:  a newline at the end of a parameter name and at the start and end of a parameter value
-input:  "{{foo|bar\n=\nbaz\n}}"
-output: [TemplateOpen(), Text(text="foo"), TemplateParamSeparator(), Text(text="bar\n"), TemplateParamEquals(), Text(text="\nbaz\n"), TemplateClose()]
+name:   newlines_enclose_template_name_named_param_name_param_value
+label:  newlines enclosing a template name and within a parameter name and within a parameter value
+input:  "{{\nfoo\n|\nb\nar\n=\nba\nz\n}}"
+output: [TemplateOpen(), Text(text="\nfoo\n"), TemplateParamSeparator(), Text(text="\nb\nar\n"), TemplateParamEquals(), Text(text="\nba\nz\n"), TemplateClose()]
 
 ---
 
-name:   newline_named_param_name_start_end_param_value_start_end
-label:  a newline at the start and end of a parameter name and at the start and end of a parameter value
-input:  "{{foo|\nbar\n=\nbaz\n}}"
-output: [TemplateOpen(), Text(text="foo"), TemplateParamSeparator(), Text(text="\nbar\n"), TemplateParamEquals(), Text(text="\nbaz\n"), TemplateClose()]
+name:   newlines_within_template_name_named_param_name
+label:  newlines within a template name and within a parameter name
+input:  "{{\nfo\no\n|\nb\nar\n=baz}}"
+output: [Text(text="{{\nfo\no\n|\nb\nar\n=baz}}")]
 
 ---
 
-name:   newline_named_param_name_start_mid_param_value_start_end
-label:  a newline at the start and middle of a parameter name and at the start and end of a parameter value
-input:  "{{foo|\nb\nar=\nbaz\n}}"
-output: [Text(text="{{foo|\nb\nar=\nbaz\n}}")]
+name:   newlines_within_template_name_named_param_name_param_value
+label:  newlines within a template name and within a parameter name and within a parameter value
+input:  "{{\nf\noo\n|\nb\nar\n=\nba\nz\n}}"
+output: [Text(text="{{\nf\noo\n|\nb\nar\n=\nba\nz\n}}")]
 
 ---
 
-name:   newline_named_param_name_mid_end_param_value_start_end
-label:  a newline at the middle and end of a parameter name and at the start and end of a parameter value
-input:  "{{foo|b\nar\n=\nbaz\n}}"
-output: [Text(text="{{foo|b\nar\n=\nbaz\n}}")]
-
-
-name:   newline_named_param_name_start_mid_end_param_value_start_end
-label:  a newline at the start, middle, and end of a parameter name and at the start and end of a parameter value
-input:  "{{foo|\nb\nar\n=\nbaz\n}}"
-output: [Text(text="{{foo|\nb\nar\n=\nbaz\n}}")]
-
-
-name:   newline_named_param_name_start_param_value_mid
-label:  a newline at the start of a parameter name and at the middle of a parameter value
-input:  "{{foo|\nbar=ba\nz}}"
-output: [TemplateOpen(), Text(text="foo"), TemplateParamSeparator(), Text(text="\nbar"), TemplateParamEquals(), Text(text="ba\nz"), TemplateClose()]
-
-
-name:   newline_named_param_name_end_param_value_mid
-label:  a newline at the end of a parameter name and at the middle of a parameter value
-input:  "{{foo|bar\n=ba\nz}}"
-output: [TemplateOpen(), Text(text="foo"), TemplateParamSeparator(), Text(text="bar\n"), TemplateParamEquals(), Text(text="ba\nz"), TemplateClose()]
-
-
-name:   newline_named_param_name_start_end_param_value_mid
-label:  a newline at the start and end of a parameter name and at the middle of a parameter value
-input:  "{{foo|\nbar\n=ba\nz}}"
-output: [TemplateOpen(), Text(text="foo"), TemplateParamSeparator(), Text(text="\nbar\n"), TemplateParamEquals(), Text(text="ba\nz"), TemplateClose()]
-
-
-name:   newline_named_param_name_start_mid_param_value_mid
-label:  a newline at the start and middle of a parameter name and at the middle of a parameter value
-input:  "{{foo|\nb\nar=ba\nz}}"
-output: [Text(text="{{foo|\nb\nar=ba\nz}}")]
-
-
-name:   newline_named_param_name_mid_end_param_value_mid
-label:  a newline at the middle and end of a parameter name and at the middle of a parameter value
-input:  "{{foo|b\nar\n=ba\nz}}"
-output: [Text(text="{{foo|b\nar\n=ba\nz}}")]
-
-
-name:   newline_named_param_start_mid_end_param_value_mid
-label:  a newline at the start, middle, and end of a parameter name and at the middle of a parameter value
-input:  "{{foo|\nb\nar\n=ba\nz}}"
-output: [Text(text="{{foo|\nb\nar\n=ba\nz}}")]
+name:   newlines_wildcard
+label:  a random, complex assortment of templates and newlines
+input:  "{{\nfoo\n|\nb\nar\n=\nb\naz\n|\nb\nuz\n}}"
+output: [TemplateOpen(), Text(text="\nfoo\n"), TemplateParamSeparator(), Text(text="\nb\nar\n"), TemplateParamEquals(), Text(text="\nb\naz\n"), TemplateParamSeparator(), Text(text="\nb\nuz\n"), TemplateClose()]
 
 ---
 
-name:   newline_wildcard
-label:  a random, complex assortment of templates and newlines
-input:  "{{\nfoo\n|\nbar\n=\nb\naz\n|\nb\nuz\n}}"
-output: [TemplateOpen(), Text(text="\nfoo\n"), TemplateParamSeparator(), Text(text="\nbar\n"), TemplateParamEquals(), Text(text="\nb\naz\n"), TemplateParamSeparator(), Text(text="\nb\nuz\n"), TemplateClose()]
+name:   newlines_wildcard_redux
+label:  an even more random and complex assortment of templates and newlines
+input:  "{{\nfoo\n|\n{{\nbar\n|\nb\naz\n=\nb\niz\n}}\n=\nb\nuzz\n}}"
+output: [TemplateOpen(), Text(text="\nfoo\n"), TemplateParamSeparator(), Text(text="\n"), TemplateOpen(), Text(text="\nbar\n"), TemplateParamSeparator(), Text(text="\nb\naz\n"), TemplateParamEquals(), Text(text="\nb\niz\n"), TemplateClose(), Text(text="\n"), TemplateParamEquals(), Text(text="\nb\nuzz\n"), TemplateClose()]
 
 ---
 
-name:   newline_wildcard_redux
-label:  an even more random and complex assortment of templates and newlines
-input:  "{{\nfoo\n|\n{{\nbar\n|\nbaz\n=\nb\niz\n}}\n=\nb\nuzz\n}}"
-output: [TemplateOpen(), Text(text="\nfoo\n"), TemplateParamSeparator(), Text(text="\n"), TemplateOpen(), Text(text="\nbar\n"), TemplateParamSeparator(), Text(text="\nbaz\n"), TemplateParamEquals(), Text(text="\nb\niz\n"), TemplateClose(), Text(text="\n"), TemplateParamEquals(), Text(text="\nb\nuzz\n"), TemplateClose()]
+name:   newlines_wildcard_redux_invalid
+label:  a variation of the newlines_wildcard_redux test that is invalid
+input:  "{{\nfoo\n|\n{{\nb\nar\n|\nb\naz\n=\nb\niz\n}}\n=\nb\nuzz\n}}"
+output: [Text(text="{{\nfoo\n|\n{{\nb\nar\n|\nb\naz\n=\nb\niz\n}}\n=\nb\nuzz\n}}")]
 
 ---
 
@@ -812,8 +441,43 @@ output: [TemplateOpen(), Text(text="foo"), TemplateParamSeparator(), Text(text="
 
 name:   valid_param_name_brackets
 label:  valid characters in unnamed template parameter: left and right brackets
-input:  "{{foo|ba[r]}}"
-output: [TemplateOpen(), Text(text="foo"), TemplateParamSeparator(), Text(text="ba[r]"), TemplateClose()]
+input:  "{{foo|ba[r]=baz}}"
+output: [TemplateOpen(), Text(text="foo"), TemplateParamSeparator(), Text(text="ba[r]"), TemplateParamEquals(), Text(text="baz"), TemplateClose()]
+
+---
+
+name:   valid_param_name_double_left_brackets
+label:  valid characters in unnamed template parameter: double left brackets
+input:  "{{foo|bar[[in\nvalid=baz}}"
+output: [TemplateOpen(), Text(text="foo"), TemplateParamSeparator(), Text(text="bar[[in\nvalid"), TemplateParamEquals(), Text(text="baz"), TemplateClose()]
+
+---
+
+name:   valid_param_name_double_right_brackets
+label:  valid characters in unnamed template parameter: double right brackets
+input:  "{{foo|bar]]=baz}}"
+output: [TemplateOpen(), Text(text="foo"), TemplateParamSeparator(), Text(text="bar]]"), TemplateParamEquals(), Text(text="baz"), TemplateClose()]
+
+---
+
+name:   valid_param_name_double_brackets
+label:  valid characters in unnamed template parameter: double left and right brackets
+input:  "{{foo|bar[[in\nvalid]]=baz}}"
+output: [TemplateOpen(), Text(text="foo"), TemplateParamSeparator(), Text(text="bar[[in\nvalid]]"), TemplateParamEquals(), Text(text="baz"), TemplateClose()]
+
+---
+
+name:   invalid_param_name_double_left_braces
+label:  invalid characters in template parameter name: double left braces
+input:  "{{foo|bar{{in\nvalid=baz}}"
+output: [Text(text="{{foo|bar{{in\nvalid=baz}}")]
+
+---
+
+name:   invalid_param_name_double_braces
+label:  invalid characters in template parameter name: double left and right braces
+input:  "{{foo|bar{{in\nvalid}}=baz}}"
+output: [TemplateOpen(), Text(text="foo"), TemplateParamSeparator(), Text(text="bar{{in\nvalid"), TemplateClose(), Text(text="=baz}}")]
 
 ---
 
@@ -919,3 +583,17 @@ name:   incomplete_two_named_params
 label:  incomplete templates that should fail gracefully: two named parameters with values
 input:  "{{stuff}} {{foo|bar=baz|biz=buzz"
 output: [TemplateOpen(), Text(text="stuff"), TemplateClose(), Text(text=" {{foo|bar=baz|biz=buzz")]
+
+---
+
+name:   incomplete_nested_template_as_unnamed_param
+label:  incomplete templates that should fail gracefully: a valid nested template as an unnamed parameter
+input:  "{{stuff}} {{foo|{{bar}}"
+output: [TemplateOpen(), Text(text="stuff"), TemplateClose(), Text(text=" {{foo|"), TemplateOpen(), Text(text="bar"), TemplateClose()]
+
+---
+
+name:   incomplete_nested_template_as_param_value
+label:  incomplete templates that should fail gracefully: a valid nested template as a parameter value
+input:  "{{stuff}} {{foo|bar={{baz}}"
+output: [TemplateOpen(), Text(text="stuff"), TemplateClose(), Text(text=" {{foo|bar="), TemplateOpen(), Text(text="baz"), TemplateClose()]