Add a few unit tests for untested code; remove a useless conditional.

mwparserfromhell/parser/tokenizer.c

@@ -2508,10 +2508,7 @@ static int Tokenizer_verify_safe(Tokenizer* self, int context, Py_UNICODE data)
         }
         else if (context & LC_FAIL_ON_RBRACE) {
             if (data == '}') {
-                if (context & LC_TEMPLATE)
-                    self->topstack->context |= LC_FAIL_ON_EQUALS;
-                else
-                    self->topstack->context |= LC_FAIL_NEXT;
+                self->topstack->context |= LC_FAIL_NEXT;
                 return 0;
             }
             self->topstack->context ^= LC_FAIL_ON_RBRACE;

mwparserfromhell/parser/tokenizer.py

@@ -1040,10 +1040,7 @@ class Tokenizer(object):
                 self._context ^= contexts.FAIL_ON_LBRACE
             elif context & contexts.FAIL_ON_RBRACE:
                 if this == "}":
-                    if context & contexts.TEMPLATE:
-                        self._context |= contexts.FAIL_ON_EQUALS
-                    else:
-                        self._context |= contexts.FAIL_NEXT
+                    self._context |= contexts.FAIL_NEXT
                     return True
                 self._context ^= contexts.FAIL_ON_RBRACE
             elif this == "{":

tests/tokenizer/comments.mwtest

@@ -37,3 +37,10 @@ name:   incomplete_partial_close
 label:  a comment that doesn't close, with a partial close
 input:  "<!-- foo --\x01>"
 output: [Text(text="<!-- foo --\x01>")]
+
+---
+
+name:   incomplete_part_only
+label:  a comment that only has a < and !
+input:  "<!foo"
+output: [Text(text="<!foo")]

tests/tokenizer/external_links.mwtest

@@ -124,8 +124,8 @@ output: [ExternalLinkOpen(brackets=True), Text(text="http://example.com."), Exte
 
 ---
 
-name:   brackets_colons_after
-label:  colons after a bracket-enclosed link that are included
+name:   brackets_punct_after
+label:  punctuation after a bracket-enclosed link that are included
 input:  "[http://example.com/foo:bar.:;baz!?, Example]"
 output: [ExternalLinkOpen(brackets=True), Text(text="http://example.com/foo:bar.:;baz!?,"), ExternalLinkSeparator(), Text(text="Example"), ExternalLinkClose()]
 

tests/tokenizer/integration.mwtest

@@ -185,3 +185,10 @@ name:   italics_inside_external_link_inside_incomplete_list
 label:  italic text inside an external link inside an incomplete list
 input:  "<li>[http://www.example.com ''example'']"
 output: [TagOpenOpen(), Text(text="li"), TagCloseSelfclose(padding="", implicit=True), ExternalLinkOpen(brackets=True), Text(text="http://www.example.com"), ExternalLinkSeparator(), TagOpenOpen(wiki_markup="''"), Text(text="i"), TagCloseOpen(), Text(text="example"), TagOpenClose(), Text(text="i"), TagCloseClose(), ExternalLinkClose()]
+
+---
+
+name:   nodes_inside_external_link_after_punct
+label:  various complex nodes inside an external link following punctuation
+input:  "http://example.com/foo.{{bar}}baz.&biz;<!--hello-->bingo"
+output: [ExternalLinkOpen(brackets=False), Text(text="http://example.com/foo."), TemplateOpen(), Text(text="bar"), TemplateClose(), Text(text="baz.&biz;"), CommentStart(), Text(text="hello"), CommentEnd(), Text(text="bingo"), ExternalLinkClose()]

tests/tokenizer/tags_wikimarkup.mwtest

@@ -244,6 +244,13 @@ output: [Text(text="''"), TagOpenOpen(wiki_markup="''"), Text(text="i"), TagClos
 
 ---
 
+name:   unending_bold_and_italics
+label:  five ticks (bold and italics) that don't end
+input:  "'''''testing"
+output: [Text(text="'''''testing")]
+
+---
+
 name:   complex_ul
 label:  ul with a lot in it
 input:  "* this is a test of an [[Unordered list|ul]] with {{plenty|of|stuff}}"