From b9324eb658eda01a874d18cf193b6647ba3d0e5e Mon Sep 17 00:00:00 2001 From: Ben Kurtovic Date: Fri, 23 Aug 2013 02:29:00 -0400 Subject: [PATCH] Fix Python tokenizer to not generate empty Text tokens. --- mwparserfromhell/parser/tokenizer.py | 3 ++- tests/tokenizer/external_links.mwtest | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/mwparserfromhell/parser/tokenizer.py b/mwparserfromhell/parser/tokenizer.py index 2340077..2c8d6d7 100644 --- a/mwparserfromhell/parser/tokenizer.py +++ b/mwparserfromhell/parser/tokenizer.py @@ -424,7 +424,8 @@ class Tokenizer(object): if brackets: self._emit_text(before) self._emit(tokens.ExternalLinkSeparator()) - self._emit_text(after) + if after: + self._emit_text(after) self._context ^= contexts.EXT_LINK_URI self._context |= contexts.EXT_LINK_TITLE self._head += 1 diff --git a/tests/tokenizer/external_links.mwtest b/tests/tokenizer/external_links.mwtest index b517ada..6666c05 100644 --- a/tests/tokenizer/external_links.mwtest +++ b/tests/tokenizer/external_links.mwtest @@ -15,7 +15,7 @@ output: [ExternalLinkOpen(brackets=True), Text(text="http://example.com/"), Exte name: brackets_space label: basic external link in brackets, with a space after input: "[http://example.com/ ]" -output: [ExternalLinkOpen(brackets=True), Text(text="http://example.com/"), ExternalLinkSeparator(), Text(text=""), ExternalLinkClose()] +output: [ExternalLinkOpen(brackets=True), Text(text="http://example.com/"), ExternalLinkSeparator(), ExternalLinkClose()] ---