From 93c51fe57c1711c674c41ea0799be5193ff3bf21 Mon Sep 17 00:00:00 2001 From: Ben Kurtovic Date: Thu, 22 Aug 2013 01:58:27 -0400 Subject: [PATCH] Tokenizer tests for external links. --- tests/tokenizer/external_links.mwtest | 459 ++++++++++++++++++++++++++++++++++ 1 file changed, 459 insertions(+) create mode 100644 tests/tokenizer/external_links.mwtest diff --git a/tests/tokenizer/external_links.mwtest b/tests/tokenizer/external_links.mwtest new file mode 100644 index 0000000..9e0ebdd --- /dev/null +++ b/tests/tokenizer/external_links.mwtest @@ -0,0 +1,459 @@ +name: basic +label: basic external link +input: "http://example.com/" +output: [ExternalLinkOpen(brackets=False), Text(text="http://example.com/"), ExternalLinkClose()] + +--- + +name: basic_brackets +label: basic external link in brackets +input: "[http://example.com/]" +output: [ExternalLinkOpen(brackets=True), Text(text="http://example.com/"), ExternalLinkClose()] + +--- + +name: brackets_space +label: basic external link in brackets, with a space after +input: "[http://example.com/ ]" +output: [ExternalLinkOpen(brackets=True), Text(text="http://example.com/"), ExternalLinkSeparator(), Text(text=""), ExternalLinkClose()] + +--- + +name: brackets_title +label: basic external link in brackets, with a title +input: "[http://example.com/ Example]" +output: [ExternalLinkOpen(brackets=True), Text(text="http://example.com/"), ExternalLinkSeparator(), Text(text="Example"), ExternalLinkClose()] + +--- + +name: brackets_multiword_title +label: basic external link in brackets, with a multi-word title +input: "[http://example.com/ Example Web Page]" +output: [ExternalLinkOpen(brackets=True), Text(text="http://example.com/"), ExternalLinkSeparator(), Text(text="Example Web Page"), ExternalLinkClose()] + +--- + +name: brackets_adjacent +label: three adjacent bracket-enclosed external links +input: "[http://foo.com/ Foo][http://bar.com/ Bar]\n[http://baz.com/ Baz]" +output: [ExternalLinkOpen(brackets=True), Text(text="http://foo.com/"), ExternalLinkSeparator(), Text(text="Foo"), ExternalLinkClose(), ExternalLinkOpen(brackets=True), Text(text="http://bar.com/"), ExternalLinkSeparator(), Text(text="Bar"), ExternalLinkClose(), Text(text="\n"), ExternalLinkOpen(brackets=True), Text(text="http://baz.com/"), ExternalLinkSeparator(), Text(text="Baz"), ExternalLinkClose()] + +--- + +name: brackets_newline_before +label: bracket-enclosed link with a newline before the title +input: "[http://example.com/ \nExample]" +output: [Text(text="["), ExternalLinkOpen(brackets=False), Text(text="http://example.com/"), ExternalLinkClose(), Text(text=" \nExample]")] + +--- + +name: brackets_newline_inside +label: bracket-enclosed link with a newline in the title +input: "[http://example.com/ Example \nWeb Page]" +output: [Text(text="["), ExternalLinkOpen(brackets=False), Text(text="http://example.com/"), ExternalLinkClose(), Text(text=" Example \nWeb Page]")] + +--- + +name: brackets_newline_after +label: bracket-enclosed link with a newline after the title +input: "[http://example.com/ Example\n]" +output: [Text(text="["), ExternalLinkOpen(brackets=False), Text(text="http://example.com/"), ExternalLinkClose(), Text(text=" Example\n]")] + +--- + +name: brackets_space_before +label: bracket-enclosed link with a space before the URL +input: "[ http://example.com Example]" +output: [Text(text="[ "), ExternalLinkOpen(brackets=False), Text(text="http://example.com"), ExternalLinkClose(), Text(text=" Example]")] + +--- + +name: brackets_title_like_url +label: bracket-enclosed link with a title that looks like a URL +input: "[http://example.com http://example.com]" +output: [ExternalLinkOpen(brackets=True), Text(text="http://example.com"), ExternalLinkSeparator(), Text(text="http://example.com"), ExternalLinkClose()] + +--- + +name: brackets_recursive +label: bracket-enclosed link with a bracket-enclosed link as the title +input: "[http://example.com [http://example.com]]" +output: [ExternalLinkOpen(brackets=True), Text(text="http://example.com"), ExternalLinkSeparator(), Text(text="[http://example.com"), ExternalLinkClose(), Text(text="]")] + +--- + +name: period_after +label: a period after a free link that is excluded +input: "http://example.com." +output: [ExternalLinkOpen(brackets=False), Text(text="http://example.com"), ExternalLinkClose(), Text(text=".")] + +--- + +name: colons_after +label: colons after a free link that are excluded +input: "http://example.com/foo:bar:::baz:::" +output: [ExternalLinkOpen(brackets=False), Text(text="http://example.com/foo:bar:::baz"), ExternalLinkClose(), Text(text=":::")] + +--- + +name: close_paren_after_excluded +label: a closing parenthesis after a free link that is excluded +input: "http://example.)com)" +output: [ExternalLinkOpen(brackets=False), Text(text="http://example.)com"), ExternalLinkClose(), Text(text=")")] + +--- + +name: close_paren_after_included +label: a closing parenthesis after a free link that is included because of an opening parenthesis in the URL +input: "http://example.(com)" +output: [ExternalLinkOpen(brackets=False), Text(text="http://example.(com)"), ExternalLinkClose()] + +--- + +name: open_bracket_inside +label: an open bracket inside a free link that causes it to be ended abruptly +input: "http://foobar[baz.com" +output: [ExternalLinkOpen(brackets=False), Text(text="http://foobar"), ExternalLinkClose(), Text(text="[baz.com")] + +--- + +name: brackets_period_after +label: a period after a bracket-enclosed link that is included +input: "[http://example.com. Example]" +output: [ExternalLinkOpen(brackets=True), Text(text="http://example.com."), ExternalLinkSeparator(), Text(text="Example"), ExternalLinkClose()] + +--- + +name: brackets_colons_after +label: colons after a bracket-enclosed link that are included +input: "[http://example.com/foo:bar:::baz::: Example]" +output: [ExternalLinkOpen(brackets=True), Text(text="http://example.com/foo:bar:::baz:::"), ExternalLinkSeparator(), Text(text="Example"), ExternalLinkClose()] + +--- + +name: brackets_close_paren_after_included +label: a closing parenthesis after a bracket-enclosed link that is included +input: "[http://example.)com) Example]" +output: [ExternalLinkOpen(brackets=True), Text(text="http://example.)com)"), ExternalLinkSeparator(), Text(text="Example"), ExternalLinkClose()] + +--- + +name: brackets_close_paren_after_included_2 +label: a closing parenthesis after a bracket-enclosed link that is also included +input: "[http://example.(com) Example]" +output: [ExternalLinkOpen(brackets=True), Text(text="http://example.(com)"), ExternalLinkSeparator(), Text(text="Example"), ExternalLinkClose()] + +--- + +name: brackets_open_bracket_inside +label: an open bracket inside a bracket-enclosed link that causes it to switch to the title context abruptly +input: "[http://foobar[baz.com Example]" +output: [ExternalLinkOpen(brackets=True), Text(text="http://foobar"), ExternalLinkSeparator(), Text(text="[baz.com Example"), ExternalLinkClose()] + +--- + +name: adjacent_space +label: two free links separated by a space +input: "http://example.com http://example.com" +output: [ExternalLinkOpen(brackets=False), Text(text="http://example.com"), ExternalLinkClose(), Text(text=" "), ExternalLinkOpen(brackets=False), Text(text="http://example.com"), ExternalLinkClose()] + +--- + +name: adjacent_newline +label: two free links separated by a newline +input: "http://example.com\nhttp://example.com" +output: [ExternalLinkOpen(brackets=False), Text(text="http://example.com"), ExternalLinkClose(), Text(text="\n"), ExternalLinkOpen(brackets=False), Text(text="http://example.com"), ExternalLinkClose()] + +--- + +name: adjacent_close_bracket +label: two free links separated by a close bracket +input: "http://example.com]http://example.com" +output: [ExternalLinkOpen(brackets=False), Text(text="http://example.com"), ExternalLinkClose(), Text(text="]"), ExternalLinkOpen(brackets=False), Text(text="http://example.com"), ExternalLinkClose()] + +--- + +name: html_entity_in_url +label: a HTML entity parsed correctly inside a free link +input: "http://exa mple.com/" +output: [ExternalLinkOpen(brackets=False), Text(text="http://exa"), HTMLEntityStart(), Text(text="nbsp"), HTMLEntityEnd(), Text(text="mple.com/"), ExternalLinkClose()] + +--- + +name: template_in_url +label: a template parsed correctly inside a free link +input: "http://exa{{template}}mple.com/" +output: [ExternalLinkOpen(brackets=False), Text(text="http://exa"), TemplateOpen(), Text(text="template"), TemplateClose(), Text(text="mple.com/"), ExternalLinkClose()] + +--- + +name: argument_in_url +label: an argument parsed correctly inside a free link +input: "http://exa{{{argument}}}mple.com/" +output: [ExternalLinkOpen(brackets=False), Text(text="http://exa"), ArgumentOpen(), Text(text="argument"), ArgumentClose(), Text(text="mple.com/"), ExternalLinkClose()] + +--- + +name: wikilink_in_url +label: a wikilink that destroys a free link +input: "http://exa[[wikilink]]mple.com/" +output: [ExternalLinkOpen(brackets=False), Text(text="http://exa"), ExternalLinkClose(), WikilinkOpen(), Text(text="wikilink"), WikilinkClose(), Text(text="mple.com/")] + +--- + +name: external_link_in_url +label: a bracketed link that destroys a free link +input: "http://exa[http://example.com/]mple.com/" +output: [ExternalLinkOpen(brackets=False), Text(text="http://exa"), ExternalLinkClose(), ExternalLinkOpen(brackets=True), Text(text="http://example.com/"), ExternalLinkClose(), Text(text="mple.com/")] + +--- + +name: spaces_padding +label: spaces padding a free link +input: " http://example.com " +output: [Text(text=" "), ExternalLinkOpen(brackets=False), Text(text="http://example.com"), ExternalLinkClose(), Text(text=" ")] + +--- + +name: text_and_spaces_padding +label: text and spaces padding a free link +input: "x http://example.com x" +output: [Text(text="x "), ExternalLinkOpen(brackets=False), Text(text="http://example.com"), ExternalLinkClose(), Text(text=" x")] + +--- + +name: template_before +label: a template before a free link +input: "{{foo}}http://example.com" +output: [TemplateOpen(), Text(text="foo"), TemplateClose(), ExternalLinkOpen(brackets=False), Text(text="http://example.com"), ExternalLinkClose()] + +--- + +name: spaces_padding_no_slashes +label: spaces padding a free link with no slashes after the colon +input: " mailto:example@example.com " +output: [Text(text=" "), ExternalLinkOpen(brackets=False), Text(text="mailto:example@example.com"), ExternalLinkClose(), Text(text=" ")] + +--- + +name: text_and_spaces_padding_no_slashes +label: text and spaces padding a free link with no slashes after the colon +input: "x mailto:example@example.com x" +output: [Text(text="x "), ExternalLinkOpen(brackets=False), Text(text="mailto:example@example.com"), ExternalLinkClose(), Text(text=" x")] + +--- + +name: template_before_no_slashes +label: a template before a free link with no slashes after the colon +input: "{{foo}}mailto:example@example.com" +output: [TemplateOpen(), Text(text="foo"), TemplateClose(), ExternalLinkOpen(brackets=False), Text(text="mailto:example@example.com"), ExternalLinkClose()] + +--- + +name: no_slashes +label: a free link with no slashes after the colon +input: "mailto:example@example.com" +output: [ExternalLinkOpen(brackets=False), Text(text="mailto:example@example.com"), ExternalLinkClose()] + +--- + +name: slashes_optional +label: a free link using a scheme that doesn't need slashes, but has them anyway +input: "mailto://example@example.com" +output: [ExternalLinkOpen(brackets=False), Text(text="mailto://example@example.com"), ExternalLinkClose()] + +--- + +name: short +label: a very short free link +input: "mailto://abc" +output: [ExternalLinkOpen(brackets=False), Text(text="mailto://abc"), ExternalLinkClose()] + +--- + +name: slashes_missing +label: slashes missing from a free link with a scheme that requires them +input: "http:example@example.com" +output: [Text(text="http:example@example.com")] + +--- + +name: no_scheme_but_slashes +label: no scheme in a free link, but slashes (protocol-relative free links are not supported) +input: "//example.com" +output: [Text(text="//example.com")] + +--- + +name: no_scheme_but_colon +label: no scheme in a free link, but a colon +input: ":example.com" +output: [Text(text=":example.com")] + +--- + +name: no_scheme_but_colon_and_slashes +label: no scheme in a free link, but a colon and slashes +input: "://example.com" +output: [Text(text="://example.com")] + +--- + +name: fake_scheme_no_slashes +label: a nonexistent scheme in a free link, without slashes +input: "fake:example.com" +output: [Text(text="fake:example.com")] + +--- + +name: fake_scheme_slashes +label: a nonexistent scheme in a free link, with slashes +input: "fake://example.com" +output: [Text(text="fake://example.com")] + +--- + +name: fake_scheme_brackets_no_slashes +label: a nonexistent scheme in a bracketed link, without slashes +input: "[fake:example.com]" +output: [Text(text="[fake:example.com]")] + +--- + +name: fake_scheme_brackets_slashes +label: #=a nonexistent scheme in a bracketed link, with slashes +input: "[fake://example.com]" +output: [Text(text="[fake://example.com]")] + +--- + +name: interrupted_scheme +label: an otherwise valid scheme with something in the middle of it, in a free link +input: "ht?tp://example.com" +output: [Text(text="ht?tp://example.com")] + +--- + +name: interrupted_scheme_brackets +label: an otherwise valid scheme with something in the middle of it, in a bracketed link +input: "[ht?tp://example.com]" +output: [Text(text="[ht?tp://example.com]")] + +--- + +name: no_slashes_brackets +label: no slashes after the colon in a bracketed link +input: "[mailto:example@example.com Example]" +output: [ExternalLinkOpen(brackets=True), Text(text="mailto:example@example.com"), ExternalLinkSeparator(), Text(text="Example"), ExternalLinkClose()] + +--- + +name: space_before_no_slashes_brackets +label: a space before a bracketed link with no slashes after the colon +input: "[ mailto:example@example.com Example]" +output: [Text(text="[ "), ExternalLinkOpen(brackets=False), Text(text="mailto:example@example.com"), ExternalLinkClose(), Text(text=" Example]")] + +--- + +name: slashes_optional_brackets +label: a bracketed link using a scheme that doesn't need slashes, but has them anyway +input: "[mailto://example@example.com Example]" +output: [ExternalLinkOpen(brackets=True), Text(text="mailto://example@example.com"), ExternalLinkSeparator(), Text(text="Example"), ExternalLinkClose()] + +--- + +name: short_brackets +label: a very short link in brackets +input: "[mailto://abc Example]" +output: [ExternalLinkOpen(brackets=True), Text(text="mailto://abc"), ExternalLinkSeparator(), Text(text="Example"), ExternalLinkClose()] + +--- + +name: slashes_missing_brackets +label: slashes missing from a scheme that requires them in a bracketed link +input: "[http:example@example.com Example]" +output: [Text(text="[http:example@example.com Example]")] + +--- + +name: protcol_relative +label: a protocol-relative link (in brackets) +input: "[//example.com Example]" +output: [ExternalLinkOpen(brackets=True), Text(text="//example.com"), ExternalLinkSeparator(), Text(text="Example"), ExternalLinkClose()] + +--- + +name: scheme_missing_but_colon_brackets +label: scheme missing from a bracketed link, but with a colon +input: "[:example.com Example]" +output: [Text(text="[:example.com Example]")] + +--- + +name: scheme_missing_but_colon_slashes_brackets +label: scheme missing from a bracketed link, but with a colon and slashes +input: "[://example.com Example]" +output: [Text(text="[://example.com Example]")] + +--- + +name: unclosed_protocol_relative +label: an unclosed protocol-relative bracketed link +input: "[//example.com" +output: [Text(text="[//example.com")] + +--- + +name: space_before_protcol_relative +label: a space before a protocol-relative bracketed link +input: "[ //example.com]" +output: [Text(text="[ //example.com]")] + +--- + +name: unclosed_just_scheme +label: an unclosed bracketed link, ending after the scheme +input: "[http" +output: [Text(text="[http")] + +--- + +name: unclosed_scheme_colon +label: an unclosed bracketed link, ending after the colon +input: "[http:" +output: [Text(text="[http:")] + +--- + +name: unclosed_scheme_colon_slashes +label: an unclosed bracketed link, ending after the slashes +input: "[http://" +output: [Text(text="[http://")] + +--- + +name: incomplete_scheme_colon +label: a free link with just a scheme and a colon +input: "http:" +output: [Text(text="http:")] + +--- + +name: incomplete_scheme_colon_slashes +label: a free link with just a scheme, colon, and slashes +input: "http://" +output: [Text(text="http://")] + +--- + +name: brackets_scheme_but_no_url +label: brackets around a scheme, colon, and slashes +input: "[http://]" +output: [Text(text="[http://]")] + +--- + +name: brackets_scheme_title_but_no_url +label: brackets around a scheme, colon, and slashes, with a title +input: "[http:// Example]" +output: [Text(text="[http:// Example]")]