Tokenizer tests for arguments.

Also add a couple for templates and one for integration.

tests/tokenizer/arguments.mwtest

@@ -1,4 +1,123 @@
-name:   no_params
+name:   blank
+label:  argument with no content
+input:  "{{{}}}"
+output: [ArgumentOpen(), ArgumentClose()]
+
+---
+
+name:   blank_with_default
+label:  argument with no content but a pipe
+input:  "{{{|}}}"
+output: [ArgumentOpen(), ArgumentSeparator(), ArgumentClose()]
+
+---
+
+name:   basic
 label:  simplest type of argument
 input:  "{{{argument}}}"
 output: [ArgumentOpen(), Text(text="argument"), ArgumentClose()]
+
+---
+
+name:   default
+label:  argument with a default value
+input:  "{{{foo|bar}}}"
+output: [ArgumentOpen(), Text(text="foo"), ArgumentSeparator(), Text(text="bar"), ArgumentClose()]
+
+---
+
+name:   blank_with_multiple_defaults
+label:  no content, multiple pipes
+input:  "{{{|||}}}"
+output: [ArgumentOpen(), ArgumentSeparator(), Text(text="||"), ArgumentClose()]
+
+---
+
+name:   multiple_defaults
+label:  multiple values separated by pipes
+input:  "{{{foo|bar|baz}}}"
+output: [ArgumentOpen(), Text(text="foo"), ArgumentSeparator(), Text(text="bar|baz"), ArgumentClose()]
+
+---
+
+name:   newline
+label:  newline as only content
+input:  "{{{\n}}}"
+output: [ArgumentOpen(), Text(text="\n"), ArgumentClose()]
+
+---
+
+name:   right_braces
+label:  multiple } scattered throughout text
+input:  "{{{foo}b}a}r}}}"
+output: [ArgumentOpen(), Text(text="foo}b}a}r"), ArgumentClose()]
+
+---
+
+name:   right_braces_default
+label:  multiple } scattered throughout text, with a default value
+input:  "{{{foo}b}|}a}r}}}"
+output: [ArgumentOpen(), Text(text="foo}b}"), ArgumentSeparator(), Text(text="}a}r"), ArgumentClose()]
+
+---
+
+name:   invalid_braces
+label:  invalid argument: multiple braces that are not part of a template or argument
+input:  "{{{foo{{[a}}}}}"
+output: [Text(text="{{{foo{{[a}}}}}")]
+
+---
+
+name:   incomplete_open_only
+label:  incomplete arguments: just an open
+input:  "{{{"
+output: [Text(text="{{{")]
+
+---
+
+name:   incomplete_open_text
+label:  incomplete arguments: an open with some text
+input:  "{{{foo"
+output: [Text(text="{{{foo")]
+
+---
+
+name:   incomplete_open_text_pipe
+label:  incomplete arguments: an open, text, then a pipe
+input:  "{{{foo|"
+output: [Text(text="{{{foo|")]
+
+---
+
+name:   incomplete_open_pipe
+label:  incomplete arguments: an open, then a pipe
+input:  "{{{|"
+output: [Text(text="{{{|")]
+
+---
+
+name:   incomplete_open_pipe_text
+label:  incomplete arguments: an open, then a pipe, then text
+input:  "{{{|foo"
+output: [Text(text="{{{|foo")]
+
+---
+
+name:   incomplete_open_pipes_text
+label:  incomplete arguments: a pipe, then text then two pipes
+input:  "{{{|f||"
+output: [Text(text="{{{|f||")]
+
+---
+
+name:   incomplete_open_partial_close
+label:  incomplete arguments: an open, then one right brace
+input:  "{{{{}"
+output: [Text(text="{{{{}")]
+
+---
+
+name:   incomplete_preserve_previous
+label:  incomplete arguments: a valid argument followed by an invalid one
+input:  "{{{foo}}} {{{bar"
+output: [ArgumentOpen(), Text(text="foo"), ArgumentClose(), Text(text=" {{{bar")]

tests/tokenizer/integration.mwtest

@@ -0,0 +1,4 @@
+name:   empty
+label:  sanity check that parsing an empty string yields nothing
+input:  ""
+output: []

tests/tokenizer/templates.mwtest

@@ -1,3 +1,17 @@
+name:   blank
+label:  template with no content
+input:  "{{}}"
+output: [TemplateOpen(), TemplateClose()]
+
+---
+
+name:   blank_with_params
+label:  template with no content, but pipes and equal signs
+input:  "{{||=|}}"
+output: [TemplateOpen(), TemplateParamSeparator(), TemplateParamSeparator(), TemplateParamEquals(), TemplateParamSeparator(), TemplateClose()]
+
+---
+
 name:   no_params
 label:  simplest type of template
 input:  "{{template}}"