Ben Kurtovic
11 years ago
1 changed files with 22 additions and 29 deletions
Unified View
Diff Options
+ 22
- 29
mwparserfromhell/parser/tokenizer.py
View File
| @@ -98,10 +98,15 @@ class Tokenizer(object): | |||||
| raise BadRoute() | raise BadRoute() | ||||
| def _write(self, token): | def _write(self, token): | ||||
| """Write a token to the current token stack.""" | |||||
| """Write a token to the end of the current token stack.""" | |||||
| self._push_textbuffer() | self._push_textbuffer() | ||||
| self._stack.append(token) | self._stack.append(token) | ||||
| def _write_first(self, token): | |||||
| """Write a token to the beginning of the current token stack.""" | |||||
| self._push_textbuffer() | |||||
| self._stack.insert(0, token) | |||||
| def _write_text(self, text): | def _write_text(self, text): | ||||
| """Write text to the current textbuffer.""" | """Write text to the current textbuffer.""" | ||||
| self._textbuffer.append(text) | self._textbuffer.append(text) | ||||
| @@ -113,6 +118,13 @@ class Tokenizer(object): | |||||
| self._push_textbuffer() | self._push_textbuffer() | ||||
| self._stack.extend(tokenlist) | self._stack.extend(tokenlist) | ||||
| def _write_text_then_stack(self, text): | |||||
| """Pop the current stack, write *text*, and then write the stack.""" | |||||
| stack = self._pop() | |||||
| self._write_text(text) | |||||
| self._write_all(stack) | |||||
| self._head -= 1 | |||||
| def _read(self, delta=0, wrap=False, strict=False): | def _read(self, delta=0, wrap=False, strict=False): | ||||
| """Read the value at a relative point in the wikicode. | """Read the value at a relative point in the wikicode. | ||||
| @@ -146,20 +158,12 @@ class Tokenizer(object): | |||||
| while braces: | while braces: | ||||
| if braces == 1: | if braces == 1: | ||||
| stack = self._pop() | |||||
| self._write_text("{") | |||||
| self._write_all(stack) | |||||
| self._head -= 1 | |||||
| return | |||||
| return self._write_text_then_stack("{") | |||||
| if braces == 2: | if braces == 2: | ||||
| try: | try: | ||||
| self._parse_template() | self._parse_template() | ||||
| except BadRoute: | except BadRoute: | ||||
| stack = self._pop() | |||||
| self._write_text("{{") | |||||
| self._write_all(stack) | |||||
| self._head -= 1 | |||||
| return | |||||
| return self._write_text_then_stack("{{") | |||||
| break | break | ||||
| try: | try: | ||||
| self._parse_argument() | self._parse_argument() | ||||
| @@ -167,16 +171,11 @@ class Tokenizer(object): | |||||
| try: | try: | ||||
| self._parse_template() | self._parse_template() | ||||
| except BadRoute: | except BadRoute: | ||||
| stack = self._pop() | |||||
| self._write_text("{" * braces) | |||||
| self._write_all(stack) | |||||
| self._head -= 1 | |||||
| return | |||||
| else: | |||||
| stack = self._pop() | |||||
| self._write_text("{") | |||||
| self._push() | |||||
| self._write_all(stack) | |||||
| return self._write_text_then_stack("{" * braces) | |||||
| stack = self._pop() | |||||
| self._write_text("{") | |||||
| self._push() | |||||
| self._write_all(stack) | |||||
| braces -= 3 | braces -= 3 | ||||
| if braces: | if braces: | ||||
| self._head += 1 | self._head += 1 | ||||
| @@ -192,10 +191,7 @@ class Tokenizer(object): | |||||
| self._head = reset | self._head = reset | ||||
| raise | raise | ||||
| else: | else: | ||||
| stack = self._pop() | |||||
| self._push() | |||||
| self._write(tokens.TemplateOpen()) | |||||
| self._write_all(stack) | |||||
| self._write_first(tokens.TemplateOpen()) | |||||
| self._write_all(template) | self._write_all(template) | ||||
| self._write(tokens.TemplateClose()) | self._write(tokens.TemplateClose()) | ||||
| @@ -208,10 +204,7 @@ class Tokenizer(object): | |||||
| self._head = reset | self._head = reset | ||||
| raise | raise | ||||
| else: | else: | ||||
| stack = self._pop() | |||||
| self._push() | |||||
| self._write(tokens.ArgumentOpen()) | |||||
| self._write_all(stack) | |||||
| self._write_first(tokens.ArgumentOpen()) | |||||
| self._write_all(argument) | self._write_all(argument) | ||||
| self._write(tokens.ArgumentClose()) | self._write(tokens.ArgumentClose()) | ||||