Browse Source

Parse *, #; add another test.

tags/v0.3
Ben Kurtovic 11 years ago
parent
commit
7bce2f4e96
2 changed files with 23 additions and 2 deletions
  1. +16
    -2
      mwparserfromhell/parser/tokenizer.py
  2. +7
    -0
      tests/tokenizer/tags_wikimarkup.mwtest

+ 16
- 2
mwparserfromhell/parser/tokenizer.py View File

@@ -629,6 +629,18 @@ class Tokenizer(object):
else: else:
self._emit_all(tag) self._emit_all(tag)


def _parse_list(self):
"""Parse a wiki-style list (``#``, ``*``, ``;``, ``:``)."""
def emit():
self._emit(tokens.TagOpenOpen(wiki_markup=self._read()))
self._emit_text("li")
self._emit(tokens.TagCloseSelfclose())

emit()
while self._read(1) in ("#", "*"):
self._head += 1
emit()

def _parse_hr(self): def _parse_hr(self):
"""Parse a wiki-style horizontal rule (``----``) at the string head.""" """Parse a wiki-style horizontal rule (``----``) at the string head."""
length = 4 length = 4
@@ -793,8 +805,10 @@ class Tokenizer(object):
self._emit_text("<") self._emit_text("<")
elif this == ">" and self._context & contexts.TAG_CLOSE: elif this == ">" and self._context & contexts.TAG_CLOSE:
return self._handle_tag_close_close() return self._handle_tag_close_close()
elif this == next == "-" and self._read(-1) in ("\n", self.START):
if self._read(2) == self._read(3) == "-":
elif self._read(-1) in ("\n", self.START):
if this in ("#", "*"):
self._parse_list()
elif this == next == self._read(2) == self._read(3) == "-":
self._parse_hr() self._parse_hr()
else: else:
self._emit_text("-") self._emit_text("-")


+ 7
- 0
tests/tokenizer/tags_wikimarkup.mwtest View File

@@ -370,6 +370,13 @@ output: [TagOpenOpen(wiki_markup=";"), Text(text="dt"), TagCloseSelfclose(), Tag


--- ---


name: ul_ol_dt_dd_mix
label: an assortment of uls, ols, dds, and dts
input: ";:#*foo\n:#*;foo\n#*;:foo\n*;:#foo"
output: [TagOpenOpen(wiki_markup=";"), Text(text="dt"), TagCloseSelfclose(), TagOpenOpen(wiki_markup=":"), Text(text="dd"), TagCloseSelfclose(), TagOpenOpen(wiki_markup="#"), Text(text="li"), TagCloseSelfclose(), TagOpenOpen(wiki_markup="*"), Text(text="li"), TagCloseSelfclose(), Text(text="foo\n"), TagOpenOpen(wiki_markup=":"), Text(text="dd"), TagCloseSelfclose(), TagOpenOpen(wiki_markup="#"), Text(text="li"), TagCloseSelfclose(), TagOpenOpen(wiki_markup="*"), Text(text="li"), TagCloseSelfclose(), TagOpenOpen(wiki_markup=";"), Text(text="dt"), TagCloseSelfclose(), Text(text="foo\n"), TagOpenOpen(wiki_markup="#"), Text(text="li"), TagCloseSelfclose(), TagOpenOpen(wiki_markup="*"), Text(text="li"), TagCloseSelfclose(), TagOpenOpen(wiki_markup=";"), Text(text="dt"), TagCloseSelfclose(), TagOpenOpen(wiki_markup=":"), Text(text="dd"), TagCloseSelfclose(), Text(text="foo\n"), TagOpenOpen(wiki_markup="*"), Text(text="li"), TagCloseSelfclose(), TagOpenOpen(wiki_markup=";"), Text(text="dt"), TagCloseSelfclose(), TagOpenOpen(wiki_markup=":"), Text(text="dd"), TagCloseSelfclose(), TagOpenOpen(wiki_markup="#"), Text(text="li"), TagCloseSelfclose(), Text(text="foo")]

---

name: hr_text_before name: hr_text_before
label: text before an otherwise-valid hr label: text before an otherwise-valid hr
input: "foo----" input: "foo----"


Loading…
Cancel
Save