A Python parser for MediaWiki wikicode https://mwparserfromhell.readthedocs.io/
25개 이상의 토픽을 선택하실 수 없습니다. Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
 
 
 
 

1363 lines
54 KiB

  1. # -*- coding: utf-8 -*-
  2. #
  3. # Copyright (C) 2012-2014 Ben Kurtovic <ben.kurtovic@gmail.com>
  4. #
  5. # Permission is hereby granted, free of charge, to any person obtaining a copy
  6. # of this software and associated documentation files (the "Software"), to deal
  7. # in the Software without restriction, including without limitation the rights
  8. # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
  9. # copies of the Software, and to permit persons to whom the Software is
  10. # furnished to do so, subject to the following conditions:
  11. #
  12. # The above copyright notice and this permission notice shall be included in
  13. # all copies or substantial portions of the Software.
  14. #
  15. # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16. # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17. # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  18. # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19. # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
  20. # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  21. # SOFTWARE.
  22. from __future__ import unicode_literals
  23. from math import log
  24. import re
  25. from . import contexts, tokens, ParserError
  26. from ..compat import htmlentities, range
  27. from ..definitions import (get_html_tag, is_parsable, is_single,
  28. is_single_only, is_scheme)
  29. __all__ = ["Tokenizer"]
  30. class BadRoute(Exception):
  31. """Raised internally when the current tokenization route is invalid."""
  32. def __init__(self, context=0):
  33. super(BadRoute, self).__init__()
  34. self.context = context
  35. class _TagOpenData(object):
  36. """Stores data about an HTML open tag, like ``<ref name="foo">``."""
  37. CX_NAME = 1 << 0
  38. CX_ATTR_READY = 1 << 1
  39. CX_ATTR_NAME = 1 << 2
  40. CX_ATTR_VALUE = 1 << 3
  41. CX_QUOTED = 1 << 4
  42. CX_NOTE_SPACE = 1 << 5
  43. CX_NOTE_EQUALS = 1 << 6
  44. CX_NOTE_QUOTE = 1 << 7
  45. def __init__(self):
  46. self.context = self.CX_NAME
  47. self.padding_buffer = {"first": "", "before_eq": "", "after_eq": ""}
  48. self.quoter = None
  49. self.reset = 0
  50. class Tokenizer(object):
  51. """Creates a list of tokens from a string of wikicode."""
  52. USES_C = False
  53. START = object()
  54. END = object()
  55. MARKERS = ["{", "}", "[", "]", "<", ">", "|", "=", "&", "'", "#", "*", ";",
  56. ":", "/", "-", "!", "\n", START, END]
  57. MAX_DEPTH = 40
  58. MAX_CYCLES = 100000
  59. regex = re.compile(r"([{}\[\]<>|=&'#*;:/\\\"\-!\n])", flags=re.IGNORECASE)
  60. tag_splitter = re.compile(r"([\s\"\'\\]+)")
  61. def __init__(self):
  62. self._text = None
  63. self._head = 0
  64. self._stacks = []
  65. self._global = 0
  66. self._depth = 0
  67. self._cycles = 0
  68. @property
  69. def _stack(self):
  70. """The current token stack."""
  71. return self._stacks[-1][0]
  72. @property
  73. def _context(self):
  74. """The current token context."""
  75. return self._stacks[-1][1]
  76. @_context.setter
  77. def _context(self, value):
  78. self._stacks[-1][1] = value
  79. @property
  80. def _textbuffer(self):
  81. """The current textbuffer."""
  82. return self._stacks[-1][2]
  83. @_textbuffer.setter
  84. def _textbuffer(self, value):
  85. self._stacks[-1][2] = value
  86. def _push(self, context=0):
  87. """Add a new token stack, context, and textbuffer to the list."""
  88. self._stacks.append([[], context, []])
  89. self._depth += 1
  90. self._cycles += 1
  91. def _push_textbuffer(self):
  92. """Push the textbuffer onto the stack as a Text node and clear it."""
  93. if self._textbuffer:
  94. self._stack.append(tokens.Text(text="".join(self._textbuffer)))
  95. self._textbuffer = []
  96. def _pop(self, keep_context=False):
  97. """Pop the current stack/context/textbuffer, returning the stack.
  98. If *keep_context* is ``True``, then we will replace the underlying
  99. stack's context with the current stack's.
  100. """
  101. self._push_textbuffer()
  102. self._depth -= 1
  103. if keep_context:
  104. context = self._context
  105. stack = self._stacks.pop()[0]
  106. self._context = context
  107. return stack
  108. return self._stacks.pop()[0]
  109. def _can_recurse(self):
  110. """Return whether or not our max recursion depth has been exceeded."""
  111. return self._depth < self.MAX_DEPTH and self._cycles < self.MAX_CYCLES
  112. def _fail_route(self):
  113. """Fail the current tokenization route.
  114. Discards the current stack/context/textbuffer and raises
  115. :exc:`.BadRoute`.
  116. """
  117. context = self._context
  118. self._pop()
  119. raise BadRoute(context)
  120. def _emit(self, token):
  121. """Write a token to the end of the current token stack."""
  122. self._push_textbuffer()
  123. self._stack.append(token)
  124. def _emit_first(self, token):
  125. """Write a token to the beginning of the current token stack."""
  126. self._push_textbuffer()
  127. self._stack.insert(0, token)
  128. def _emit_text(self, text):
  129. """Write text to the current textbuffer."""
  130. self._textbuffer.append(text)
  131. def _emit_all(self, tokenlist):
  132. """Write a series of tokens to the current stack at once."""
  133. if tokenlist and isinstance(tokenlist[0], tokens.Text):
  134. self._emit_text(tokenlist.pop(0).text)
  135. self._push_textbuffer()
  136. self._stack.extend(tokenlist)
  137. def _emit_text_then_stack(self, text):
  138. """Pop the current stack, write *text*, and then write the stack."""
  139. stack = self._pop()
  140. self._emit_text(text)
  141. if stack:
  142. self._emit_all(stack)
  143. self._head -= 1
  144. def _read(self, delta=0, wrap=False, strict=False):
  145. """Read the value at a relative point in the wikicode.
  146. The value is read from :attr:`self._head <_head>` plus the value of
  147. *delta* (which can be negative). If *wrap* is ``False``, we will not
  148. allow attempts to read from the end of the string if ``self._head +
  149. delta`` is negative. If *strict* is ``True``, the route will be failed
  150. (with :meth:`_fail_route`) if we try to read from past the end of the
  151. string; otherwise, :attr:`self.END <END>` is returned. If we try to
  152. read from before the start of the string, :attr:`self.START <START>` is
  153. returned.
  154. """
  155. index = self._head + delta
  156. if index < 0 and (not wrap or abs(index) > len(self._text)):
  157. return self.START
  158. try:
  159. return self._text[index]
  160. except IndexError:
  161. if strict:
  162. self._fail_route()
  163. return self.END
  164. def _parse_template(self):
  165. """Parse a template at the head of the wikicode string."""
  166. reset = self._head
  167. try:
  168. template = self._parse(contexts.TEMPLATE_NAME)
  169. except BadRoute:
  170. self._head = reset
  171. raise
  172. self._emit_first(tokens.TemplateOpen())
  173. self._emit_all(template)
  174. self._emit(tokens.TemplateClose())
  175. def _parse_argument(self):
  176. """Parse an argument at the head of the wikicode string."""
  177. reset = self._head
  178. try:
  179. argument = self._parse(contexts.ARGUMENT_NAME)
  180. except BadRoute:
  181. self._head = reset
  182. raise
  183. self._emit_first(tokens.ArgumentOpen())
  184. self._emit_all(argument)
  185. self._emit(tokens.ArgumentClose())
  186. def _parse_template_or_argument(self):
  187. """Parse a template or argument at the head of the wikicode string."""
  188. self._head += 2
  189. braces = 2
  190. while self._read() == "{":
  191. self._head += 1
  192. braces += 1
  193. self._push()
  194. while braces:
  195. if braces == 1:
  196. return self._emit_text_then_stack("{")
  197. if braces == 2:
  198. try:
  199. self._parse_template()
  200. except BadRoute:
  201. return self._emit_text_then_stack("{{")
  202. break
  203. try:
  204. self._parse_argument()
  205. braces -= 3
  206. except BadRoute:
  207. try:
  208. self._parse_template()
  209. braces -= 2
  210. except BadRoute:
  211. return self._emit_text_then_stack("{" * braces)
  212. if braces:
  213. self._head += 1
  214. self._emit_all(self._pop())
  215. if self._context & contexts.FAIL_NEXT:
  216. self._context ^= contexts.FAIL_NEXT
  217. def _handle_template_param(self):
  218. """Handle a template parameter at the head of the string."""
  219. if self._context & contexts.TEMPLATE_NAME:
  220. self._context ^= contexts.TEMPLATE_NAME
  221. elif self._context & contexts.TEMPLATE_PARAM_VALUE:
  222. self._context ^= contexts.TEMPLATE_PARAM_VALUE
  223. else:
  224. self._emit_all(self._pop(keep_context=True))
  225. self._context |= contexts.TEMPLATE_PARAM_KEY
  226. self._emit(tokens.TemplateParamSeparator())
  227. self._push(self._context)
  228. def _handle_template_param_value(self):
  229. """Handle a template parameter's value at the head of the string."""
  230. self._emit_all(self._pop(keep_context=True))
  231. self._context ^= contexts.TEMPLATE_PARAM_KEY
  232. self._context |= contexts.TEMPLATE_PARAM_VALUE
  233. self._emit(tokens.TemplateParamEquals())
  234. def _handle_template_end(self):
  235. """Handle the end of a template at the head of the string."""
  236. if self._context & contexts.TEMPLATE_PARAM_KEY:
  237. self._emit_all(self._pop(keep_context=True))
  238. self._head += 1
  239. return self._pop()
  240. def _handle_argument_separator(self):
  241. """Handle the separator between an argument's name and default."""
  242. self._context ^= contexts.ARGUMENT_NAME
  243. self._context |= contexts.ARGUMENT_DEFAULT
  244. self._emit(tokens.ArgumentSeparator())
  245. def _handle_argument_end(self):
  246. """Handle the end of an argument at the head of the string."""
  247. self._head += 2
  248. return self._pop()
  249. def _parse_wikilink(self):
  250. """Parse an internal wikilink at the head of the wikicode string."""
  251. self._head += 2
  252. reset = self._head - 1
  253. try:
  254. wikilink = self._parse(contexts.WIKILINK_TITLE)
  255. except BadRoute:
  256. self._head = reset
  257. self._emit_text("[[")
  258. else:
  259. self._emit(tokens.WikilinkOpen())
  260. self._emit_all(wikilink)
  261. self._emit(tokens.WikilinkClose())
  262. def _handle_wikilink_separator(self):
  263. """Handle the separator between a wikilink's title and its text."""
  264. self._context ^= contexts.WIKILINK_TITLE
  265. self._context |= contexts.WIKILINK_TEXT
  266. self._emit(tokens.WikilinkSeparator())
  267. def _handle_wikilink_end(self):
  268. """Handle the end of a wikilink at the head of the string."""
  269. self._head += 1
  270. return self._pop()
  271. def _parse_bracketed_uri_scheme(self):
  272. """Parse the URI scheme of a bracket-enclosed external link."""
  273. self._push(contexts.EXT_LINK_URI)
  274. if self._read() == self._read(1) == "/":
  275. self._emit_text("//")
  276. self._head += 2
  277. else:
  278. valid = "abcdefghijklmnopqrstuvwxyz0123456789+.-"
  279. all_valid = lambda: all(char in valid for char in self._read())
  280. scheme = ""
  281. while self._read() is not self.END and all_valid():
  282. scheme += self._read()
  283. self._emit_text(self._read())
  284. self._head += 1
  285. if self._read() != ":":
  286. self._fail_route()
  287. self._emit_text(":")
  288. self._head += 1
  289. slashes = self._read() == self._read(1) == "/"
  290. if slashes:
  291. self._emit_text("//")
  292. self._head += 2
  293. if not is_scheme(scheme, slashes):
  294. self._fail_route()
  295. def _parse_free_uri_scheme(self):
  296. """Parse the URI scheme of a free (no brackets) external link."""
  297. valid = "abcdefghijklmnopqrstuvwxyz0123456789+.-"
  298. scheme = []
  299. try:
  300. # We have to backtrack through the textbuffer looking for our
  301. # scheme since it was just parsed as text:
  302. for chunk in reversed(self._textbuffer):
  303. for char in reversed(chunk):
  304. if char.isspace() or char in self.MARKERS:
  305. raise StopIteration()
  306. if char not in valid:
  307. raise BadRoute()
  308. scheme.append(char)
  309. except StopIteration:
  310. pass
  311. scheme = "".join(reversed(scheme))
  312. slashes = self._read() == self._read(1) == "/"
  313. if not is_scheme(scheme, slashes):
  314. raise BadRoute()
  315. self._push(self._context | contexts.EXT_LINK_URI)
  316. self._emit_text(scheme)
  317. self._emit_text(":")
  318. if slashes:
  319. self._emit_text("//")
  320. self._head += 2
  321. def _handle_free_link_text(self, punct, tail, this):
  322. """Handle text in a free ext link, including trailing punctuation."""
  323. if "(" in this and ")" in punct:
  324. punct = punct[:-1] # ')' is not longer valid punctuation
  325. if this.endswith(punct):
  326. for i in range(len(this) - 1, 0, -1):
  327. if this[i - 1] not in punct:
  328. break
  329. else:
  330. i = 0
  331. stripped = this[:i]
  332. if stripped and tail:
  333. self._emit_text(tail)
  334. tail = ""
  335. tail += this[i:]
  336. this = stripped
  337. elif tail:
  338. self._emit_text(tail)
  339. tail = ""
  340. self._emit_text(this)
  341. return punct, tail
  342. def _is_free_link_end(self, this, next):
  343. """Return whether the current head is the end of a free link."""
  344. # Built from _parse()'s end sentinels:
  345. after, ctx = self._read(2), self._context
  346. equal_sign_contexts = contexts.TEMPLATE_PARAM_KEY | contexts.HEADING
  347. return (this in (self.END, "\n", "[", "]", "<", ">") or
  348. this == next == "'" or
  349. (this == "|" and ctx & contexts.TEMPLATE) or
  350. (this == "=" and ctx & equal_sign_contexts) or
  351. (this == next == "}" and ctx & contexts.TEMPLATE) or
  352. (this == next == after == "}" and ctx & contexts.ARGUMENT))
  353. def _really_parse_external_link(self, brackets):
  354. """Really parse an external link."""
  355. if brackets:
  356. self._parse_bracketed_uri_scheme()
  357. invalid = ("\n", " ", "]")
  358. else:
  359. self._parse_free_uri_scheme()
  360. invalid = ("\n", " ", "[", "]")
  361. punct = tuple(",;\.:!?)")
  362. if self._read() is self.END or self._read()[0] in invalid:
  363. self._fail_route()
  364. tail = ""
  365. while True:
  366. this, next = self._read(), self._read(1)
  367. if this == "&":
  368. if tail:
  369. self._emit_text(tail)
  370. tail = ""
  371. self._parse_entity()
  372. elif (this == "<" and next == "!" and self._read(2) ==
  373. self._read(3) == "-"):
  374. if tail:
  375. self._emit_text(tail)
  376. tail = ""
  377. self._parse_comment()
  378. elif not brackets and self._is_free_link_end(this, next):
  379. return self._pop(), tail, -1
  380. elif this is self.END or this == "\n":
  381. self._fail_route()
  382. elif this == next == "{" and self._can_recurse():
  383. if tail:
  384. self._emit_text(tail)
  385. tail = ""
  386. self._parse_template_or_argument()
  387. elif this == "]":
  388. return self._pop(), tail, 0
  389. elif " " in this:
  390. before, after = this.split(" ", 1)
  391. if brackets:
  392. self._emit_text(before)
  393. self._emit(tokens.ExternalLinkSeparator())
  394. if after:
  395. self._emit_text(after)
  396. self._context ^= contexts.EXT_LINK_URI
  397. self._context |= contexts.EXT_LINK_TITLE
  398. self._head += 1
  399. return self._parse(push=False), None, 0
  400. punct, tail = self._handle_free_link_text(punct, tail, before)
  401. return self._pop(), tail + " " + after, 0
  402. elif not brackets:
  403. punct, tail = self._handle_free_link_text(punct, tail, this)
  404. else:
  405. self._emit_text(this)
  406. self._head += 1
  407. def _remove_uri_scheme_from_textbuffer(self, scheme):
  408. """Remove the URI scheme of a new external link from the textbuffer."""
  409. length = len(scheme)
  410. while length:
  411. if length < len(self._textbuffer[-1]):
  412. self._textbuffer[-1] = self._textbuffer[-1][:-length]
  413. break
  414. length -= len(self._textbuffer[-1])
  415. self._textbuffer.pop()
  416. def _parse_external_link(self, brackets):
  417. """Parse an external link at the head of the wikicode string."""
  418. reset = self._head
  419. self._head += 1
  420. try:
  421. bad_context = self._context & contexts.NO_EXT_LINKS
  422. if bad_context or not self._can_recurse():
  423. raise BadRoute()
  424. link, extra, delta = self._really_parse_external_link(brackets)
  425. except BadRoute:
  426. self._head = reset
  427. if not brackets and self._context & contexts.DL_TERM:
  428. self._handle_dl_term()
  429. else:
  430. self._emit_text(self._read())
  431. else:
  432. if not brackets:
  433. scheme = link[0].text.split(":", 1)[0]
  434. self._remove_uri_scheme_from_textbuffer(scheme)
  435. self._emit(tokens.ExternalLinkOpen(brackets=brackets))
  436. self._emit_all(link)
  437. self._emit(tokens.ExternalLinkClose())
  438. self._head += delta
  439. if extra:
  440. self._emit_text(extra)
  441. def _parse_heading(self):
  442. """Parse a section heading at the head of the wikicode string."""
  443. self._global |= contexts.GL_HEADING
  444. reset = self._head
  445. self._head += 1
  446. best = 1
  447. while self._read() == "=":
  448. best += 1
  449. self._head += 1
  450. context = contexts.HEADING_LEVEL_1 << min(best - 1, 5)
  451. try:
  452. title, level = self._parse(context)
  453. except BadRoute:
  454. self._head = reset + best - 1
  455. self._emit_text("=" * best)
  456. else:
  457. self._emit(tokens.HeadingStart(level=level))
  458. if level < best:
  459. self._emit_text("=" * (best - level))
  460. self._emit_all(title)
  461. self._emit(tokens.HeadingEnd())
  462. finally:
  463. self._global ^= contexts.GL_HEADING
  464. def _handle_heading_end(self):
  465. """Handle the end of a section heading at the head of the string."""
  466. reset = self._head
  467. self._head += 1
  468. best = 1
  469. while self._read() == "=":
  470. best += 1
  471. self._head += 1
  472. current = int(log(self._context / contexts.HEADING_LEVEL_1, 2)) + 1
  473. level = min(current, min(best, 6))
  474. try: # Try to check for a heading closure after this one
  475. after, after_level = self._parse(self._context)
  476. except BadRoute:
  477. if level < best:
  478. self._emit_text("=" * (best - level))
  479. self._head = reset + best - 1
  480. return self._pop(), level
  481. else: # Found another closure
  482. self._emit_text("=" * best)
  483. self._emit_all(after)
  484. return self._pop(), after_level
  485. def _really_parse_entity(self):
  486. """Actually parse an HTML entity and ensure that it is valid."""
  487. self._emit(tokens.HTMLEntityStart())
  488. self._head += 1
  489. this = self._read(strict=True)
  490. if this == "#":
  491. numeric = True
  492. self._emit(tokens.HTMLEntityNumeric())
  493. self._head += 1
  494. this = self._read(strict=True)
  495. if this[0].lower() == "x":
  496. hexadecimal = True
  497. self._emit(tokens.HTMLEntityHex(char=this[0]))
  498. this = this[1:]
  499. if not this:
  500. self._fail_route()
  501. else:
  502. hexadecimal = False
  503. else:
  504. numeric = hexadecimal = False
  505. valid = "0123456789abcdefABCDEF" if hexadecimal else "0123456789"
  506. if not numeric and not hexadecimal:
  507. valid += "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"
  508. if not all([char in valid for char in this]):
  509. self._fail_route()
  510. self._head += 1
  511. if self._read() != ";":
  512. self._fail_route()
  513. if numeric:
  514. test = int(this, 16) if hexadecimal else int(this)
  515. if test < 1 or test > 0x10FFFF:
  516. self._fail_route()
  517. else:
  518. if this not in htmlentities.entitydefs:
  519. self._fail_route()
  520. self._emit(tokens.Text(text=this))
  521. self._emit(tokens.HTMLEntityEnd())
  522. def _parse_entity(self):
  523. """Parse an HTML entity at the head of the wikicode string."""
  524. reset = self._head
  525. self._push()
  526. try:
  527. self._really_parse_entity()
  528. except BadRoute:
  529. self._head = reset
  530. self._emit_text(self._read())
  531. else:
  532. self._emit_all(self._pop())
  533. def _parse_comment(self):
  534. """Parse an HTML comment at the head of the wikicode string."""
  535. self._head += 4
  536. reset = self._head - 1
  537. self._push()
  538. while True:
  539. this = self._read()
  540. if this == self.END:
  541. self._pop()
  542. self._head = reset
  543. self._emit_text("<!--")
  544. return
  545. if this == self._read(1) == "-" and self._read(2) == ">":
  546. self._emit_first(tokens.CommentStart())
  547. self._emit(tokens.CommentEnd())
  548. self._emit_all(self._pop())
  549. self._head += 2
  550. if self._context & contexts.FAIL_NEXT:
  551. # _verify_safe() sets this flag while parsing a template
  552. # name when it encounters what might be a comment -- we
  553. # must unset it to let _verify_safe() know it was correct:
  554. self._context ^= contexts.FAIL_NEXT
  555. return
  556. self._emit_text(this)
  557. self._head += 1
  558. def _push_tag_buffer(self, data):
  559. """Write a pending tag attribute from *data* to the stack."""
  560. if data.context & data.CX_QUOTED:
  561. self._emit_first(tokens.TagAttrQuote(char=data.quoter))
  562. self._emit_all(self._pop())
  563. buf = data.padding_buffer
  564. self._emit_first(tokens.TagAttrStart(pad_first=buf["first"],
  565. pad_before_eq=buf["before_eq"], pad_after_eq=buf["after_eq"]))
  566. self._emit_all(self._pop())
  567. for key in data.padding_buffer:
  568. data.padding_buffer[key] = ""
  569. def _handle_tag_space(self, data, text):
  570. """Handle whitespace (*text*) inside of an HTML open tag."""
  571. ctx = data.context
  572. end_of_value = ctx & data.CX_ATTR_VALUE and not ctx & (data.CX_QUOTED | data.CX_NOTE_QUOTE)
  573. if end_of_value or (ctx & data.CX_QUOTED and ctx & data.CX_NOTE_SPACE):
  574. self._push_tag_buffer(data)
  575. data.context = data.CX_ATTR_READY
  576. elif ctx & data.CX_NOTE_SPACE:
  577. data.context = data.CX_ATTR_READY
  578. elif ctx & data.CX_ATTR_NAME:
  579. data.context |= data.CX_NOTE_EQUALS
  580. data.padding_buffer["before_eq"] += text
  581. if ctx & data.CX_QUOTED and not ctx & data.CX_NOTE_SPACE:
  582. self._emit_text(text)
  583. elif data.context & data.CX_ATTR_READY:
  584. data.padding_buffer["first"] += text
  585. elif data.context & data.CX_ATTR_VALUE:
  586. data.padding_buffer["after_eq"] += text
  587. def _handle_tag_text(self, text):
  588. """Handle regular *text* inside of an HTML open tag."""
  589. next = self._read(1)
  590. if not self._can_recurse() or text not in self.MARKERS:
  591. self._emit_text(text)
  592. elif text == next == "{":
  593. self._parse_template_or_argument()
  594. elif text == next == "[":
  595. self._parse_wikilink()
  596. elif text == "<":
  597. self._parse_tag()
  598. else:
  599. self._emit_text(text)
  600. def _handle_tag_data(self, data, text):
  601. """Handle all sorts of *text* data inside of an HTML open tag."""
  602. for chunk in self.tag_splitter.split(text):
  603. if not chunk:
  604. continue
  605. if data.context & data.CX_NAME:
  606. if chunk in self.MARKERS or chunk.isspace():
  607. self._fail_route() # Tags must start with text, not spaces
  608. data.context = data.CX_NOTE_SPACE
  609. elif chunk.isspace():
  610. self._handle_tag_space(data, chunk)
  611. continue
  612. elif data.context & data.CX_NOTE_SPACE:
  613. if data.context & data.CX_QUOTED:
  614. data.context = data.CX_ATTR_VALUE
  615. self._pop()
  616. self._head = data.reset - 1 # Will be auto-incremented
  617. return # Break early
  618. self._fail_route()
  619. elif data.context & data.CX_ATTR_READY:
  620. data.context = data.CX_ATTR_NAME
  621. self._push(contexts.TAG_ATTR)
  622. elif data.context & data.CX_ATTR_NAME:
  623. if chunk == "=":
  624. data.context = data.CX_ATTR_VALUE | data.CX_NOTE_QUOTE
  625. self._emit(tokens.TagAttrEquals())
  626. continue
  627. if data.context & data.CX_NOTE_EQUALS:
  628. self._push_tag_buffer(data)
  629. data.context = data.CX_ATTR_NAME
  630. self._push(contexts.TAG_ATTR)
  631. else: # data.context & data.CX_ATTR_VALUE assured
  632. escaped = self._read(-1) == "\\" and self._read(-2) != "\\"
  633. if data.context & data.CX_NOTE_QUOTE:
  634. data.context ^= data.CX_NOTE_QUOTE
  635. if chunk in "'\"" and not escaped:
  636. data.context |= data.CX_QUOTED
  637. data.quoter = chunk
  638. data.reset = self._head
  639. self._push(self._context)
  640. continue
  641. elif data.context & data.CX_QUOTED:
  642. if chunk == data.quoter and not escaped:
  643. data.context |= data.CX_NOTE_SPACE
  644. continue
  645. self._handle_tag_text(chunk)
  646. def _handle_tag_close_open(self, data, token):
  647. """Handle the closing of a open tag (``<foo>``)."""
  648. if data.context & (data.CX_ATTR_NAME | data.CX_ATTR_VALUE):
  649. self._push_tag_buffer(data)
  650. self._emit(token(padding=data.padding_buffer["first"]))
  651. self._head += 1
  652. def _handle_tag_open_close(self):
  653. """Handle the opening of a closing tag (``</foo>``)."""
  654. self._emit(tokens.TagOpenClose())
  655. self._push(contexts.TAG_CLOSE)
  656. self._head += 1
  657. def _handle_tag_close_close(self):
  658. """Handle the ending of a closing tag (``</foo>``)."""
  659. strip = lambda tok: tok.text.rstrip().lower()
  660. closing = self._pop()
  661. if len(closing) != 1 or (not isinstance(closing[0], tokens.Text) or
  662. strip(closing[0]) != strip(self._stack[1])):
  663. self._fail_route()
  664. self._emit_all(closing)
  665. self._emit(tokens.TagCloseClose())
  666. return self._pop()
  667. def _handle_blacklisted_tag(self):
  668. """Handle the body of an HTML tag that is parser-blacklisted."""
  669. while True:
  670. this, next = self._read(), self._read(1)
  671. if this is self.END:
  672. self._fail_route()
  673. elif this == "<" and next == "/":
  674. self._handle_tag_open_close()
  675. self._head += 1
  676. return self._parse(push=False)
  677. elif this == "&":
  678. self._parse_entity()
  679. else:
  680. self._emit_text(this)
  681. self._head += 1
  682. def _handle_single_only_tag_end(self):
  683. """Handle the end of an implicitly closing single-only HTML tag."""
  684. padding = self._stack.pop().padding
  685. self._emit(tokens.TagCloseSelfclose(padding=padding, implicit=True))
  686. self._head -= 1 # Offset displacement done by _handle_tag_close_open
  687. return self._pop()
  688. def _handle_single_tag_end(self):
  689. """Handle the stream end when inside a single-supporting HTML tag."""
  690. stack = self._stack
  691. # We need to find the index of the TagCloseOpen token corresponding to
  692. # the TagOpenOpen token located at index 0:
  693. depth = 1
  694. for index, token in enumerate(stack[2:], 2):
  695. if isinstance(token, tokens.TagOpenOpen):
  696. depth += 1
  697. elif isinstance(token, tokens.TagCloseOpen):
  698. depth -= 1
  699. if depth == 0:
  700. break
  701. else: # pragma: no cover (untestable/exceptional case)
  702. raise ParserError("_handle_single_tag_end() missed a TagCloseOpen")
  703. padding = stack[index].padding
  704. stack[index] = tokens.TagCloseSelfclose(padding=padding, implicit=True)
  705. return self._pop()
  706. def _really_parse_tag(self):
  707. """Actually parse an HTML tag, starting with the open (``<foo>``)."""
  708. data = _TagOpenData()
  709. self._push(contexts.TAG_OPEN)
  710. self._emit(tokens.TagOpenOpen())
  711. while True:
  712. this, next = self._read(), self._read(1)
  713. can_exit = (not data.context & (data.CX_QUOTED | data.CX_NAME) or
  714. data.context & data.CX_NOTE_SPACE)
  715. if this is self.END:
  716. if self._context & contexts.TAG_ATTR:
  717. if data.context & data.CX_QUOTED:
  718. # Unclosed attribute quote: reset, don't die
  719. data.context = data.CX_ATTR_VALUE
  720. self._pop()
  721. self._head = data.reset
  722. continue
  723. self._pop()
  724. self._fail_route()
  725. elif this == ">" and can_exit:
  726. self._handle_tag_close_open(data, tokens.TagCloseOpen)
  727. self._context = contexts.TAG_BODY
  728. if is_single_only(self._stack[1].text):
  729. return self._handle_single_only_tag_end()
  730. if is_parsable(self._stack[1].text):
  731. return self._parse(push=False)
  732. return self._handle_blacklisted_tag()
  733. elif this == "/" and next == ">" and can_exit:
  734. self._handle_tag_close_open(data, tokens.TagCloseSelfclose)
  735. return self._pop()
  736. else:
  737. self._handle_tag_data(data, this)
  738. self._head += 1
  739. def _handle_invalid_tag_start(self):
  740. """Handle the (possible) start of an implicitly closing single tag."""
  741. reset = self._head + 1
  742. self._head += 2
  743. try:
  744. if not is_single_only(self.tag_splitter.split(self._read())[0]):
  745. raise BadRoute()
  746. tag = self._really_parse_tag()
  747. except BadRoute:
  748. self._head = reset
  749. self._emit_text("</")
  750. else:
  751. tag[0].invalid = True # Set flag of TagOpenOpen
  752. self._emit_all(tag)
  753. def _parse_tag(self):
  754. """Parse an HTML tag at the head of the wikicode string."""
  755. reset = self._head
  756. self._head += 1
  757. try:
  758. tag = self._really_parse_tag()
  759. except BadRoute:
  760. self._head = reset
  761. self._emit_text("<")
  762. else:
  763. self._emit_all(tag)
  764. def _emit_style_tag(self, tag, markup, body):
  765. """Write the body of a tag and the tokens that should surround it."""
  766. self._emit(tokens.TagOpenOpen(wiki_markup=markup))
  767. self._emit_text(tag)
  768. self._emit(tokens.TagCloseOpen())
  769. self._emit_all(body)
  770. self._emit(tokens.TagOpenClose())
  771. self._emit_text(tag)
  772. self._emit(tokens.TagCloseClose())
  773. def _parse_italics(self):
  774. """Parse wiki-style italics."""
  775. reset = self._head
  776. try:
  777. stack = self._parse(contexts.STYLE_ITALICS)
  778. except BadRoute as route:
  779. self._head = reset
  780. if route.context & contexts.STYLE_PASS_AGAIN:
  781. new_ctx = contexts.STYLE_ITALICS | contexts.STYLE_SECOND_PASS
  782. stack = self._parse(new_ctx)
  783. else:
  784. return self._emit_text("''")
  785. self._emit_style_tag("i", "''", stack)
  786. def _parse_bold(self):
  787. """Parse wiki-style bold."""
  788. reset = self._head
  789. try:
  790. stack = self._parse(contexts.STYLE_BOLD)
  791. except BadRoute:
  792. self._head = reset
  793. if self._context & contexts.STYLE_SECOND_PASS:
  794. self._emit_text("'")
  795. return True
  796. elif self._context & contexts.STYLE_ITALICS:
  797. self._context |= contexts.STYLE_PASS_AGAIN
  798. self._emit_text("'''")
  799. else:
  800. self._emit_text("'")
  801. self._parse_italics()
  802. else:
  803. self._emit_style_tag("b", "'''", stack)
  804. def _parse_italics_and_bold(self):
  805. """Parse wiki-style italics and bold together (i.e., five ticks)."""
  806. reset = self._head
  807. try:
  808. stack = self._parse(contexts.STYLE_BOLD)
  809. except BadRoute:
  810. self._head = reset
  811. try:
  812. stack = self._parse(contexts.STYLE_ITALICS)
  813. except BadRoute:
  814. self._head = reset
  815. self._emit_text("'''''")
  816. else:
  817. reset = self._head
  818. try:
  819. stack2 = self._parse(contexts.STYLE_BOLD)
  820. except BadRoute:
  821. self._head = reset
  822. self._emit_text("'''")
  823. self._emit_style_tag("i", "''", stack)
  824. else:
  825. self._push()
  826. self._emit_style_tag("i", "''", stack)
  827. self._emit_all(stack2)
  828. self._emit_style_tag("b", "'''", self._pop())
  829. else:
  830. reset = self._head
  831. try:
  832. stack2 = self._parse(contexts.STYLE_ITALICS)
  833. except BadRoute:
  834. self._head = reset
  835. self._emit_text("''")
  836. self._emit_style_tag("b", "'''", stack)
  837. else:
  838. self._push()
  839. self._emit_style_tag("b", "'''", stack)
  840. self._emit_all(stack2)
  841. self._emit_style_tag("i", "''", self._pop())
  842. def _parse_style(self):
  843. """Parse wiki-style formatting (``''``/``'''`` for italics/bold)."""
  844. self._head += 2
  845. ticks = 2
  846. while self._read() == "'":
  847. self._head += 1
  848. ticks += 1
  849. italics = self._context & contexts.STYLE_ITALICS
  850. bold = self._context & contexts.STYLE_BOLD
  851. if ticks > 5:
  852. self._emit_text("'" * (ticks - 5))
  853. ticks = 5
  854. elif ticks == 4:
  855. self._emit_text("'")
  856. ticks = 3
  857. if (italics and ticks in (2, 5)) or (bold and ticks in (3, 5)):
  858. if ticks == 5:
  859. self._head -= 3 if italics else 2
  860. return self._pop()
  861. elif not self._can_recurse():
  862. if ticks == 3:
  863. if self._context & contexts.STYLE_SECOND_PASS:
  864. self._emit_text("'")
  865. return self._pop()
  866. if self._context & contexts.STYLE_ITALICS:
  867. self._context |= contexts.STYLE_PASS_AGAIN
  868. self._emit_text("'" * ticks)
  869. elif ticks == 2:
  870. self._parse_italics()
  871. elif ticks == 3:
  872. if self._parse_bold():
  873. return self._pop()
  874. else: # ticks == 5
  875. self._parse_italics_and_bold()
  876. self._head -= 1
  877. def _handle_list_marker(self):
  878. """Handle a list marker at the head (``#``, ``*``, ``;``, ``:``)."""
  879. markup = self._read()
  880. if markup == ";":
  881. self._context |= contexts.DL_TERM
  882. self._emit(tokens.TagOpenOpen(wiki_markup=markup))
  883. self._emit_text(get_html_tag(markup))
  884. self._emit(tokens.TagCloseSelfclose())
  885. def _handle_list(self):
  886. """Handle a wiki-style list (``#``, ``*``, ``;``, ``:``)."""
  887. self._handle_list_marker()
  888. while self._read(1) in ("#", "*", ";", ":"):
  889. self._head += 1
  890. self._handle_list_marker()
  891. def _handle_hr(self):
  892. """Handle a wiki-style horizontal rule (``----``) in the string."""
  893. length = 4
  894. self._head += 3
  895. while self._read(1) == "-":
  896. length += 1
  897. self._head += 1
  898. self._emit(tokens.TagOpenOpen(wiki_markup="-" * length))
  899. self._emit_text("hr")
  900. self._emit(tokens.TagCloseSelfclose())
  901. def _handle_dl_term(self):
  902. """Handle the term in a description list (``foo`` in ``;foo:bar``)."""
  903. self._context ^= contexts.DL_TERM
  904. if self._read() == ":":
  905. self._handle_list_marker()
  906. else:
  907. self._emit_text("\n")
  908. def _handle_end(self):
  909. """Handle the end of the stream of wikitext."""
  910. if self._context & contexts.FAIL:
  911. if self._context & contexts.TAG_BODY:
  912. if is_single(self._stack[1].text):
  913. return self._handle_single_tag_end()
  914. if self._context & contexts.DOUBLE:
  915. self._pop()
  916. self._fail_route()
  917. return self._pop()
  918. def _parse_as_table_style(self, end_token, break_on_table_end=False):
  919. """Parse until ``end_token`` as style attributes for a table."""
  920. data = _TagOpenData()
  921. data.context = _TagOpenData.CX_ATTR_READY
  922. while True:
  923. this, next = self._read(), self._read(1)
  924. table_end = break_on_table_end and this == "|" and next == "}"
  925. can_exit = (not data.context & data.CX_QUOTED or
  926. data.context & data.CX_NOTE_SPACE)
  927. if (this == end_token and can_exit) or table_end:
  928. if data.context & (data.CX_ATTR_NAME | data.CX_ATTR_VALUE):
  929. self._push_tag_buffer(data)
  930. if this.isspace():
  931. data.padding_buffer["first"] += this
  932. return data.padding_buffer["first"]
  933. elif this is self.END or table_end or this == end_token:
  934. if self._context & contexts.TAG_ATTR:
  935. if data.context & data.CX_QUOTED:
  936. # Unclosed attribute quote: reset, don't die
  937. data.context = data.CX_ATTR_VALUE
  938. self._pop()
  939. self._head = data.reset
  940. continue
  941. self._pop()
  942. self._fail_route()
  943. else:
  944. self._handle_tag_data(data, this)
  945. self._head += 1
  946. def _handle_table_start(self):
  947. """Handle the start of a table."""
  948. self._head += 2
  949. reset = self._head
  950. style, table = None, None
  951. try:
  952. self._push(contexts.TABLE_OPEN)
  953. padding = self._parse_as_table_style("\n", break_on_table_end=True)
  954. style = self._pop()
  955. # continue to parse if it is NOT an inline table
  956. if "\n" in padding:
  957. self._head += 1
  958. table = self._parse(contexts.TABLE_OPEN)
  959. else:
  960. # close tag
  961. self._head += 2
  962. except BadRoute:
  963. # offset displacement done by _parse()
  964. self._head = reset - 1
  965. self._emit_text("{|")
  966. else:
  967. self._emit(tokens.TagOpenOpen(wiki_markup="{|"))
  968. self._emit_text("table")
  969. if style:
  970. self._emit_all(style)
  971. self._emit(tokens.TagCloseOpen(padding=padding))
  972. if table:
  973. self._emit_all(table)
  974. self._emit(tokens.TagOpenClose(wiki_markup="|}"))
  975. self._emit_text("table")
  976. self._emit(tokens.TagCloseClose())
  977. # offset displacement done by _parse()
  978. self._head -= 1
  979. def _handle_table_end(self):
  980. """Return the stack in order to handle the table end."""
  981. self._head += 2
  982. return self._pop()
  983. def _handle_table_row(self):
  984. """Parse as style until end of the line, then continue."""
  985. reset = self._head
  986. self._head += 2
  987. style, padding = None, ""
  988. # If we can't recurse, still tokenize tag but parse style attrs as text
  989. if self._can_recurse():
  990. try:
  991. self._push(contexts.TABLE_OPEN)
  992. padding = self._parse_as_table_style("\n")
  993. style = self._pop()
  994. except BadRoute:
  995. self._head = reset
  996. raise
  997. self._emit(tokens.TagOpenOpen(wiki_markup="|-"))
  998. self._emit_text("tr")
  999. if style:
  1000. self._emit_all(style)
  1001. self._emit(tokens.TagCloseSelfclose(padding=padding))
  1002. def _handle_table_cell(self, markup, tag, line_context):
  1003. """Parse as normal syntax unless we hit a style marker, then parse style
  1004. as HTML attributes and the remainder as normal syntax."""
  1005. if not self._can_recurse():
  1006. self._emit_text(markup)
  1007. self._head += len(markup) - 1
  1008. return
  1009. old_context = self._context
  1010. reset = self._head
  1011. self._head += len(markup)
  1012. reset_for_style, padding = False, ""
  1013. try:
  1014. cell = self._parse(contexts.TABLE_OPEN | contexts.TABLE_CELL_OPEN | line_context | contexts.TABLE_CELL_STYLE)
  1015. cell_context = self._context
  1016. self._context = old_context
  1017. reset_for_style = cell_context & contexts.TABLE_CELL_STYLE
  1018. except BadRoute:
  1019. self._head = reset
  1020. raise
  1021. if reset_for_style:
  1022. self._head = reset + len(markup)
  1023. try:
  1024. self._push(contexts.TABLE_OPEN | contexts.TABLE_CELL_OPEN | line_context)
  1025. padding = self._parse_as_table_style("|")
  1026. style = self._pop()
  1027. # Don't parse the style separator
  1028. self._head += 1
  1029. cell = self._parse(contexts.TABLE_OPEN | contexts.TABLE_CELL_OPEN | line_context)
  1030. cell_context = self._context
  1031. self._context = old_context
  1032. except BadRoute:
  1033. self._head = reset
  1034. raise
  1035. self._emit(tokens.TagOpenOpen(wiki_markup=markup))
  1036. self._emit_text(tag)
  1037. if reset_for_style:
  1038. self._emit_all(style)
  1039. self._emit(tokens.TagCloseSelfclose(wiki_markup="|", padding=padding))
  1040. else:
  1041. self._emit(tokens.TagCloseSelfclose(padding=padding))
  1042. self._emit_all(cell)
  1043. # keep header/cell line contexts
  1044. self._context |= cell_context & (contexts.TABLE_TH_LINE | contexts.TABLE_TD_LINE)
  1045. # offset displacement done by parse()
  1046. self._head -= 1
  1047. def _handle_table_cell_end(self, reset_for_style=False):
  1048. """Returns the current context, with the TABLE_CELL_STYLE flag set if
  1049. it is necessary to reset and parse style attributes."""
  1050. if reset_for_style:
  1051. self._context |= contexts.TABLE_CELL_STYLE
  1052. else:
  1053. self._context &= ~contexts.TABLE_CELL_STYLE
  1054. return self._pop(keep_context=True)
  1055. def _verify_safe(self, this):
  1056. """Make sure we are not trying to write an invalid character."""
  1057. context = self._context
  1058. if context & contexts.FAIL_NEXT:
  1059. return False
  1060. if context & contexts.WIKILINK_TITLE:
  1061. if this == "]" or this == "{":
  1062. self._context |= contexts.FAIL_NEXT
  1063. elif this == "\n" or this == "[" or this == "}":
  1064. return False
  1065. return True
  1066. elif context & contexts.EXT_LINK_TITLE:
  1067. return this != "\n"
  1068. elif context & contexts.TEMPLATE_NAME:
  1069. if this == "{" or this == "}" or this == "[":
  1070. self._context |= contexts.FAIL_NEXT
  1071. return True
  1072. if this == "]":
  1073. return False
  1074. if this == "|":
  1075. return True
  1076. if context & contexts.HAS_TEXT:
  1077. if context & contexts.FAIL_ON_TEXT:
  1078. if this is self.END or not this.isspace():
  1079. if this == "<" and self._read(1) == "!":
  1080. self._context |= contexts.FAIL_NEXT
  1081. return True
  1082. return False
  1083. else:
  1084. if this == "\n":
  1085. self._context |= contexts.FAIL_ON_TEXT
  1086. elif this is self.END or not this.isspace():
  1087. self._context |= contexts.HAS_TEXT
  1088. return True
  1089. elif context & contexts.TAG_CLOSE:
  1090. return this != "<"
  1091. else:
  1092. if context & contexts.FAIL_ON_EQUALS:
  1093. if this == "=":
  1094. return False
  1095. elif context & contexts.FAIL_ON_LBRACE:
  1096. if this == "{" or (self._read(-1) == self._read(-2) == "{"):
  1097. if context & contexts.TEMPLATE:
  1098. self._context |= contexts.FAIL_ON_EQUALS
  1099. else:
  1100. self._context |= contexts.FAIL_NEXT
  1101. return True
  1102. self._context ^= contexts.FAIL_ON_LBRACE
  1103. elif context & contexts.FAIL_ON_RBRACE:
  1104. if this == "}":
  1105. self._context |= contexts.FAIL_NEXT
  1106. return True
  1107. self._context ^= contexts.FAIL_ON_RBRACE
  1108. elif this == "{":
  1109. self._context |= contexts.FAIL_ON_LBRACE
  1110. elif this == "}":
  1111. self._context |= contexts.FAIL_ON_RBRACE
  1112. return True
  1113. def _parse(self, context=0, push=True):
  1114. """Parse the wikicode string, using *context* for when to stop."""
  1115. if push:
  1116. self._push(context)
  1117. while True:
  1118. this = self._read()
  1119. if self._context & contexts.UNSAFE:
  1120. if not self._verify_safe(this):
  1121. if self._context & contexts.DOUBLE:
  1122. self._pop()
  1123. self._fail_route()
  1124. if this not in self.MARKERS:
  1125. self._emit_text(this)
  1126. self._head += 1
  1127. continue
  1128. if this is self.END:
  1129. return self._handle_end()
  1130. next = self._read(1)
  1131. if this == next == "{":
  1132. if self._can_recurse():
  1133. self._parse_template_or_argument()
  1134. else:
  1135. self._emit_text("{")
  1136. elif this == "|" and self._context & contexts.TEMPLATE:
  1137. self._handle_template_param()
  1138. elif this == "=" and self._context & contexts.TEMPLATE_PARAM_KEY:
  1139. self._handle_template_param_value()
  1140. elif this == next == "}" and self._context & contexts.TEMPLATE:
  1141. return self._handle_template_end()
  1142. elif this == "|" and self._context & contexts.ARGUMENT_NAME:
  1143. self._handle_argument_separator()
  1144. elif this == next == "}" and self._context & contexts.ARGUMENT:
  1145. if self._read(2) == "}":
  1146. return self._handle_argument_end()
  1147. else:
  1148. self._emit_text("}")
  1149. elif this == next == "[" and self._can_recurse():
  1150. if not self._context & contexts.NO_WIKILINKS:
  1151. self._parse_wikilink()
  1152. else:
  1153. self._emit_text("[")
  1154. elif this == "|" and self._context & contexts.WIKILINK_TITLE:
  1155. self._handle_wikilink_separator()
  1156. elif this == next == "]" and self._context & contexts.WIKILINK:
  1157. return self._handle_wikilink_end()
  1158. elif this == "[":
  1159. self._parse_external_link(True)
  1160. elif this == ":" and self._read(-1) not in self.MARKERS:
  1161. self._parse_external_link(False)
  1162. elif this == "]" and self._context & contexts.EXT_LINK_TITLE:
  1163. return self._pop()
  1164. elif this == "=" and not self._global & contexts.GL_HEADING:
  1165. if self._read(-1) in ("\n", self.START):
  1166. self._parse_heading()
  1167. else:
  1168. self._emit_text("=")
  1169. elif this == "=" and self._context & contexts.HEADING:
  1170. return self._handle_heading_end()
  1171. elif this == "\n" and self._context & contexts.HEADING:
  1172. self._fail_route()
  1173. elif this == "&":
  1174. self._parse_entity()
  1175. elif this == "<" and next == "!":
  1176. if self._read(2) == self._read(3) == "-":
  1177. self._parse_comment()
  1178. else:
  1179. self._emit_text(this)
  1180. elif this == "<" and next == "/" and self._read(2) is not self.END:
  1181. if self._context & contexts.TAG_BODY:
  1182. self._handle_tag_open_close()
  1183. else:
  1184. self._handle_invalid_tag_start()
  1185. elif this == "<" and not self._context & contexts.TAG_CLOSE:
  1186. if self._can_recurse():
  1187. self._parse_tag()
  1188. else:
  1189. self._emit_text("<")
  1190. elif this == ">" and self._context & contexts.TAG_CLOSE:
  1191. return self._handle_tag_close_close()
  1192. elif this == next == "'" and not self._skip_style_tags:
  1193. result = self._parse_style()
  1194. if result is not None:
  1195. return result
  1196. elif self._read(-1) in ("\n", self.START) and this in ("#", "*", ";", ":"):
  1197. self._handle_list()
  1198. elif self._read(-1) in ("\n", self.START) and this == next == self._read(2) == self._read(3) == "-":
  1199. self._handle_hr()
  1200. elif this in ("\n", ":") and self._context & contexts.DL_TERM:
  1201. self._handle_dl_term()
  1202. if this == "\n":
  1203. # kill potential table contexts
  1204. self._context &= ~contexts.TABLE_CELL_LINE_CONTEXTS
  1205. # Start of table parsing
  1206. elif this == "{" and next == "|" and (self._read(-1) in ("\n", self.START) or
  1207. (self._read(-2) in ("\n", self.START) and self._read(-1).isspace())):
  1208. if self._can_recurse():
  1209. self._handle_table_start()
  1210. else:
  1211. self._emit_text("{|")
  1212. elif self._context & contexts.TABLE_OPEN:
  1213. if this == "|" and next == "|" and self._context & contexts.TABLE_TD_LINE:
  1214. if self._context & contexts.TABLE_CELL_OPEN:
  1215. return self._handle_table_cell_end()
  1216. self._handle_table_cell("||", "td", contexts.TABLE_TD_LINE)
  1217. elif this == "|" and next == "|" and self._context & contexts.TABLE_TH_LINE:
  1218. if self._context & contexts.TABLE_CELL_OPEN:
  1219. return self._handle_table_cell_end()
  1220. self._handle_table_cell("||", "th", contexts.TABLE_TH_LINE)
  1221. elif this == "!" and next == "!" and self._context & contexts.TABLE_TH_LINE:
  1222. if self._context & contexts.TABLE_CELL_OPEN:
  1223. return self._handle_table_cell_end()
  1224. self._handle_table_cell("!!", "th", contexts.TABLE_TH_LINE)
  1225. elif this == "|" and self._context & contexts.TABLE_CELL_STYLE:
  1226. return self._handle_table_cell_end(reset_for_style=True)
  1227. # on newline, clear out cell line contexts
  1228. elif this == "\n" and self._context & contexts.TABLE_CELL_LINE_CONTEXTS:
  1229. self._context &= ~contexts.TABLE_CELL_LINE_CONTEXTS
  1230. self._emit_text(this)
  1231. elif (self._read(-1) in ("\n", self.START) or
  1232. (self._read(-2) in ("\n", self.START) and self._read(-1).isspace())):
  1233. if this == "|" and next == "}":
  1234. if self._context & contexts.TABLE_CELL_OPEN:
  1235. return self._handle_table_cell_end()
  1236. return self._handle_table_end()
  1237. elif this == "|" and next == "-":
  1238. if self._context & contexts.TABLE_CELL_OPEN:
  1239. return self._handle_table_cell_end()
  1240. self._handle_table_row()
  1241. elif this == "|":
  1242. if self._context & contexts.TABLE_CELL_OPEN:
  1243. return self._handle_table_cell_end()
  1244. self._handle_table_cell("|", "td", contexts.TABLE_TD_LINE)
  1245. elif this == "!":
  1246. if self._context & contexts.TABLE_CELL_OPEN:
  1247. return self._handle_table_cell_end()
  1248. self._handle_table_cell("!", "th", contexts.TABLE_TH_LINE)
  1249. else:
  1250. self._emit_text(this)
  1251. else:
  1252. self._emit_text(this)
  1253. else:
  1254. self._emit_text(this)
  1255. self._head += 1
  1256. def tokenize(self, text, context=0, skip_style_tags=False):
  1257. """Build a list of tokens from a string of wikicode and return it."""
  1258. self._skip_style_tags = skip_style_tags
  1259. split = self.regex.split(text)
  1260. self._text = [segment for segment in split if segment]
  1261. self._head = self._global = self._depth = self._cycles = 0
  1262. try:
  1263. return self._parse(context)
  1264. except BadRoute: # pragma: no cover (untestable/exceptional case)
  1265. raise ParserError("Python tokenizer exited with BadRoute")