A Python parser for MediaWiki wikicode https://mwparserfromhell.readthedocs.io/
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
 
 
 
 

215 regels
7.3 KiB

  1. # -*- coding: utf-8 -*-
  2. #
  3. # Copyright (C) 2012 Ben Kurtovic <ben.kurtovic@verizon.net>
  4. #
  5. # Permission is hereby granted, free of charge, to any person obtaining a copy
  6. # of this software and associated documentation files (the "Software"), to deal
  7. # in the Software without restriction, including without limitation the rights
  8. # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
  9. # copies of the Software, and to permit persons to whom the Software is
  10. # furnished to do so, subject to the following conditions:
  11. #
  12. # The above copyright notice and this permission notice shall be included in
  13. # all copies or substantial portions of the Software.
  14. #
  15. # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16. # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17. # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  18. # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19. # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
  20. # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  21. # SOFTWARE.
  22. import htmlentitydefs
  23. import re
  24. import string
  25. from . import contexts
  26. from . import tokens
  27. __all__ = ["Tokenizer"]
  28. class BadRoute(Exception):
  29. pass
  30. class Tokenizer(object):
  31. START = object()
  32. END = object()
  33. SENTINELS = ["{", "}", "[", "]", "<", ">", "|", "=", "&", "#", "*", ";",
  34. ":", "/", "-", END]
  35. regex = re.compile(r"([{}\[\]<>|=&#*;:/-])", flags=re.IGNORECASE)
  36. def __init__(self):
  37. self._text = None
  38. self._head = 0
  39. self._stacks = []
  40. @property
  41. def _stack(self):
  42. return self._stacks[-1][0]
  43. @property
  44. def _context(self):
  45. return self._stacks[-1][1]
  46. @_context.setter
  47. def _context(self, value):
  48. self._stacks[-1][1] = value
  49. @property
  50. def _textbuffer(self):
  51. return self._stacks[-1][2]
  52. @_textbuffer.setter
  53. def _textbuffer(self, value):
  54. self._stacks[-1][2] = value
  55. def _push(self, context=0):
  56. self._stacks.append([[], context, []])
  57. def _push_textbuffer(self):
  58. if self._textbuffer:
  59. self._stack.append(tokens.Text(text="".join(self._textbuffer)))
  60. self._textbuffer = []
  61. def _pop(self):
  62. self._push_textbuffer()
  63. return self._stacks.pop()[0]
  64. def _write(self, token):
  65. self._push_textbuffer()
  66. self._stack.append(token)
  67. def _write_text(self, text):
  68. self._textbuffer.append(text)
  69. def _write_all(self, tokenlist):
  70. self._push_textbuffer()
  71. self._stack.extend(tokenlist)
  72. def _read(self, delta=0, wrap=False):
  73. index = self._head + delta
  74. if index < 0 and (not wrap or abs(index) > len(self._text)):
  75. return self.START
  76. try:
  77. return self._text[index]
  78. except IndexError:
  79. return self.END
  80. def _parse_template(self):
  81. reset = self._head
  82. self._head += 2
  83. try:
  84. template = self._parse(contexts.TEMPLATE_NAME)
  85. except BadRoute:
  86. self._head = reset
  87. self._write_text(self._read())
  88. else:
  89. self._write(tokens.TemplateOpen())
  90. self._write_all(template)
  91. self._write(tokens.TemplateClose())
  92. def _verify_template_name(self):
  93. self._push_textbuffer()
  94. if self._stack:
  95. text = [tok for tok in self._stack if isinstance(tok, tokens.Text)]
  96. text = "".join([token.text for token in text])
  97. if text.strip() and "\n" in text.strip():
  98. raise BadRoute(self._pop())
  99. def _handle_template_param(self):
  100. if self._context & contexts.TEMPLATE_NAME:
  101. self._verify_template_name()
  102. self._context ^= contexts.TEMPLATE_NAME
  103. if self._context & contexts.TEMPLATE_PARAM_VALUE:
  104. self._context ^= contexts.TEMPLATE_PARAM_VALUE
  105. self._context |= contexts.TEMPLATE_PARAM_KEY
  106. self._write(tokens.TemplateParamSeparator())
  107. def _handle_template_param_value(self):
  108. self._context ^= contexts.TEMPLATE_PARAM_KEY
  109. self._context |= contexts.TEMPLATE_PARAM_VALUE
  110. self._write(tokens.TemplateParamEquals())
  111. def _handle_template_end(self):
  112. if self._context & contexts.TEMPLATE_NAME:
  113. self._verify_template_name()
  114. self._head += 1
  115. return self._pop()
  116. def _parse_entity(self):
  117. reset = self._head
  118. self._push()
  119. try:
  120. self._write(tokens.HTMLEntityStart())
  121. self._head += 1
  122. this = self._read()
  123. if not this or this is self.END:
  124. raise BadRoute(self._pop())
  125. numeric = hexadecimal = False
  126. if this == "#":
  127. numeric = True
  128. self._write(tokens.HTMLEntityNumeric())
  129. self._head += 1
  130. this = self._read()
  131. if not this or this is self.END:
  132. raise BadRoute(self._pop())
  133. if this[0].lower() == "x":
  134. hexadecimal = True
  135. self._write(tokens.HTMLEntityHex(char=this[0]))
  136. this = this[1:]
  137. valid = string.hexdigits if hexadecimal else string.digits
  138. if not numeric and not hexadecimal:
  139. valid += string.ascii_letters
  140. if not all([char in valid for char in this]):
  141. raise BadRoute(self._pop())
  142. self._head += 1
  143. if self._read() != ";":
  144. raise BadRoute(self._pop())
  145. if numeric:
  146. test = int(this, 16) if hexadecimal else int(this)
  147. if test < 1 or test > 0x10FFFF:
  148. raise BadRoute(self._pop())
  149. else:
  150. if this not in htmlentitydefs.entitydefs:
  151. raise BadRoute(self._pop())
  152. self._write(tokens.Text(text=this))
  153. self._write(tokens.HTMLEntityEnd())
  154. except BadRoute:
  155. self._head = reset
  156. self._write_text(self._read())
  157. else:
  158. self._write_all(self._pop())
  159. def _parse(self, context=0):
  160. self._push(context)
  161. while True:
  162. this = self._read()
  163. if this not in self.SENTINELS:
  164. self._write_text(this)
  165. self._head += 1
  166. continue
  167. if this is self.END:
  168. if self._context & contexts.TEMPLATE:
  169. raise BadRoute(self._pop())
  170. return self._pop()
  171. next = self._read(1)
  172. if this == next == "{":
  173. self._parse_template()
  174. elif this == "|" and self._context & contexts.TEMPLATE:
  175. self._handle_template_param()
  176. elif this == "=" and self._context & contexts.TEMPLATE_PARAM_KEY:
  177. self._handle_template_param_value()
  178. elif this == next == "}" and self._context & contexts.TEMPLATE:
  179. return self._handle_template_end()
  180. elif this == "&":
  181. self._parse_entity()
  182. else:
  183. self._write_text(this)
  184. self._head += 1
  185. def tokenize(self, text):
  186. split = self.regex.split(text)
  187. self._text = [segment for segment in split if segment]
  188. return self._parse()