A Python parser for MediaWiki wikicode https://mwparserfromhell.readthedocs.io/
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

685 lines
30KB

  1. # -*- coding: utf-8 -*-
  2. #
  3. # Copyright (C) 2012-2019 Ben Kurtovic <ben.kurtovic@gmail.com>
  4. #
  5. # Permission is hereby granted, free of charge, to any person obtaining a copy
  6. # of this software and associated documentation files (the "Software"), to deal
  7. # in the Software without restriction, including without limitation the rights
  8. # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
  9. # copies of the Software, and to permit persons to whom the Software is
  10. # furnished to do so, subject to the following conditions:
  11. #
  12. # The above copyright notice and this permission notice shall be included in
  13. # all copies or substantial portions of the Software.
  14. #
  15. # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16. # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17. # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  18. # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19. # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
  20. # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  21. # SOFTWARE.
  22. from __future__ import unicode_literals
  23. from itertools import chain
  24. import re
  25. from .compat import bytes, py3k, range, str
  26. from .nodes import (Argument, Comment, ExternalLink, Heading, HTMLEntity,
  27. Node, Tag, Template, Text, Wikilink)
  28. from .smart_list import _ListProxy
  29. from .string_mixin import StringMixIn
  30. from .utils import parse_anything
  31. __all__ = ["Wikicode"]
  32. FLAGS = re.IGNORECASE | re.DOTALL | re.UNICODE
  33. class Wikicode(StringMixIn):
  34. """A ``Wikicode`` is a container for nodes that operates like a string.
  35. Additionally, it contains methods that can be used to extract data from or
  36. modify the nodes, implemented in an interface similar to a list. For
  37. example, :meth:`index` can get the index of a node in the list, and
  38. :meth:`insert` can add a new node at that index. The :meth:`filter()
  39. <ifilter>` series of functions is very useful for extracting and iterating
  40. over, for example, all of the templates in the object.
  41. """
  42. RECURSE_OTHERS = 2
  43. def __init__(self, nodes):
  44. super(Wikicode, self).__init__()
  45. self._nodes = nodes
  46. def __unicode__(self):
  47. return "".join([str(node) for node in self.nodes])
  48. @staticmethod
  49. def _get_children(node, contexts=False, restrict=None, parent=None):
  50. """Iterate over all child :class:`.Node`\\ s of a given *node*."""
  51. yield (parent, node) if contexts else node
  52. if restrict and isinstance(node, restrict):
  53. return
  54. for code in node.__children__():
  55. for child in code.nodes:
  56. sub = Wikicode._get_children(child, contexts, restrict, code)
  57. for result in sub:
  58. yield result
  59. @staticmethod
  60. def _slice_replace(code, index, old, new):
  61. """Replace the string *old* with *new* across *index* in *code*."""
  62. nodes = [str(node) for node in code.get(index)]
  63. substring = "".join(nodes).replace(old, new)
  64. code.nodes[index] = parse_anything(substring).nodes
  65. @staticmethod
  66. def _build_matcher(matches, flags):
  67. """Helper for :meth:`_indexed_ifilter` and others.
  68. If *matches* is a function, return it. If it's a regex, return a
  69. wrapper around it that can be called with a node to do a search. If
  70. it's ``None``, return a function that always returns ``True``.
  71. """
  72. if matches:
  73. if callable(matches):
  74. return matches
  75. return lambda obj: re.search(matches, str(obj), flags)
  76. return lambda obj: True
  77. def _indexed_ifilter(self, recursive=True, matches=None, flags=FLAGS,
  78. forcetype=None):
  79. """Iterate over nodes and their corresponding indices in the node list.
  80. The arguments are interpreted as for :meth:`ifilter`. For each tuple
  81. ``(i, node)`` yielded by this method, ``self.index(node) == i``. Note
  82. that if *recursive* is ``True``, ``self.nodes[i]`` might not be the
  83. node itself, but will still contain it.
  84. """
  85. match = self._build_matcher(matches, flags)
  86. if recursive:
  87. restrict = forcetype if recursive == self.RECURSE_OTHERS else None
  88. def getter(i, node):
  89. for ch in self._get_children(node, restrict=restrict):
  90. yield (i, ch)
  91. inodes = chain(*(getter(i, n) for i, n in enumerate(self.nodes)))
  92. else:
  93. inodes = enumerate(self.nodes)
  94. for i, node in inodes:
  95. if (not forcetype or isinstance(node, forcetype)) and match(node):
  96. yield (i, node)
  97. def _is_child_wikicode(self, obj, recursive=True):
  98. """Return whether the given :class:`.Wikicode` is a descendant."""
  99. def deref(nodes):
  100. if isinstance(nodes, _ListProxy):
  101. return nodes._parent # pylint: disable=protected-access
  102. return nodes
  103. target = deref(obj.nodes)
  104. if target is deref(self.nodes):
  105. return True
  106. if recursive:
  107. todo = [self]
  108. while todo:
  109. code = todo.pop()
  110. if target is deref(code.nodes):
  111. return True
  112. for node in code.nodes:
  113. todo += list(node.__children__())
  114. return False
  115. def _do_strong_search(self, obj, recursive=True):
  116. """Search for the specific element *obj* within the node list.
  117. *obj* can be either a :class:`.Node` or a :class:`.Wikicode` object. If
  118. found, we return a tuple (*context*, *index*) where *context* is the
  119. :class:`.Wikicode` that contains *obj* and *index* is its index there,
  120. as a :class:`slice`. Note that if *recursive* is ``False``, *context*
  121. will always be ``self`` (since we only look for *obj* among immediate
  122. descendants), but if *recursive* is ``True``, then it could be any
  123. :class:`.Wikicode` contained by a node within ``self``. If *obj* is not
  124. found, :exc:`ValueError` is raised.
  125. """
  126. if isinstance(obj, Wikicode):
  127. if not self._is_child_wikicode(obj, recursive):
  128. raise ValueError(obj)
  129. return obj, slice(0, len(obj.nodes))
  130. if isinstance(obj, Node):
  131. mkslice = lambda i: slice(i, i + 1)
  132. if not recursive:
  133. return self, mkslice(self.index(obj))
  134. for node in self.nodes:
  135. for context, child in self._get_children(node, contexts=True):
  136. if obj is child:
  137. if not context:
  138. context = self
  139. return context, mkslice(context.index(child))
  140. raise ValueError(obj)
  141. raise TypeError(obj)
  142. def _do_weak_search(self, obj, recursive):
  143. """Search for an element that looks like *obj* within the node list.
  144. This follows the same rules as :meth:`_do_strong_search` with some
  145. differences. *obj* is treated as a string that might represent any
  146. :class:`.Node`, :class:`.Wikicode`, or combination of the two present
  147. in the node list. Thus, matching is weak (using string comparisons)
  148. rather than strong (using ``is``). Because multiple nodes can match
  149. *obj*, the result is a list of tuples instead of just one (however,
  150. :exc:`ValueError` is still raised if nothing is found). Individual
  151. matches will never overlap.
  152. The tuples contain a new first element, *exact*, which is ``True`` if
  153. we were able to match *obj* exactly to one or more adjacent nodes, or
  154. ``False`` if we found *obj* inside a node or incompletely spanning
  155. multiple nodes.
  156. """
  157. obj = parse_anything(obj)
  158. if not obj or obj not in self:
  159. raise ValueError(obj)
  160. results = []
  161. contexts = [self]
  162. while contexts:
  163. context = contexts.pop()
  164. i = len(context.nodes) - 1
  165. while i >= 0:
  166. node = context.get(i)
  167. if obj.get(-1) == node:
  168. for j in range(-len(obj.nodes), -1):
  169. if obj.get(j) != context.get(i + j + 1):
  170. break
  171. else:
  172. i -= len(obj.nodes) - 1
  173. index = slice(i, i + len(obj.nodes))
  174. results.append((True, context, index))
  175. elif recursive and obj in node:
  176. contexts.extend(node.__children__())
  177. i -= 1
  178. if not results:
  179. if not recursive:
  180. raise ValueError(obj)
  181. results.append((False, self, slice(0, len(self.nodes))))
  182. return results
  183. def _get_tree(self, code, lines, marker, indent):
  184. """Build a tree to illustrate the way the Wikicode object was parsed.
  185. The method that builds the actual tree is ``__showtree__`` of ``Node``
  186. objects. *code* is the ``Wikicode`` object to build a tree for. *lines*
  187. is the list to append the tree to, which is returned at the end of the
  188. method. *marker* is some object to be used to indicate that the builder
  189. should continue on from the last line instead of starting a new one; it
  190. should be any object that can be tested for with ``is``. *indent* is
  191. the starting indentation.
  192. """
  193. def write(*args):
  194. """Write a new line following the proper indentation rules."""
  195. if lines and lines[-1] is marker: # Continue from the last line
  196. lines.pop() # Remove the marker
  197. last = lines.pop()
  198. lines.append(last + " ".join(args))
  199. else:
  200. lines.append(" " * 6 * indent + " ".join(args))
  201. get = lambda code: self._get_tree(code, lines, marker, indent + 1)
  202. mark = lambda: lines.append(marker)
  203. for node in code.nodes:
  204. node.__showtree__(write, get, mark)
  205. return lines
  206. @classmethod
  207. def _build_filter_methods(cls, **meths):
  208. """Given Node types, build the corresponding i?filter shortcuts.
  209. The should be given as keys storing the method's base name paired with
  210. values storing the corresponding :class:`.Node` type. For example, the
  211. dict may contain the pair ``("templates", Template)``, which will
  212. produce the methods :meth:`ifilter_templates` and
  213. :meth:`filter_templates`, which are shortcuts for
  214. :meth:`ifilter(forcetype=Template) <ifilter>` and
  215. :meth:`filter(forcetype=Template) <filter>`, respectively. These
  216. shortcuts are added to the class itself, with an appropriate docstring.
  217. """
  218. doc = """Iterate over {0}.
  219. This is equivalent to :meth:`{1}` with *forcetype* set to
  220. :class:`~{2.__module__}.{2.__name__}`.
  221. """
  222. make_ifilter = lambda ftype: (lambda self, *a, **kw:
  223. self.ifilter(forcetype=ftype, *a, **kw))
  224. make_filter = lambda ftype: (lambda self, *a, **kw:
  225. self.filter(forcetype=ftype, *a, **kw))
  226. for name, ftype in (meths.items() if py3k else meths.iteritems()):
  227. ifilter = make_ifilter(ftype)
  228. filter = make_filter(ftype)
  229. ifilter.__doc__ = doc.format(name, "ifilter", ftype)
  230. filter.__doc__ = doc.format(name, "filter", ftype)
  231. setattr(cls, "ifilter_" + name, ifilter)
  232. setattr(cls, "filter_" + name, filter)
  233. @property
  234. def nodes(self):
  235. """A list of :class:`.Node` objects.
  236. This is the internal data actually stored within a :class:`.Wikicode`
  237. object.
  238. """
  239. return self._nodes
  240. @nodes.setter
  241. def nodes(self, value):
  242. if not isinstance(value, list):
  243. value = parse_anything(value).nodes
  244. self._nodes = value
  245. def get(self, index):
  246. """Return the *index*\\ th node within the list of nodes."""
  247. return self.nodes[index]
  248. def set(self, index, value):
  249. """Set the ``Node`` at *index* to *value*.
  250. Raises :exc:`IndexError` if *index* is out of range, or
  251. :exc:`ValueError` if *value* cannot be coerced into one :class:`.Node`.
  252. To insert multiple nodes at an index, use :meth:`get` with either
  253. :meth:`remove` and :meth:`insert` or :meth:`replace`.
  254. """
  255. nodes = parse_anything(value).nodes
  256. if len(nodes) > 1:
  257. raise ValueError("Cannot coerce multiple nodes into one index")
  258. if index >= len(self.nodes) or -1 * index > len(self.nodes):
  259. raise IndexError("List assignment index out of range")
  260. if nodes:
  261. self.nodes[index] = nodes[0]
  262. else:
  263. self.nodes.pop(index)
  264. def contains(self, obj):
  265. """Return whether this Wikicode object contains *obj*.
  266. If *obj* is a :class:`.Node` or :class:`.Wikicode` object, then we
  267. search for it exactly among all of our children, recursively.
  268. Otherwise, this method just uses :meth:`.__contains__` on the string.
  269. """
  270. if not isinstance(obj, (Node, Wikicode)):
  271. return obj in self
  272. try:
  273. self._do_strong_search(obj, recursive=True)
  274. except ValueError:
  275. return False
  276. return True
  277. def index(self, obj, recursive=False):
  278. """Return the index of *obj* in the list of nodes.
  279. Raises :exc:`ValueError` if *obj* is not found. If *recursive* is
  280. ``True``, we will look in all nodes of ours and their descendants, and
  281. return the index of our direct descendant node within *our* list of
  282. nodes. Otherwise, the lookup is done only on direct descendants.
  283. """
  284. strict = isinstance(obj, Node)
  285. equivalent = (lambda o, n: o is n) if strict else (lambda o, n: o == n)
  286. for i, node in enumerate(self.nodes):
  287. if recursive:
  288. for child in self._get_children(node):
  289. if equivalent(obj, child):
  290. return i
  291. elif equivalent(obj, node):
  292. return i
  293. raise ValueError(obj)
  294. def get_ancestors(self, obj):
  295. """Return a list of all ancestor nodes of the :class:`.Node` *obj*.
  296. The list is ordered from the most shallow ancestor (greatest great-
  297. grandparent) to the direct parent. The node itself is not included in
  298. the list. For example::
  299. >>> text = "{{a|{{b|{{c|{{d}}}}}}}}"
  300. >>> code = mwparserfromhell.parse(text)
  301. >>> node = code.filter_templates(matches=lambda n: n == "{{d}}")[0]
  302. >>> code.get_ancestors(node)
  303. ['{{a|{{b|{{c|{{d}}}}}}}}', '{{b|{{c|{{d}}}}}}', '{{c|{{d}}}}']
  304. Will return an empty list if *obj* is at the top level of this Wikicode
  305. object. Will raise :exc:`ValueError` if it wasn't found.
  306. """
  307. def _get_ancestors(code, needle):
  308. for node in code.nodes:
  309. if node is needle:
  310. return []
  311. for code in node.__children__():
  312. ancestors = _get_ancestors(code, needle)
  313. if ancestors is not None:
  314. return [node] + ancestors
  315. if isinstance(obj, Wikicode):
  316. obj = obj.get(0)
  317. elif not isinstance(obj, Node):
  318. raise ValueError(obj)
  319. ancestors = _get_ancestors(self, obj)
  320. if ancestors is None:
  321. raise ValueError(obj)
  322. return ancestors
  323. def get_parent(self, obj):
  324. """Return the direct parent node of the :class:`.Node` *obj*.
  325. This function is equivalent to calling :meth:`.get_ancestors` and
  326. taking the last element of the resulting list. Will return None if
  327. the node exists but does not have a parent; i.e., it is at the top
  328. level of the Wikicode object.
  329. """
  330. ancestors = self.get_ancestors(obj)
  331. return ancestors[-1] if ancestors else None
  332. def insert(self, index, value):
  333. """Insert *value* at *index* in the list of nodes.
  334. *value* can be anything parsable by :func:`.parse_anything`, which
  335. includes strings or other :class:`.Wikicode` or :class:`.Node` objects.
  336. """
  337. nodes = parse_anything(value).nodes
  338. for node in reversed(nodes):
  339. self.nodes.insert(index, node)
  340. def insert_before(self, obj, value, recursive=True):
  341. """Insert *value* immediately before *obj*.
  342. *obj* can be either a string, a :class:`.Node`, or another
  343. :class:`.Wikicode` object (as created by :meth:`get_sections`, for
  344. example). If *obj* is a string, we will operate on all instances of
  345. that string within the code, otherwise only on the specific instance
  346. given. *value* can be anything parsable by :func:`.parse_anything`. If
  347. *recursive* is ``True``, we will try to find *obj* within our child
  348. nodes even if it is not a direct descendant of this :class:`.Wikicode`
  349. object. If *obj* is not found, :exc:`ValueError` is raised.
  350. """
  351. if isinstance(obj, (Node, Wikicode)):
  352. context, index = self._do_strong_search(obj, recursive)
  353. context.insert(index.start, value)
  354. else:
  355. for exact, context, index in self._do_weak_search(obj, recursive):
  356. if exact:
  357. context.insert(index.start, value)
  358. else:
  359. obj = str(obj)
  360. self._slice_replace(context, index, obj, str(value) + obj)
  361. def insert_after(self, obj, value, recursive=True):
  362. """Insert *value* immediately after *obj*.
  363. *obj* can be either a string, a :class:`.Node`, or another
  364. :class:`.Wikicode` object (as created by :meth:`get_sections`, for
  365. example). If *obj* is a string, we will operate on all instances of
  366. that string within the code, otherwise only on the specific instance
  367. given. *value* can be anything parsable by :func:`.parse_anything`. If
  368. *recursive* is ``True``, we will try to find *obj* within our child
  369. nodes even if it is not a direct descendant of this :class:`.Wikicode`
  370. object. If *obj* is not found, :exc:`ValueError` is raised.
  371. """
  372. if isinstance(obj, (Node, Wikicode)):
  373. context, index = self._do_strong_search(obj, recursive)
  374. context.insert(index.stop, value)
  375. else:
  376. for exact, context, index in self._do_weak_search(obj, recursive):
  377. if exact:
  378. context.insert(index.stop, value)
  379. else:
  380. obj = str(obj)
  381. self._slice_replace(context, index, obj, obj + str(value))
  382. def replace(self, obj, value, recursive=True):
  383. """Replace *obj* with *value*.
  384. *obj* can be either a string, a :class:`.Node`, or another
  385. :class:`.Wikicode` object (as created by :meth:`get_sections`, for
  386. example). If *obj* is a string, we will operate on all instances of
  387. that string within the code, otherwise only on the specific instance
  388. given. *value* can be anything parsable by :func:`.parse_anything`.
  389. If *recursive* is ``True``, we will try to find *obj* within our child
  390. nodes even if it is not a direct descendant of this :class:`.Wikicode`
  391. object. If *obj* is not found, :exc:`ValueError` is raised.
  392. """
  393. if isinstance(obj, (Node, Wikicode)):
  394. context, index = self._do_strong_search(obj, recursive)
  395. for i in range(index.start, index.stop):
  396. context.nodes.pop(index.start)
  397. context.insert(index.start, value)
  398. else:
  399. for exact, context, index in self._do_weak_search(obj, recursive):
  400. if exact:
  401. for i in range(index.start, index.stop):
  402. context.nodes.pop(index.start)
  403. context.insert(index.start, value)
  404. else:
  405. self._slice_replace(context, index, str(obj), str(value))
  406. def append(self, value):
  407. """Insert *value* at the end of the list of nodes.
  408. *value* can be anything parsable by :func:`.parse_anything`.
  409. """
  410. nodes = parse_anything(value).nodes
  411. for node in nodes:
  412. self.nodes.append(node)
  413. def remove(self, obj, recursive=True):
  414. """Remove *obj* from the list of nodes.
  415. *obj* can be either a string, a :class:`.Node`, or another
  416. :class:`.Wikicode` object (as created by :meth:`get_sections`, for
  417. example). If *obj* is a string, we will operate on all instances of
  418. that string within the code, otherwise only on the specific instance
  419. given. If *recursive* is ``True``, we will try to find *obj* within our
  420. child nodes even if it is not a direct descendant of this
  421. :class:`.Wikicode` object. If *obj* is not found, :exc:`ValueError` is
  422. raised.
  423. """
  424. if isinstance(obj, (Node, Wikicode)):
  425. context, index = self._do_strong_search(obj, recursive)
  426. for i in range(index.start, index.stop):
  427. context.nodes.pop(index.start)
  428. else:
  429. for exact, context, index in self._do_weak_search(obj, recursive):
  430. if exact:
  431. for i in range(index.start, index.stop):
  432. context.nodes.pop(index.start)
  433. else:
  434. self._slice_replace(context, index, str(obj), "")
  435. def matches(self, other):
  436. """Do a loose equivalency test suitable for comparing page names.
  437. *other* can be any string-like object, including :class:`.Wikicode`, or
  438. an iterable of these. This operation is symmetric; both sides are
  439. adjusted. Specifically, whitespace and markup is stripped and the first
  440. letter's case is normalized. Typical usage is
  441. ``if template.name.matches("stub"): ...``.
  442. """
  443. normalize = lambda s: (s[0].upper() + s[1:]).replace("_", " ") if s else s
  444. this = normalize(self.strip_code().strip())
  445. if isinstance(other, (str, bytes, Wikicode, Node)):
  446. that = parse_anything(other).strip_code().strip()
  447. return this == normalize(that)
  448. for obj in other:
  449. that = parse_anything(obj).strip_code().strip()
  450. if this == normalize(that):
  451. return True
  452. return False
  453. def ifilter(self, recursive=True, matches=None, flags=FLAGS,
  454. forcetype=None):
  455. """Iterate over nodes in our list matching certain conditions.
  456. If *forcetype* is given, only nodes that are instances of this type (or
  457. tuple of types) are yielded. Setting *recursive* to ``True`` will
  458. iterate over all children and their descendants. ``RECURSE_OTHERS``
  459. will only iterate over children that are not the instances of
  460. *forcetype*. ``False`` will only iterate over immediate children.
  461. ``RECURSE_OTHERS`` can be used to iterate over all un-nested templates,
  462. even if they are inside of HTML tags, like so:
  463. >>> code = mwparserfromhell.parse("{{foo}}<b>{{foo|{{bar}}}}</b>")
  464. >>> code.filter_templates(code.RECURSE_OTHERS)
  465. ["{{foo}}", "{{foo|{{bar}}}}"]
  466. *matches* can be used to further restrict the nodes, either as a
  467. function (taking a single :class:`.Node` and returning a boolean) or a
  468. regular expression (matched against the node's string representation
  469. with :func:`re.search`). If *matches* is a regex, the flags passed to
  470. :func:`re.search` are :const:`re.IGNORECASE`, :const:`re.DOTALL`, and
  471. :const:`re.UNICODE`, but custom flags can be specified by passing
  472. *flags*.
  473. """
  474. gen = self._indexed_ifilter(recursive, matches, flags, forcetype)
  475. return (node for i, node in gen)
  476. def filter(self, *args, **kwargs):
  477. """Return a list of nodes within our list matching certain conditions.
  478. This is equivalent to calling :func:`list` on :meth:`ifilter`.
  479. """
  480. return list(self.ifilter(*args, **kwargs))
  481. def get_sections(self, levels=None, matches=None, flags=FLAGS, flat=False,
  482. include_lead=None, include_headings=True):
  483. """Return a list of sections within the page.
  484. Sections are returned as :class:`.Wikicode` objects with a shared node
  485. list (implemented using :class:`.SmartList`) so that changes to
  486. sections are reflected in the parent Wikicode object.
  487. Each section contains all of its subsections, unless *flat* is
  488. ``True``. If *levels* is given, it should be a iterable of integers;
  489. only sections whose heading levels are within it will be returned. If
  490. *matches* is given, it should be either a function or a regex; only
  491. sections whose headings match it (without the surrounding equal signs)
  492. will be included. *flags* can be used to override the default regex
  493. flags (see :meth:`ifilter`) if a regex *matches* is used.
  494. If *include_lead* is ``True``, the first, lead section (without a
  495. heading) will be included in the list; ``False`` will not include it;
  496. the default will include it only if no specific *levels* were given. If
  497. *include_headings* is ``True``, the section's beginning
  498. :class:`.Heading` object will be included; otherwise, this is skipped.
  499. """
  500. title_matcher = self._build_matcher(matches, flags)
  501. matcher = lambda heading: (title_matcher(heading.title) and
  502. (not levels or heading.level in levels))
  503. iheadings = self._indexed_ifilter(recursive=False, forcetype=Heading)
  504. sections = [] # Tuples of (index_of_first_node, section)
  505. open_headings = [] # Tuples of (index, heading), where index and
  506. # heading.level are both monotonically increasing
  507. # Add the lead section if appropriate:
  508. if include_lead or not (include_lead is not None or matches or levels):
  509. itr = self._indexed_ifilter(recursive=False, forcetype=Heading)
  510. try:
  511. first = next(itr)[0]
  512. sections.append((0, Wikicode(self.nodes[:first])))
  513. except StopIteration: # No headings in page
  514. sections.append((0, Wikicode(self.nodes[:])))
  515. # Iterate over headings, adding sections to the list as they end:
  516. for i, heading in iheadings:
  517. if flat: # With flat, all sections close at the next heading
  518. newly_closed, open_headings = open_headings, []
  519. else: # Otherwise, figure out which sections have closed, if any
  520. closed_start_index = len(open_headings)
  521. for j, (start, last_heading) in enumerate(open_headings):
  522. if heading.level <= last_heading.level:
  523. closed_start_index = j
  524. break
  525. newly_closed = open_headings[closed_start_index:]
  526. del open_headings[closed_start_index:]
  527. for start, closed_heading in newly_closed:
  528. if matcher(closed_heading):
  529. sections.append((start, Wikicode(self.nodes[start:i])))
  530. start = i if include_headings else (i + 1)
  531. open_headings.append((start, heading))
  532. # Add any remaining open headings to the list of sections:
  533. for start, heading in open_headings:
  534. if matcher(heading):
  535. sections.append((start, Wikicode(self.nodes[start:])))
  536. # Ensure that earlier sections are earlier in the returned list:
  537. return [section for i, section in sorted(sections)]
  538. def strip_code(self, normalize=True, collapse=True,
  539. keep_template_params=False):
  540. """Return a rendered string without unprintable code such as templates.
  541. The way a node is stripped is handled by the
  542. :meth:`~.Node.__strip__` method of :class:`.Node` objects, which
  543. generally return a subset of their nodes or ``None``. For example,
  544. templates and tags are removed completely, links are stripped to just
  545. their display part, headings are stripped to just their title.
  546. If *normalize* is ``True``, various things may be done to strip code
  547. further, such as converting HTML entities like ``&Sigma;``, ``&#931;``,
  548. and ``&#x3a3;`` to ``Σ``. If *collapse* is ``True``, we will try to
  549. remove excess whitespace as well (three or more newlines are converted
  550. to two, for example). If *keep_template_params* is ``True``, then
  551. template parameters will be preserved in the output (normally, they are
  552. removed completely).
  553. """
  554. kwargs = {
  555. "normalize": normalize,
  556. "collapse": collapse,
  557. "keep_template_params": keep_template_params
  558. }
  559. nodes = []
  560. for node in self.nodes:
  561. stripped = node.__strip__(**kwargs)
  562. if stripped:
  563. nodes.append(str(stripped))
  564. if collapse:
  565. stripped = "".join(nodes).strip("\n")
  566. while "\n\n\n" in stripped:
  567. stripped = stripped.replace("\n\n\n", "\n\n")
  568. return stripped
  569. else:
  570. return "".join(nodes)
  571. def get_tree(self):
  572. """Return a hierarchical tree representation of the object.
  573. The representation is a string makes the most sense printed. It is
  574. built by calling :meth:`_get_tree` on the :class:`.Wikicode` object and
  575. its children recursively. The end result may look something like the
  576. following::
  577. >>> text = "Lorem ipsum {{foo|bar|{{baz}}|spam=eggs}}"
  578. >>> print(mwparserfromhell.parse(text).get_tree())
  579. Lorem ipsum
  580. {{
  581. foo
  582. | 1
  583. = bar
  584. | 2
  585. = {{
  586. baz
  587. }}
  588. | spam
  589. = eggs
  590. }}
  591. """
  592. marker = object() # Random object we can find with certainty in a list
  593. return "\n".join(self._get_tree(self, [], marker, 0))
  594. Wikicode._build_filter_methods(
  595. arguments=Argument, comments=Comment, external_links=ExternalLink,
  596. headings=Heading, html_entities=HTMLEntity, tags=Tag, templates=Template,
  597. text=Text, wikilinks=Wikilink)