A Python parser for MediaWiki wikicode https://mwparserfromhell.readthedocs.io/
Nelze vybrat více než 25 témat Téma musí začínat písmenem nebo číslem, může obsahovat pomlčky („-“) a může být dlouhé až 35 znaků.
 
 
 
 

663 řádky
29 KiB

  1. # -*- coding: utf-8 -*-
  2. #
  3. # Copyright (C) 2012-2017 Ben Kurtovic <ben.kurtovic@gmail.com>
  4. #
  5. # Permission is hereby granted, free of charge, to any person obtaining a copy
  6. # of this software and associated documentation files (the "Software"), to deal
  7. # in the Software without restriction, including without limitation the rights
  8. # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
  9. # copies of the Software, and to permit persons to whom the Software is
  10. # furnished to do so, subject to the following conditions:
  11. #
  12. # The above copyright notice and this permission notice shall be included in
  13. # all copies or substantial portions of the Software.
  14. #
  15. # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16. # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17. # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  18. # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19. # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
  20. # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  21. # SOFTWARE.
  22. from __future__ import unicode_literals
  23. from itertools import chain
  24. import re
  25. from .compat import bytes, py3k, range, str
  26. from .nodes import (Argument, Comment, ExternalLink, Heading, HTMLEntity,
  27. Node, Tag, Template, Text, Wikilink)
  28. from .string_mixin import StringMixIn
  29. from .utils import parse_anything
  30. __all__ = ["Wikicode"]
  31. FLAGS = re.IGNORECASE | re.DOTALL | re.UNICODE
  32. class Wikicode(StringMixIn):
  33. """A ``Wikicode`` is a container for nodes that operates like a string.
  34. Additionally, it contains methods that can be used to extract data from or
  35. modify the nodes, implemented in an interface similar to a list. For
  36. example, :meth:`index` can get the index of a node in the list, and
  37. :meth:`insert` can add a new node at that index. The :meth:`filter()
  38. <ifilter>` series of functions is very useful for extracting and iterating
  39. over, for example, all of the templates in the object.
  40. """
  41. RECURSE_OTHERS = 2
  42. def __init__(self, nodes):
  43. super(Wikicode, self).__init__()
  44. self._nodes = nodes
  45. def __unicode__(self):
  46. return "".join([str(node) for node in self.nodes])
  47. @staticmethod
  48. def _get_children(node, contexts=False, restrict=None, parent=None):
  49. """Iterate over all child :class:`.Node`\ s of a given *node*."""
  50. yield (parent, node) if contexts else node
  51. if restrict and isinstance(node, restrict):
  52. return
  53. for code in node.__children__():
  54. for child in code.nodes:
  55. sub = Wikicode._get_children(child, contexts, restrict, code)
  56. for result in sub:
  57. yield result
  58. @staticmethod
  59. def _slice_replace(code, index, old, new):
  60. """Replace the string *old* with *new* across *index* in *code*."""
  61. nodes = [str(node) for node in code.get(index)]
  62. substring = "".join(nodes).replace(old, new)
  63. code.nodes[index] = parse_anything(substring).nodes
  64. @staticmethod
  65. def _build_matcher(matches, flags):
  66. """Helper for :meth:`_indexed_ifilter` and others.
  67. If *matches* is a function, return it. If it's a regex, return a
  68. wrapper around it that can be called with a node to do a search. If
  69. it's ``None``, return a function that always returns ``True``.
  70. """
  71. if matches:
  72. if callable(matches):
  73. return matches
  74. return lambda obj: re.search(matches, str(obj), flags)
  75. return lambda obj: True
  76. def _indexed_ifilter(self, recursive=True, matches=None, flags=FLAGS,
  77. forcetype=None):
  78. """Iterate over nodes and their corresponding indices in the node list.
  79. The arguments are interpreted as for :meth:`ifilter`. For each tuple
  80. ``(i, node)`` yielded by this method, ``self.index(node) == i``. Note
  81. that if *recursive* is ``True``, ``self.nodes[i]`` might not be the
  82. node itself, but will still contain it.
  83. """
  84. match = self._build_matcher(matches, flags)
  85. if recursive:
  86. restrict = forcetype if recursive == self.RECURSE_OTHERS else None
  87. def getter(i, node):
  88. for ch in self._get_children(node, restrict=restrict):
  89. yield (i, ch)
  90. inodes = chain(*(getter(i, n) for i, n in enumerate(self.nodes)))
  91. else:
  92. inodes = enumerate(self.nodes)
  93. for i, node in inodes:
  94. if (not forcetype or isinstance(node, forcetype)) and match(node):
  95. yield (i, node)
  96. def _do_strong_search(self, obj, recursive=True):
  97. """Search for the specific element *obj* within the node list.
  98. *obj* can be either a :class:`.Node` or a :class:`.Wikicode` object. If
  99. found, we return a tuple (*context*, *index*) where *context* is the
  100. :class:`.Wikicode` that contains *obj* and *index* is its index there,
  101. as a :class:`slice`. Note that if *recursive* is ``False``, *context*
  102. will always be ``self`` (since we only look for *obj* among immediate
  103. descendants), but if *recursive* is ``True``, then it could be any
  104. :class:`.Wikicode` contained by a node within ``self``. If *obj* is not
  105. found, :exc:`ValueError` is raised.
  106. """
  107. if isinstance(obj, Node):
  108. mkslice = lambda i: slice(i, i + 1)
  109. if not recursive:
  110. return self, mkslice(self.index(obj))
  111. for i, node in enumerate(self.nodes):
  112. for context, child in self._get_children(node, contexts=True):
  113. if obj is child:
  114. if not context:
  115. context = self
  116. return context, mkslice(context.index(child))
  117. raise ValueError(obj)
  118. context, ind = self._do_strong_search(obj.get(0), recursive)
  119. for i in range(1, len(obj.nodes)):
  120. if obj.get(i) is not context.get(ind.start + i):
  121. raise ValueError(obj)
  122. return context, slice(ind.start, ind.start + len(obj.nodes))
  123. def _do_weak_search(self, obj, recursive):
  124. """Search for an element that looks like *obj* within the node list.
  125. This follows the same rules as :meth:`_do_strong_search` with some
  126. differences. *obj* is treated as a string that might represent any
  127. :class:`.Node`, :class:`.Wikicode`, or combination of the two present
  128. in the node list. Thus, matching is weak (using string comparisons)
  129. rather than strong (using ``is``). Because multiple nodes can match
  130. *obj*, the result is a list of tuples instead of just one (however,
  131. :exc:`ValueError` is still raised if nothing is found). Individual
  132. matches will never overlap.
  133. The tuples contain a new first element, *exact*, which is ``True`` if
  134. we were able to match *obj* exactly to one or more adjacent nodes, or
  135. ``False`` if we found *obj* inside a node or incompletely spanning
  136. multiple nodes.
  137. """
  138. obj = parse_anything(obj)
  139. if not obj or obj not in self:
  140. raise ValueError(obj)
  141. results = []
  142. contexts = [self]
  143. while contexts:
  144. context = contexts.pop()
  145. i = len(context.nodes) - 1
  146. while i >= 0:
  147. node = context.get(i)
  148. if obj.get(-1) == node:
  149. for j in range(-len(obj.nodes), -1):
  150. if obj.get(j) != context.get(i + j + 1):
  151. break
  152. else:
  153. i -= len(obj.nodes) - 1
  154. index = slice(i, i + len(obj.nodes))
  155. results.append((True, context, index))
  156. elif recursive and obj in node:
  157. contexts.extend(node.__children__())
  158. i -= 1
  159. if not results:
  160. if not recursive:
  161. raise ValueError(obj)
  162. results.append((False, self, slice(0, len(self.nodes))))
  163. return results
  164. def _get_tree(self, code, lines, marker, indent):
  165. """Build a tree to illustrate the way the Wikicode object was parsed.
  166. The method that builds the actual tree is ``__showtree__`` of ``Node``
  167. objects. *code* is the ``Wikicode`` object to build a tree for. *lines*
  168. is the list to append the tree to, which is returned at the end of the
  169. method. *marker* is some object to be used to indicate that the builder
  170. should continue on from the last line instead of starting a new one; it
  171. should be any object that can be tested for with ``is``. *indent* is
  172. the starting indentation.
  173. """
  174. def write(*args):
  175. """Write a new line following the proper indentation rules."""
  176. if lines and lines[-1] is marker: # Continue from the last line
  177. lines.pop() # Remove the marker
  178. last = lines.pop()
  179. lines.append(last + " ".join(args))
  180. else:
  181. lines.append(" " * 6 * indent + " ".join(args))
  182. get = lambda code: self._get_tree(code, lines, marker, indent + 1)
  183. mark = lambda: lines.append(marker)
  184. for node in code.nodes:
  185. node.__showtree__(write, get, mark)
  186. return lines
  187. @classmethod
  188. def _build_filter_methods(cls, **meths):
  189. """Given Node types, build the corresponding i?filter shortcuts.
  190. The should be given as keys storing the method's base name paired with
  191. values storing the corresponding :class:`.Node` type. For example, the
  192. dict may contain the pair ``("templates", Template)``, which will
  193. produce the methods :meth:`ifilter_templates` and
  194. :meth:`filter_templates`, which are shortcuts for
  195. :meth:`ifilter(forcetype=Template) <ifilter>` and
  196. :meth:`filter(forcetype=Template) <filter>`, respectively. These
  197. shortcuts are added to the class itself, with an appropriate docstring.
  198. """
  199. doc = """Iterate over {0}.
  200. This is equivalent to :meth:`{1}` with *forcetype* set to
  201. :class:`~{2.__module__}.{2.__name__}`.
  202. """
  203. make_ifilter = lambda ftype: (lambda self, *a, **kw:
  204. self.ifilter(forcetype=ftype, *a, **kw))
  205. make_filter = lambda ftype: (lambda self, *a, **kw:
  206. self.filter(forcetype=ftype, *a, **kw))
  207. for name, ftype in (meths.items() if py3k else meths.iteritems()):
  208. ifilter = make_ifilter(ftype)
  209. filter = make_filter(ftype)
  210. ifilter.__doc__ = doc.format(name, "ifilter", ftype)
  211. filter.__doc__ = doc.format(name, "filter", ftype)
  212. setattr(cls, "ifilter_" + name, ifilter)
  213. setattr(cls, "filter_" + name, filter)
  214. @property
  215. def nodes(self):
  216. """A list of :class:`.Node` objects.
  217. This is the internal data actually stored within a :class:`.Wikicode`
  218. object.
  219. """
  220. return self._nodes
  221. @nodes.setter
  222. def nodes(self, value):
  223. if not isinstance(value, list):
  224. value = parse_anything(value).nodes
  225. self._nodes = value
  226. def get(self, index):
  227. """Return the *index*\ th node within the list of nodes."""
  228. return self.nodes[index]
  229. def set(self, index, value):
  230. """Set the ``Node`` at *index* to *value*.
  231. Raises :exc:`IndexError` if *index* is out of range, or
  232. :exc:`ValueError` if *value* cannot be coerced into one :class:`.Node`.
  233. To insert multiple nodes at an index, use :meth:`get` with either
  234. :meth:`remove` and :meth:`insert` or :meth:`replace`.
  235. """
  236. nodes = parse_anything(value).nodes
  237. if len(nodes) > 1:
  238. raise ValueError("Cannot coerce multiple nodes into one index")
  239. if index >= len(self.nodes) or -1 * index > len(self.nodes):
  240. raise IndexError("List assignment index out of range")
  241. if nodes:
  242. self.nodes[index] = nodes[0]
  243. else:
  244. self.nodes.pop(index)
  245. def contains(self, obj):
  246. """Return whether this Wikicode object contains *obj*.
  247. If *obj* is a :class:`.Node` or :class:`.Wikicode` object, then we
  248. search for it exactly among all of our children, recursively.
  249. Otherwise, this method just uses :meth:`.__contains__` on the string.
  250. """
  251. if not isinstance(obj, (Node, Wikicode)):
  252. return obj in self
  253. try:
  254. self._do_strong_search(obj, recursive=True)
  255. except ValueError:
  256. return False
  257. return True
  258. def index(self, obj, recursive=False):
  259. """Return the index of *obj* in the list of nodes.
  260. Raises :exc:`ValueError` if *obj* is not found. If *recursive* is
  261. ``True``, we will look in all nodes of ours and their descendants, and
  262. return the index of our direct descendant node within *our* list of
  263. nodes. Otherwise, the lookup is done only on direct descendants.
  264. """
  265. strict = isinstance(obj, Node)
  266. equivalent = (lambda o, n: o is n) if strict else (lambda o, n: o == n)
  267. for i, node in enumerate(self.nodes):
  268. if recursive:
  269. for child in self._get_children(node):
  270. if equivalent(obj, child):
  271. return i
  272. elif equivalent(obj, node):
  273. return i
  274. raise ValueError(obj)
  275. def get_ancestors(self, obj):
  276. """Return a list of all ancestor nodes of the :class:`.Node` *obj*.
  277. The list is ordered from the most shallow ancestor (greatest great-
  278. grandparent) to the direct parent. The node itself is not included in
  279. the list. For example::
  280. >>> text = "{{a|{{b|{{c|{{d}}}}}}}}"
  281. >>> code = mwparserfromhell.parse(text)
  282. >>> node = code.filter_templates(matches=lambda n: n == "{{d}}")[0]
  283. >>> code.get_ancestors(node)
  284. ['{{a|{{b|{{c|{{d}}}}}}}}', '{{b|{{c|{{d}}}}}}', '{{c|{{d}}}}']
  285. Will return an empty list if *obj* is at the top level of this Wikicode
  286. object. Will raise :exc:`ValueError` if it wasn't found.
  287. """
  288. def _get_ancestors(code, needle):
  289. for node in code.nodes:
  290. if node is needle:
  291. return []
  292. for code in node.__children__():
  293. ancestors = _get_ancestors(code, needle)
  294. if ancestors is not None:
  295. return [node] + ancestors
  296. if isinstance(obj, Wikicode):
  297. obj = obj.get(0)
  298. elif not isinstance(obj, Node):
  299. raise ValueError(obj)
  300. ancestors = _get_ancestors(self, obj)
  301. if ancestors is None:
  302. raise ValueError(obj)
  303. return ancestors
  304. def get_parent(self, obj):
  305. """Return the direct parent node of the :class:`.Node` *obj*.
  306. This function is equivalent to calling :meth:`.get_ancestors` and
  307. taking the last element of the resulting list. Will return None if
  308. the node exists but does not have a parent; i.e., it is at the top
  309. level of the Wikicode object.
  310. """
  311. ancestors = self.get_ancestors(obj)
  312. return ancestors[-1] if ancestors else None
  313. def insert(self, index, value):
  314. """Insert *value* at *index* in the list of nodes.
  315. *value* can be anything parsable by :func:`.parse_anything`, which
  316. includes strings or other :class:`.Wikicode` or :class:`.Node` objects.
  317. """
  318. nodes = parse_anything(value).nodes
  319. for node in reversed(nodes):
  320. self.nodes.insert(index, node)
  321. def insert_before(self, obj, value, recursive=True):
  322. """Insert *value* immediately before *obj*.
  323. *obj* can be either a string, a :class:`.Node`, or another
  324. :class:`.Wikicode` object (as created by :meth:`get_sections`, for
  325. example). If *obj* is a string, we will operate on all instances of
  326. that string within the code, otherwise only on the specific instance
  327. given. *value* can be anything parsable by :func:`.parse_anything`. If
  328. *recursive* is ``True``, we will try to find *obj* within our child
  329. nodes even if it is not a direct descendant of this :class:`.Wikicode`
  330. object. If *obj* is not found, :exc:`ValueError` is raised.
  331. """
  332. if isinstance(obj, (Node, Wikicode)):
  333. context, index = self._do_strong_search(obj, recursive)
  334. context.insert(index.start, value)
  335. else:
  336. for exact, context, index in self._do_weak_search(obj, recursive):
  337. if exact:
  338. context.insert(index.start, value)
  339. else:
  340. obj = str(obj)
  341. self._slice_replace(context, index, obj, str(value) + obj)
  342. def insert_after(self, obj, value, recursive=True):
  343. """Insert *value* immediately after *obj*.
  344. *obj* can be either a string, a :class:`.Node`, or another
  345. :class:`.Wikicode` object (as created by :meth:`get_sections`, for
  346. example). If *obj* is a string, we will operate on all instances of
  347. that string within the code, otherwise only on the specific instance
  348. given. *value* can be anything parsable by :func:`.parse_anything`. If
  349. *recursive* is ``True``, we will try to find *obj* within our child
  350. nodes even if it is not a direct descendant of this :class:`.Wikicode`
  351. object. If *obj* is not found, :exc:`ValueError` is raised.
  352. """
  353. if isinstance(obj, (Node, Wikicode)):
  354. context, index = self._do_strong_search(obj, recursive)
  355. context.insert(index.stop, value)
  356. else:
  357. for exact, context, index in self._do_weak_search(obj, recursive):
  358. if exact:
  359. context.insert(index.stop, value)
  360. else:
  361. obj = str(obj)
  362. self._slice_replace(context, index, obj, obj + str(value))
  363. def replace(self, obj, value, recursive=True):
  364. """Replace *obj* with *value*.
  365. *obj* can be either a string, a :class:`.Node`, or another
  366. :class:`.Wikicode` object (as created by :meth:`get_sections`, for
  367. example). If *obj* is a string, we will operate on all instances of
  368. that string within the code, otherwise only on the specific instance
  369. given. *value* can be anything parsable by :func:`.parse_anything`.
  370. If *recursive* is ``True``, we will try to find *obj* within our child
  371. nodes even if it is not a direct descendant of this :class:`.Wikicode`
  372. object. If *obj* is not found, :exc:`ValueError` is raised.
  373. """
  374. if isinstance(obj, (Node, Wikicode)):
  375. context, index = self._do_strong_search(obj, recursive)
  376. for i in range(index.start, index.stop):
  377. context.nodes.pop(index.start)
  378. context.insert(index.start, value)
  379. else:
  380. for exact, context, index in self._do_weak_search(obj, recursive):
  381. if exact:
  382. for i in range(index.start, index.stop):
  383. context.nodes.pop(index.start)
  384. context.insert(index.start, value)
  385. else:
  386. self._slice_replace(context, index, str(obj), str(value))
  387. def append(self, value):
  388. """Insert *value* at the end of the list of nodes.
  389. *value* can be anything parsable by :func:`.parse_anything`.
  390. """
  391. nodes = parse_anything(value).nodes
  392. for node in nodes:
  393. self.nodes.append(node)
  394. def remove(self, obj, recursive=True):
  395. """Remove *obj* from the list of nodes.
  396. *obj* can be either a string, a :class:`.Node`, or another
  397. :class:`.Wikicode` object (as created by :meth:`get_sections`, for
  398. example). If *obj* is a string, we will operate on all instances of
  399. that string within the code, otherwise only on the specific instance
  400. given. If *recursive* is ``True``, we will try to find *obj* within our
  401. child nodes even if it is not a direct descendant of this
  402. :class:`.Wikicode` object. If *obj* is not found, :exc:`ValueError` is
  403. raised.
  404. """
  405. if isinstance(obj, (Node, Wikicode)):
  406. context, index = self._do_strong_search(obj, recursive)
  407. for i in range(index.start, index.stop):
  408. context.nodes.pop(index.start)
  409. else:
  410. for exact, context, index in self._do_weak_search(obj, recursive):
  411. if exact:
  412. for i in range(index.start, index.stop):
  413. context.nodes.pop(index.start)
  414. else:
  415. self._slice_replace(context, index, str(obj), "")
  416. def matches(self, other):
  417. """Do a loose equivalency test suitable for comparing page names.
  418. *other* can be any string-like object, including :class:`.Wikicode`, or
  419. an iterable of these. This operation is symmetric; both sides are
  420. adjusted. Specifically, whitespace and markup is stripped and the first
  421. letter's case is normalized. Typical usage is
  422. ``if template.name.matches("stub"): ...``.
  423. """
  424. cmp = lambda a, b: (a[0].upper() + a[1:] == b[0].upper() + b[1:]
  425. if a and b else a == b)
  426. this = self.strip_code().strip()
  427. if isinstance(other, (str, bytes, Wikicode, Node)):
  428. that = parse_anything(other).strip_code().strip()
  429. return cmp(this, that)
  430. for obj in other:
  431. that = parse_anything(obj).strip_code().strip()
  432. if cmp(this, that):
  433. return True
  434. return False
  435. def ifilter(self, recursive=True, matches=None, flags=FLAGS,
  436. forcetype=None):
  437. """Iterate over nodes in our list matching certain conditions.
  438. If *forcetype* is given, only nodes that are instances of this type (or
  439. tuple of types) are yielded. Setting *recursive* to ``True`` will
  440. iterate over all children and their descendants. ``RECURSE_OTHERS``
  441. will only iterate over children that are not the instances of
  442. *forcetype*. ``False`` will only iterate over immediate children.
  443. ``RECURSE_OTHERS`` can be used to iterate over all un-nested templates,
  444. even if they are inside of HTML tags, like so:
  445. >>> code = mwparserfromhell.parse("{{foo}}<b>{{foo|{{bar}}}}</b>")
  446. >>> code.filter_templates(code.RECURSE_OTHERS)
  447. ["{{foo}}", "{{foo|{{bar}}}}"]
  448. *matches* can be used to further restrict the nodes, either as a
  449. function (taking a single :class:`.Node` and returning a boolean) or a
  450. regular expression (matched against the node's string representation
  451. with :func:`re.search`). If *matches* is a regex, the flags passed to
  452. :func:`re.search` are :const:`re.IGNORECASE`, :const:`re.DOTALL`, and
  453. :const:`re.UNICODE`, but custom flags can be specified by passing
  454. *flags*.
  455. """
  456. gen = self._indexed_ifilter(recursive, matches, flags, forcetype)
  457. return (node for i, node in gen)
  458. def filter(self, *args, **kwargs):
  459. """Return a list of nodes within our list matching certain conditions.
  460. This is equivalent to calling :func:`list` on :meth:`ifilter`.
  461. """
  462. return list(self.ifilter(*args, **kwargs))
  463. def get_sections(self, levels=None, matches=None, flags=FLAGS, flat=False,
  464. include_lead=None, include_headings=True):
  465. """Return a list of sections within the page.
  466. Sections are returned as :class:`.Wikicode` objects with a shared node
  467. list (implemented using :class:`.SmartList`) so that changes to
  468. sections are reflected in the parent Wikicode object.
  469. Each section contains all of its subsections, unless *flat* is
  470. ``True``. If *levels* is given, it should be a iterable of integers;
  471. only sections whose heading levels are within it will be returned. If
  472. *matches* is given, it should be either a function or a regex; only
  473. sections whose headings match it (without the surrounding equal signs)
  474. will be included. *flags* can be used to override the default regex
  475. flags (see :meth:`ifilter`) if a regex *matches* is used.
  476. If *include_lead* is ``True``, the first, lead section (without a
  477. heading) will be included in the list; ``False`` will not include it;
  478. the default will include it only if no specific *levels* were given. If
  479. *include_headings* is ``True``, the section's beginning
  480. :class:`.Heading` object will be included; otherwise, this is skipped.
  481. """
  482. title_matcher = self._build_matcher(matches, flags)
  483. matcher = lambda heading: (title_matcher(heading.title) and
  484. (not levels or heading.level in levels))
  485. iheadings = self._indexed_ifilter(recursive=False, forcetype=Heading)
  486. sections = [] # Tuples of (index_of_first_node, section)
  487. open_headings = [] # Tuples of (index, heading), where index and
  488. # heading.level are both monotonically increasing
  489. # Add the lead section if appropriate:
  490. if include_lead or not (include_lead is not None or matches or levels):
  491. itr = self._indexed_ifilter(recursive=False, forcetype=Heading)
  492. try:
  493. first = next(itr)[0]
  494. sections.append((0, Wikicode(self.nodes[:first])))
  495. except StopIteration: # No headings in page
  496. sections.append((0, Wikicode(self.nodes[:])))
  497. # Iterate over headings, adding sections to the list as they end:
  498. for i, heading in iheadings:
  499. if flat: # With flat, all sections close at the next heading
  500. newly_closed, open_headings = open_headings, []
  501. else: # Otherwise, figure out which sections have closed, if any
  502. closed_start_index = len(open_headings)
  503. for j, (start, last_heading) in enumerate(open_headings):
  504. if heading.level <= last_heading.level:
  505. closed_start_index = j
  506. break
  507. newly_closed = open_headings[closed_start_index:]
  508. del open_headings[closed_start_index:]
  509. for start, closed_heading in newly_closed:
  510. if matcher(closed_heading):
  511. sections.append((start, Wikicode(self.nodes[start:i])))
  512. start = i if include_headings else (i + 1)
  513. open_headings.append((start, heading))
  514. # Add any remaining open headings to the list of sections:
  515. for start, heading in open_headings:
  516. if matcher(heading):
  517. sections.append((start, Wikicode(self.nodes[start:])))
  518. # Ensure that earlier sections are earlier in the returned list:
  519. return [section for i, section in sorted(sections)]
  520. def strip_code(self, normalize=True, collapse=True,
  521. keep_template_params=False):
  522. """Return a rendered string without unprintable code such as templates.
  523. The way a node is stripped is handled by the
  524. :meth:`~.Node.__strip__` method of :class:`.Node` objects, which
  525. generally return a subset of their nodes or ``None``. For example,
  526. templates and tags are removed completely, links are stripped to just
  527. their display part, headings are stripped to just their title.
  528. If *normalize* is ``True``, various things may be done to strip code
  529. further, such as converting HTML entities like ``&Sigma;``, ``&#931;``,
  530. and ``&#x3a3;`` to ``Σ``. If *collapse* is ``True``, we will try to
  531. remove excess whitespace as well (three or more newlines are converted
  532. to two, for example). If *keep_template_params* is ``True``, then
  533. template parameters will be preserved in the output (normally, they are
  534. removed completely).
  535. """
  536. kwargs = {
  537. "normalize": normalize,
  538. "collapse": collapse,
  539. "keep_template_params": keep_template_params
  540. }
  541. nodes = []
  542. for node in self.nodes:
  543. stripped = node.__strip__(**kwargs)
  544. if stripped:
  545. nodes.append(str(stripped))
  546. if collapse:
  547. stripped = "".join(nodes).strip("\n")
  548. while "\n\n\n" in stripped:
  549. stripped = stripped.replace("\n\n\n", "\n\n")
  550. return stripped
  551. else:
  552. return "".join(nodes)
  553. def get_tree(self):
  554. """Return a hierarchical tree representation of the object.
  555. The representation is a string makes the most sense printed. It is
  556. built by calling :meth:`_get_tree` on the :class:`.Wikicode` object and
  557. its children recursively. The end result may look something like the
  558. following::
  559. >>> text = "Lorem ipsum {{foo|bar|{{baz}}|spam=eggs}}"
  560. >>> print(mwparserfromhell.parse(text).get_tree())
  561. Lorem ipsum
  562. {{
  563. foo
  564. | 1
  565. = bar
  566. | 2
  567. = {{
  568. baz
  569. }}
  570. | spam
  571. = eggs
  572. }}
  573. """
  574. marker = object() # Random object we can find with certainty in a list
  575. return "\n".join(self._get_tree(self, [], marker, 0))
  576. Wikicode._build_filter_methods(
  577. arguments=Argument, comments=Comment, external_links=ExternalLink,
  578. headings=Heading, html_entities=HTMLEntity, tags=Tag, templates=Template,
  579. text=Text, wikilinks=Wikilink)