Wikilinks in builder; some additions; cleanup.

docs/api/mwparserfromhell.nodes.rst

@@ -17,6 +17,14 @@ nodes Package
     :undoc-members:
     :show-inheritance:
 
+:mod:`comment` Module
+---------------------
+
+.. automodule:: mwparserfromhell.nodes.comment
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
 :mod:`heading` Module
 ---------------------
 
@@ -56,6 +64,14 @@ nodes Package
     :undoc-members:
     :show-inheritance:
 
+:mod:`wikilink` Module
+----------------------
+
+.. automodule:: mwparserfromhell.nodes.wikilink
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
 Subpackages
 -----------
 

mwparserfromhell/nodes/__init__.py

@@ -73,3 +73,4 @@ from .heading import Heading
 from .html_entity import HTMLEntity
 from .tag import Tag
 from .template import Template
+from .wikilink import Wikilink

mwparserfromhell/parser/builder.py

@@ -24,7 +24,8 @@ from __future__ import unicode_literals
 
 from . import tokens
 from ..compat import str
-from ..nodes import Argument, Comment, Heading, HTMLEntity, Tag, Template, Text
+from ..nodes import (Argument, Comment, Heading, HTMLEntity, Tag, Template,
+                     Text, Wikilink)
 from ..nodes.extras import Attribute, Parameter
 from ..smart_list import SmartList
 from ..wikicode import Wikicode
@@ -125,6 +126,22 @@ class Builder(object):
             else:
                 self._write(self._handle_token(token))
 
+    def _handle_wikilink(self):
+        """Handle a case where a wikilink is at the head of the tokens."""
+        title = None
+        self._push()
+        while self._tokens:
+            token = self._tokens.pop()
+            if isinstance(token, tokens.WikilinkSeparator):
+                title = self._pop()
+                self._push()
+            elif isinstance(token, tokens.WikilinkClose):
+                if title is not None:
+                    return Wikilink(title, self._pop())
+                return Wikilink(self._pop())
+            else:
+                self._write(self._handle_token(token))
+
     def _handle_entity(self):
         """Handle a case where an HTML entity is at the head of the tokens."""
         token = self._tokens.pop()
@@ -216,6 +233,8 @@ class Builder(object):
             return self._handle_template()
         elif isinstance(token, tokens.ArgumentOpen):
             return self._handle_argument()
+        elif isinstance(token, tokens.WikilinkOpen):
+            return self._handle_wikilink()
         elif isinstance(token, tokens.HTMLEntityStart):
             return self._handle_entity()
         elif isinstance(token, tokens.HeadingStart):

mwparserfromhell/parser/tokens.py

@@ -65,38 +65,43 @@ class Token(object):
 
 def make(name):
     """Create a new Token class using ``type()`` and add it to ``__all__``."""
+    token = type(name if py3k else name.encode("utf8"), (Token,), {})
+    globals()[name] = token
     __all__.append(name)
-    return type(name if py3k else name.encode("utf8"), (Token,), {})
-
-Text = make("Text")
-
-TemplateOpen = make("TemplateOpen")                                 # {{
-TemplateParamSeparator = make("TemplateParamSeparator")             # |
-TemplateParamEquals = make("TemplateParamEquals")                   # =
-TemplateClose = make("TemplateClose")                               # }}
-
-ArgumentOpen = make("ArgumentOpen")                                 # {{{
-ArgumentSeparator = make("ArgumentSeparator")                       # |
-ArgumentClose = make("ArgumentClose")                               # }}}
-
-HTMLEntityStart = make("HTMLEntityStart")                           # &
-HTMLEntityNumeric = make("HTMLEntityNumeric")                       # #
-HTMLEntityHex = make("HTMLEntityHex")                               # x
-HTMLEntityEnd = make("HTMLEntityEnd")                               # ;
-
-HeadingStart = make("HeadingStart")                                 # =...
-HeadingEnd = make("HeadingEnd")                                     # =...
-
-CommentStart = make("CommentStart")                                 # <!--
-CommentEnd = make("CommentEnd")                                     # -->
-
-TagOpenOpen = make("TagOpenOpen")                                   # <
-TagAttrStart = make("TagAttrStart")
-TagAttrEquals = make("TagAttrEquals")                               # =
-TagAttrQuote = make("TagAttrQuote")                                 # "
-TagCloseOpen = make("TagCloseOpen")                                 # >
-TagCloseSelfclose = make("TagCloseSelfclose")                       # />
-TagOpenClose = make("TagOpenClose")                                 # </
-TagCloseClose = make("TagCloseClose")                               # >
+
+make("Text")
+
+make("TemplateOpen")                                                # {{
+make("TemplateParamSeparator")                                      # |
+make("TemplateParamEquals")                                         # =
+make("TemplateClose")                                               # }}
+
+make("ArgumentOpen")                                                # {{{
+make("ArgumentSeparator")                                           # |
+make("ArgumentClose")                                               # }}}
+
+make("WikilinkOpen")                                                # [[
+make("WikilinkSeparator")                                           # |
+make("WikilinkClose")                                               # ]]
+
+make("HTMLEntityStart")                                             # &
+make("HTMLEntityNumeric")                                           # #
+make("HTMLEntityHex")                                               # x
+make("HTMLEntityEnd")                                               # ;
+
+make("HeadingStart")                                                # =...
+make("HeadingEnd")                                                  # =...
+
+make("CommentStart")                                                # <!--
+make("CommentEnd")                                                  # -->
+
+make("TagOpenOpen")                                                 # <
+make("TagAttrStart")
+make("TagAttrEquals")                                               # =
+make("TagAttrQuote")                                                # "
+make("TagCloseOpen")                                                # >
+make("TagCloseSelfclose")                                           # />
+make("TagOpenClose")                                                # </
+make("TagCloseClose")                                               # >
 
 del make

mwparserfromhell/wikicode.py

@@ -24,7 +24,7 @@ from __future__ import unicode_literals
 import re
 
 from .compat import maxsize, str
-from .nodes import Heading, Node, Tag, Template, Text
+from .nodes import Heading, Node, Tag, Template, Text, Wikilink
 from .string_mixin import StringMixIn
 from .utils import parse_anything
 
@@ -303,6 +303,14 @@ class Wikicode(StringMixIn):
                 if not matches or re.search(matches, str(node), flags):
                     yield node
 
+    def ifilter_links(self, recursive=False, matches=None, flags=FLAGS):
+        """Iterate over wikilink nodes.
+
+        This is equivalent to :py:meth:`ifilter` with *forcetype* set to
+        :py:class:`~.Wikilink`.
+        """
+        return self.ifilter(recursive, matches, flags, forcetype=Wikilink)
+
     def ifilter_templates(self, recursive=False, matches=None, flags=FLAGS):
         """Iterate over template nodes.
 
@@ -335,6 +343,14 @@ class Wikicode(StringMixIn):
         """
         return list(self.ifilter(recursive, matches, flags, forcetype))
 
+    def filter_links(self, recursive=False, matches=None, flags=FLAGS):
+        """Return a list of wikilink nodes.
+
+        This is equivalent to calling :py:func:`list` on
+        :py:meth:`ifilter_links`.
+        """
+        return list(self.ifilter_links(recursive, matches, flags))
+
     def filter_templates(self, recursive=False, matches=None, flags=FLAGS):
         """Return a list of template nodes.