Merge develop into master (release/0.4.4)

.travis.yml

@@ -6,6 +6,8 @@ python:
     - 3.3
     - 3.4
     - 3.5
+    - 3.6
+    - nightly
 sudo: false
 install:
     - if [[ $TRAVIS_PYTHON_VERSION == '3.2' ]]; then pip install coverage==3.7.1; fi

CHANGELOG

@@ -1,3 +1,14 @@
+v0.4.4 (released December 30, 2016):
+
+- Added support for Python 3.6.
+- Fixed parsing bugs involving:
+  - wikitables nested in templates;
+  - wikitable error recovery when unable to recurse;
+  - templates nested in template parameters before other parameters.
+- Fixed parsing file-like objects.
+- Made builds deterministic.
+- Documented caveats.
+
 v0.4.3 (released October 29, 2015):
 
 - Added Windows binaries for Python 3.5.

LICENSE

@@ -1,4 +1,4 @@
-Copyright (C) 2012-2015 Ben Kurtovic <ben.kurtovic@gmail.com>
+Copyright (C) 2012-2016 Ben Kurtovic <ben.kurtovic@gmail.com>
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

README.rst

@@ -21,8 +21,7 @@ Installation
 
 The easiest way to install the parser is through the `Python Package Index`_;
 you can install the latest release with ``pip install mwparserfromhell``
-(`get pip`_). On Windows, make sure you have the latest version of pip
-installed by running ``pip install --upgrade pip``.
+(`get pip`_). Make sure your pip is up-to-date first, especially on Windows.
 
 Alternatively, get the latest development version::
 
@@ -114,6 +113,24 @@ saving the page!) by calling ``str()`` on it::
 
 Likewise, use ``unicode(code)`` in Python 2.
 
+Caveats
+-------
+
+An inherent limitation in wikicode prevents us from generating complete parse
+trees in certain cases. For example, the string ``{{echo|''Hello}}, world!''``
+produces the valid output ``<i>Hello, world!</i>`` in MediaWiki, assuming
+``{{echo}}`` is a template that returns its first parameter. But since
+representing this in mwparserfromhell's node tree would be impossible, we
+compromise by treating the first node (i.e., the template) as plain text,
+parsing only the italics.
+
+The current workaround for cases where you are not interested in text
+formatting is to pass ``skip_style_tags=True`` to ``mwparserfromhell.parse()``.
+This treats ``''`` and ``'''`` like plain text.
+
+A future version of mwparserfromhell will include multiple parsing modes to get
+around this restriction.
+
 Integration
 -----------
 
@@ -132,8 +149,8 @@ If you're using Pywikibot_, your code might look like this::
         text = page.get()
         return mwparserfromhell.parse(text)
 
-If you're not using a library, you can parse any page using the following code
-(via the API_)::
+If you're not using a library, you can parse any page using the following
+Python 3 code (via the API_)::
 
     import json
     from urllib.parse import urlencode

appveyor.yml

@@ -1,6 +1,6 @@
 # This config file is used by appveyor.com to build Windows release binaries
 
-version: 0.4.3-b{build}
+version: 0.4.4-b{build}
 
 branches:
   only:
@@ -15,7 +15,6 @@ environment:
     WRAPPER: "cmd /E:ON /V:ON /C .\\scripts\\win_wrapper.cmd"
     PIP:     "%WRAPPER% %PYTHON%\\Scripts\\pip.exe"
     SETUPPY: "%WRAPPER% %PYTHON%\\python setup.py --with-extension"
-    PYMOD:   "%WRAPPER% %PYTHON%\\python -m"
     PYPI_USERNAME: "earwigbot"
     PYPI_PASSWORD:
       secure: gOIcvPxSC2ujuhwOzwj3v8xjq3CCYd8keFWVnguLM+gcL0e02qshDHy7gwZZwj0+
@@ -67,7 +66,7 @@ after_test:
   - "%SETUPPY% bdist_wheel"
 
 on_success:
-  - "IF %APPVEYOR_REPO_BRANCH%==master %PYMOD% twine upload dist\\* -u %PYPI_USERNAME% -p %PYPI_PASSWORD%"
+  - "IF %APPVEYOR_REPO_BRANCH%==master %WRAPPER% %PYTHON%\\python -m twine upload dist\\* -u %PYPI_USERNAME% -p %PYPI_PASSWORD%"
 
 artifacts:
   - path: dist\*

docs/caveats.rst

@@ -0,0 +1,17 @@
+Caveats
+=======
+
+An inherent limitation in wikicode prevents us from generating complete parse
+trees in certain cases. For example, the string ``{{echo|''Hello}}, world!''``
+produces the valid output ``<i>Hello, world!</i>`` in MediaWiki, assuming
+``{{echo}}`` is a template that returns its first parameter. But since
+representing this in mwparserfromhell's node tree would be impossible, we
+compromise by treating the first node (i.e., the template) as plain text,
+parsing only the italics.
+
+The current workaround for cases where you are not interested in text
+formatting is to pass *skip_style_tags=True* to :func:`mwparserfromhell.parse`.
+This treats ``''`` and ``'''`` like plain text.
+
+A future version of mwparserfromhell will include multiple parsing modes to get
+around this restriction.

docs/changelog.rst

@@ -1,6 +1,23 @@
 Changelog
 =========
 
+v0.4.4
+------
+
+`Released December 30, 2016 <https://github.com/earwig/mwparserfromhell/tree/v0.4.4>`_
+(`changes <https://github.com/earwig/mwparserfromhell/compare/v0.4.3...v0.4.4>`__):
+
+- Added support for Python 3.6.
+- Fixed parsing bugs involving:
+
+  - wikitables nested in templates;
+  - wikitable error recovery when unable to recurse;
+  - templates nested in template parameters before other parameters.
+
+- Fixed parsing file-like objects.
+- Made builds deterministic.
+- Documented caveats.
+
 v0.4.3
 ------
 

docs/conf.py

@@ -42,7 +42,7 @@ master_doc = 'index'
 
 # General information about the project.
 project = u'mwparserfromhell'
-copyright = u'2012, 2013, 2014, 2015 Ben Kurtovic'
+copyright = u'2012, 2013, 2014, 2015, 2016 Ben Kurtovic'
 
 # The version info for the project you're documenting, acts as replacement for
 # |version| and |release|, also used in various other places throughout the

docs/index.rst

@@ -19,8 +19,7 @@ Installation
 
 The easiest way to install the parser is through the `Python Package Index`_;
 you can install the latest release with ``pip install mwparserfromhell``
-(`get pip`_). On Windows, make sure you have the latest version of pip
-installed by running ``pip install --upgrade pip``.
+(`get pip`_). Make sure your pip is up-to-date first, especially on Windows.
 
 Alternatively, get the latest development version::
 
@@ -41,6 +40,7 @@ Contents
    :maxdepth: 2
 
    usage
+   caveats
    integration
    changelog
    API Reference <api/modules>

mwparserfromhell/__init__.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8  -*-
 #
-# Copyright (C) 2012-2015 Ben Kurtovic <ben.kurtovic@gmail.com>
+# Copyright (C) 2012-2016 Ben Kurtovic <ben.kurtovic@gmail.com>
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy
 # of this software and associated documentation files (the "Software"), to deal
@@ -27,9 +27,9 @@ outrageously powerful parser for `MediaWiki <http://mediawiki.org>`_ wikicode.
 """
 
 __author__ = "Ben Kurtovic"
-__copyright__ = "Copyright (C) 2012, 2013, 2014, 2015 Ben Kurtovic"
+__copyright__ = "Copyright (C) 2012, 2013, 2014, 2015, 2016 Ben Kurtovic"
 __license__ = "MIT License"
-__version__ = "0.4.3"
+__version__ = "0.4.4"
 __email__ = "ben.kurtovic@gmail.com"
 
 from . import (compat, definitions, nodes, parser, smart_list, string_mixin,

mwparserfromhell/definitions.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8  -*-
 #
-# Copyright (C) 2012-2015 Ben Kurtovic <ben.kurtovic@gmail.com>
+# Copyright (C) 2012-2016 Ben Kurtovic <ben.kurtovic@gmail.com>
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy
 # of this software and associated documentation files (the "Software"), to deal

mwparserfromhell/nodes/__init__.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8  -*-
 #
-# Copyright (C) 2012-2015 Ben Kurtovic <ben.kurtovic@gmail.com>
+# Copyright (C) 2012-2016 Ben Kurtovic <ben.kurtovic@gmail.com>
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy
 # of this software and associated documentation files (the "Software"), to deal

mwparserfromhell/nodes/argument.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8  -*-
 #
-# Copyright (C) 2012-2015 Ben Kurtovic <ben.kurtovic@gmail.com>
+# Copyright (C) 2012-2016 Ben Kurtovic <ben.kurtovic@gmail.com>
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy
 # of this software and associated documentation files (the "Software"), to deal

mwparserfromhell/nodes/comment.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8  -*-
 #
-# Copyright (C) 2012-2015 Ben Kurtovic <ben.kurtovic@gmail.com>
+# Copyright (C) 2012-2016 Ben Kurtovic <ben.kurtovic@gmail.com>
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy
 # of this software and associated documentation files (the "Software"), to deal

mwparserfromhell/nodes/external_link.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8  -*-
 #
-# Copyright (C) 2012-2015 Ben Kurtovic <ben.kurtovic@gmail.com>
+# Copyright (C) 2012-2016 Ben Kurtovic <ben.kurtovic@gmail.com>
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy
 # of this software and associated documentation files (the "Software"), to deal

mwparserfromhell/nodes/extras/__init__.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8  -*-
 #
-# Copyright (C) 2012-2015 Ben Kurtovic <ben.kurtovic@gmail.com>
+# Copyright (C) 2012-2016 Ben Kurtovic <ben.kurtovic@gmail.com>
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy
 # of this software and associated documentation files (the "Software"), to deal

mwparserfromhell/nodes/extras/attribute.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8  -*-
 #
-# Copyright (C) 2012-2015 Ben Kurtovic <ben.kurtovic@gmail.com>
+# Copyright (C) 2012-2016 Ben Kurtovic <ben.kurtovic@gmail.com>
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy
 # of this software and associated documentation files (the "Software"), to deal

mwparserfromhell/nodes/extras/parameter.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8  -*-
 #
-# Copyright (C) 2012-2015 Ben Kurtovic <ben.kurtovic@gmail.com>
+# Copyright (C) 2012-2016 Ben Kurtovic <ben.kurtovic@gmail.com>
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy
 # of this software and associated documentation files (the "Software"), to deal

mwparserfromhell/nodes/heading.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8  -*-
 #
-# Copyright (C) 2012-2015 Ben Kurtovic <ben.kurtovic@gmail.com>
+# Copyright (C) 2012-2016 Ben Kurtovic <ben.kurtovic@gmail.com>
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy
 # of this software and associated documentation files (the "Software"), to deal

mwparserfromhell/nodes/html_entity.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8  -*-
 #
-# Copyright (C) 2012-2015 Ben Kurtovic <ben.kurtovic@gmail.com>
+# Copyright (C) 2012-2016 Ben Kurtovic <ben.kurtovic@gmail.com>
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy
 # of this software and associated documentation files (the "Software"), to deal

mwparserfromhell/nodes/tag.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8  -*-
 #
-# Copyright (C) 2012-2015 Ben Kurtovic <ben.kurtovic@gmail.com>
+# Copyright (C) 2012-2016 Ben Kurtovic <ben.kurtovic@gmail.com>
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy
 # of this software and associated documentation files (the "Software"), to deal

mwparserfromhell/nodes/template.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8  -*-
 #
-# Copyright (C) 2012-2015 Ben Kurtovic <ben.kurtovic@gmail.com>
+# Copyright (C) 2012-2016 Ben Kurtovic <ben.kurtovic@gmail.com>
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy
 # of this software and associated documentation files (the "Software"), to deal

mwparserfromhell/nodes/text.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8  -*-
 #
-# Copyright (C) 2012-2015 Ben Kurtovic <ben.kurtovic@gmail.com>
+# Copyright (C) 2012-2016 Ben Kurtovic <ben.kurtovic@gmail.com>
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy
 # of this software and associated documentation files (the "Software"), to deal

mwparserfromhell/nodes/wikilink.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8  -*-
 #
-# Copyright (C) 2012-2015 Ben Kurtovic <ben.kurtovic@gmail.com>
+# Copyright (C) 2012-2016 Ben Kurtovic <ben.kurtovic@gmail.com>
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy
 # of this software and associated documentation files (the "Software"), to deal

mwparserfromhell/parser/__init__.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8  -*-
 #
-# Copyright (C) 2012-2015 Ben Kurtovic <ben.kurtovic@gmail.com>
+# Copyright (C) 2012-2016 Ben Kurtovic <ben.kurtovic@gmail.com>
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy
 # of this software and associated documentation files (the "Software"), to deal

mwparserfromhell/parser/builder.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8  -*-
 #
-# Copyright (C) 2012-2015 Ben Kurtovic <ben.kurtovic@gmail.com>
+# Copyright (C) 2012-2016 Ben Kurtovic <ben.kurtovic@gmail.com>
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy
 # of this software and associated documentation files (the "Software"), to deal

mwparserfromhell/parser/contexts.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8  -*-
 #
-# Copyright (C) 2012-2015 Ben Kurtovic <ben.kurtovic@gmail.com>
+# Copyright (C) 2012-2016 Ben Kurtovic <ben.kurtovic@gmail.com>
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy
 # of this software and associated documentation files (the "Software"), to deal

mwparserfromhell/parser/ctokenizer/common.h

@@ -1,5 +1,5 @@
 /*
-Copyright (C) 2012-2015 Ben Kurtovic <ben.kurtovic@gmail.com>
+Copyright (C) 2012-2016 Ben Kurtovic <ben.kurtovic@gmail.com>
 
 Permission is hereby granted, free of charge, to any person obtaining a copy of
 this software and associated documentation files (the "Software"), to deal in

mwparserfromhell/parser/ctokenizer/contexts.h

@@ -1,5 +1,5 @@
 /*
-Copyright (C) 2012-2015 Ben Kurtovic <ben.kurtovic@gmail.com>
+Copyright (C) 2012-2016 Ben Kurtovic <ben.kurtovic@gmail.com>
 
 Permission is hereby granted, free of charge, to any person obtaining a copy of
 this software and associated documentation files (the "Software"), to deal in

mwparserfromhell/parser/ctokenizer/definitions.c

@@ -1,5 +1,5 @@
 /*
-Copyright (C) 2012-2015 Ben Kurtovic <ben.kurtovic@gmail.com>
+Copyright (C) 2012-2016 Ben Kurtovic <ben.kurtovic@gmail.com>
 
 Permission is hereby granted, free of charge, to any person obtaining a copy of
 this software and associated documentation files (the "Software"), to deal in

mwparserfromhell/parser/ctokenizer/definitions.h

@@ -1,5 +1,5 @@
 /*
-Copyright (C) 2012-2015 Ben Kurtovic <ben.kurtovic@gmail.com>
+Copyright (C) 2012-2016 Ben Kurtovic <ben.kurtovic@gmail.com>
 
 Permission is hereby granted, free of charge, to any person obtaining a copy of
 this software and associated documentation files (the "Software"), to deal in

mwparserfromhell/parser/ctokenizer/tag_data.c

@@ -1,5 +1,5 @@
 /*
-Copyright (C) 2012-2015 Ben Kurtovic <ben.kurtovic@gmail.com>
+Copyright (C) 2012-2016 Ben Kurtovic <ben.kurtovic@gmail.com>
 
 Permission is hereby granted, free of charge, to any person obtaining a copy of
 this software and associated documentation files (the "Software"), to deal in

mwparserfromhell/parser/ctokenizer/tag_data.h

@@ -1,5 +1,5 @@
 /*
-Copyright (C) 2012-2015 Ben Kurtovic <ben.kurtovic@gmail.com>
+Copyright (C) 2012-2016 Ben Kurtovic <ben.kurtovic@gmail.com>
 
 Permission is hereby granted, free of charge, to any person obtaining a copy of
 this software and associated documentation files (the "Software"), to deal in

mwparserfromhell/parser/ctokenizer/textbuffer.c

@@ -1,5 +1,5 @@
 /*
-Copyright (C) 2012-2015 Ben Kurtovic <ben.kurtovic@gmail.com>
+Copyright (C) 2012-2016 Ben Kurtovic <ben.kurtovic@gmail.com>
 
 Permission is hereby granted, free of charge, to any person obtaining a copy of
 this software and associated documentation files (the "Software"), to deal in

mwparserfromhell/parser/ctokenizer/textbuffer.h

@@ -1,5 +1,5 @@
 /*
-Copyright (C) 2012-2015 Ben Kurtovic <ben.kurtovic@gmail.com>
+Copyright (C) 2012-2016 Ben Kurtovic <ben.kurtovic@gmail.com>
 
 Permission is hereby granted, free of charge, to any person obtaining a copy of
 this software and associated documentation files (the "Software"), to deal in

mwparserfromhell/parser/ctokenizer/tok_parse.c

@@ -1,5 +1,5 @@
 /*
-Copyright (C) 2012-2015 Ben Kurtovic <ben.kurtovic@gmail.com>
+Copyright (C) 2012-2016 Ben Kurtovic <ben.kurtovic@gmail.com>
 
 Permission is hereby granted, free of charge, to any person obtaining a copy of
 this software and associated documentation files (the "Software"), to deal in
@@ -249,7 +249,7 @@ static int Tokenizer_handle_template_param(Tokenizer* self)
     else if (self->topstack->context & LC_TEMPLATE_PARAM_VALUE)
         self->topstack->context ^= LC_TEMPLATE_PARAM_VALUE;
     if (self->topstack->context & LC_TEMPLATE_PARAM_KEY) {
-        stack = Tokenizer_pop_keeping_context(self);
+        stack = Tokenizer_pop(self);
         if (!stack)
             return -1;
         if (Tokenizer_emit_all(self, stack)) {
@@ -274,7 +274,7 @@ static int Tokenizer_handle_template_param_value(Tokenizer* self)
 {
     PyObject *stack;
 
-    stack = Tokenizer_pop_keeping_context(self);
+    stack = Tokenizer_pop(self);
     if (!stack)
         return -1;
     if (Tokenizer_emit_all(self, stack)) {
@@ -301,7 +301,7 @@ static PyObject* Tokenizer_handle_template_end(Tokenizer* self)
             return Tokenizer_fail_route(self);
     }
     else if (self->topstack->context & LC_TEMPLATE_PARAM_KEY) {
-        stack = Tokenizer_pop_keeping_context(self);
+        stack = Tokenizer_pop(self);
         if (!stack)
             return NULL;
         if (Tokenizer_emit_all(self, stack)) {
@@ -2190,7 +2190,7 @@ static PyObject* Tokenizer_handle_table_style(Tokenizer* self, Unicode end_token
 */
 static int Tokenizer_parse_table(Tokenizer* self)
 {
-    Py_ssize_t reset = self->head + 1;
+    Py_ssize_t reset = self->head;
     PyObject *style, *padding;
     PyObject *table = NULL;
     self->head += 2;
@@ -2201,7 +2201,7 @@ static int Tokenizer_parse_table(Tokenizer* self)
     if (BAD_ROUTE) {
         RESET_ROUTE();
         self->head = reset;
-        if (Tokenizer_emit_text(self, "{|"))
+        if (Tokenizer_emit_char(self, '{'))
             return -1;
         return 0;
     }
@@ -2220,7 +2220,7 @@ static int Tokenizer_parse_table(Tokenizer* self)
         Py_DECREF(padding);
         Py_DECREF(style);
         self->head = reset;
-        if (Tokenizer_emit_text(self, "{|"))
+        if (Tokenizer_emit_char(self, '{'))
             return -1;
         return 0;
     }
@@ -2689,10 +2689,8 @@ PyObject* Tokenizer_parse(Tokenizer* self, uint64_t context, int push)
                 if (Tokenizer_parse_table(self))
                     return NULL;
             }
-            else if (Tokenizer_emit_char(self, this) || Tokenizer_emit_char(self, next))
+            else if (Tokenizer_emit_char(self, this))
                 return NULL;
-            else
-                self->head++;
         }
         else if (this_context & LC_TABLE_OPEN) {
             if (this == '|' && next == '|' && this_context & LC_TABLE_TD_LINE) {

mwparserfromhell/parser/ctokenizer/tok_parse.h

@@ -1,5 +1,5 @@
 /*
-Copyright (C) 2012-2015 Ben Kurtovic <ben.kurtovic@gmail.com>
+Copyright (C) 2012-2016 Ben Kurtovic <ben.kurtovic@gmail.com>
 
 Permission is hereby granted, free of charge, to any person obtaining a copy of
 this software and associated documentation files (the "Software"), to deal in

mwparserfromhell/parser/ctokenizer/tok_support.c

@@ -1,5 +1,5 @@
 /*
-Copyright (C) 2012-2015 Ben Kurtovic <ben.kurtovic@gmail.com>
+Copyright (C) 2012-2016 Ben Kurtovic <ben.kurtovic@gmail.com>
 
 Permission is hereby granted, free of charge, to any person obtaining a copy of
 this software and associated documentation files (the "Software"), to deal in

mwparserfromhell/parser/ctokenizer/tok_support.h

@@ -1,5 +1,5 @@
 /*
-Copyright (C) 2012-2015 Ben Kurtovic <ben.kurtovic@gmail.com>
+Copyright (C) 2012-2016 Ben Kurtovic <ben.kurtovic@gmail.com>
 
 Permission is hereby granted, free of charge, to any person obtaining a copy of
 this software and associated documentation files (the "Software"), to deal in

mwparserfromhell/parser/ctokenizer/tokenizer.c

@@ -1,5 +1,5 @@
 /*
-Copyright (C) 2012-2015 Ben Kurtovic <ben.kurtovic@gmail.com>
+Copyright (C) 2012-2016 Ben Kurtovic <ben.kurtovic@gmail.com>
 
 Permission is hereby granted, free of charge, to any person obtaining a copy of
 this software and associated documentation files (the "Software"), to deal in

mwparserfromhell/parser/ctokenizer/tokenizer.h

@@ -1,5 +1,5 @@
 /*
-Copyright (C) 2012-2015 Ben Kurtovic <ben.kurtovic@gmail.com>
+Copyright (C) 2012-2016 Ben Kurtovic <ben.kurtovic@gmail.com>
 
 Permission is hereby granted, free of charge, to any person obtaining a copy of
 this software and associated documentation files (the "Software"), to deal in

mwparserfromhell/parser/ctokenizer/tokens.c

@@ -1,5 +1,5 @@
 /*
-Copyright (C) 2012-2015 Ben Kurtovic <ben.kurtovic@gmail.com>
+Copyright (C) 2012-2016 Ben Kurtovic <ben.kurtovic@gmail.com>
 
 Permission is hereby granted, free of charge, to any person obtaining a copy of
 this software and associated documentation files (the "Software"), to deal in

mwparserfromhell/parser/ctokenizer/tokens.h

@@ -1,5 +1,5 @@
 /*
-Copyright (C) 2012-2015 Ben Kurtovic <ben.kurtovic@gmail.com>
+Copyright (C) 2012-2016 Ben Kurtovic <ben.kurtovic@gmail.com>
 
 Permission is hereby granted, free of charge, to any person obtaining a copy of
 this software and associated documentation files (the "Software"), to deal in

mwparserfromhell/parser/tokenizer.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8  -*-
 #
-# Copyright (C) 2012-2015 Ben Kurtovic <ben.kurtovic@gmail.com>
+# Copyright (C) 2012-2016 Ben Kurtovic <ben.kurtovic@gmail.com>
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy
 # of this software and associated documentation files (the "Software"), to deal
@@ -264,14 +264,14 @@ class Tokenizer(object):
         elif self._context & contexts.TEMPLATE_PARAM_VALUE:
             self._context ^= contexts.TEMPLATE_PARAM_VALUE
         else:
-            self._emit_all(self._pop(keep_context=True))
+            self._emit_all(self._pop())
         self._context |= contexts.TEMPLATE_PARAM_KEY
         self._emit(tokens.TemplateParamSeparator())
         self._push(self._context)
 
     def _handle_template_param_value(self):
         """Handle a template parameter's value at the head of the string."""
-        self._emit_all(self._pop(keep_context=True))
+        self._emit_all(self._pop())
         self._context ^= contexts.TEMPLATE_PARAM_KEY
         self._context |= contexts.TEMPLATE_PARAM_VALUE
         self._emit(tokens.TemplateParamEquals())
@@ -282,7 +282,7 @@ class Tokenizer(object):
             if not self._context & (contexts.HAS_TEXT | contexts.HAS_TEMPLATE):
                 self._fail_route()
         elif self._context & contexts.TEMPLATE_PARAM_KEY:
-            self._emit_all(self._pop(keep_context=True))
+            self._emit_all(self._pop())
         self._head += 1
         return self._pop()
 
@@ -1074,14 +1074,14 @@ class Tokenizer(object):
 
     def _parse_table(self):
         """Parse a wikicode table by starting with the first line."""
-        reset = self._head + 1
+        reset = self._head
         self._head += 2
         self._push(contexts.TABLE_OPEN)
         try:
             padding = self._handle_table_style("\n")
         except BadRoute:
             self._head = reset
-            self._emit_text("{|")
+            self._emit_text("{")
             return
         style = self._pop()
 
@@ -1090,7 +1090,7 @@ class Tokenizer(object):
             table = self._parse(contexts.TABLE_OPEN)
         except BadRoute:
             self._head = reset
-            self._emit_text("{|")
+            self._emit_text("{")
             return
 
         self._emit_table_tag("{|", "table", style, padding, None, table, "|}")
@@ -1338,9 +1338,10 @@ class Tokenizer(object):
                 if result is not None:
                     return result
             elif self._read(-1) in ("\n", self.START) and this in ("#", "*", ";", ":"):
-                    self._handle_list()
-            elif self._read(-1) in ("\n", self.START) and this == next == self._read(2) == self._read(3) == "-":
-                    self._handle_hr()
+                self._handle_list()
+            elif self._read(-1) in ("\n", self.START) and (
+                    this == next == self._read(2) == self._read(3) == "-"):
+                self._handle_hr()
             elif this in ("\n", ":") and self._context & contexts.DL_TERM:
                 self._handle_dl_term()
                 if this == "\n":
@@ -1352,7 +1353,7 @@ class Tokenizer(object):
                 if self._can_recurse():
                     self._parse_table()
                 else:
-                    self._emit_text("{|")
+                    self._emit_text("{")
             elif self._context & contexts.TABLE_OPEN:
                 if this == next == "|" and self._context & contexts.TABLE_TD_LINE:
                     if self._context & contexts.TABLE_CELL_OPEN:

mwparserfromhell/parser/tokens.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8  -*-
 #
-# Copyright (C) 2012-2015 Ben Kurtovic <ben.kurtovic@gmail.com>
+# Copyright (C) 2012-2016 Ben Kurtovic <ben.kurtovic@gmail.com>
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy
 # of this software and associated documentation files (the "Software"), to deal

mwparserfromhell/smart_list.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8  -*-
 #
-# Copyright (C) 2012-2015 Ben Kurtovic <ben.kurtovic@gmail.com>
+# Copyright (C) 2012-2016 Ben Kurtovic <ben.kurtovic@gmail.com>
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy
 # of this software and associated documentation files (the "Software"), to deal

mwparserfromhell/string_mixin.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8  -*-
 #
-# Copyright (C) 2012-2015 Ben Kurtovic <ben.kurtovic@gmail.com>
+# Copyright (C) 2012-2016 Ben Kurtovic <ben.kurtovic@gmail.com>
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy
 # of this software and associated documentation files (the "Software"), to deal

mwparserfromhell/utils.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8  -*-
 #
-# Copyright (C) 2012-2015 Ben Kurtovic <ben.kurtovic@gmail.com>
+# Copyright (C) 2012-2016 Ben Kurtovic <ben.kurtovic@gmail.com>
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy
 # of this software and associated documentation files (the "Software"), to deal
@@ -62,6 +62,8 @@ def parse_anything(value, context=0, skip_style_tags=False):
         return Parser().parse(str(value), context, skip_style_tags)
     elif value is None:
         return Wikicode(SmartList())
+    elif hasattr(value, "read"):
+        return parse_anything(value.read(), context, skip_style_tags)
     try:
         nodelist = SmartList()
         for item in value:

mwparserfromhell/wikicode.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8  -*-
 #
-# Copyright (C) 2012-2015 Ben Kurtovic <ben.kurtovic@gmail.com>
+# Copyright (C) 2012-2016 Ben Kurtovic <ben.kurtovic@gmail.com>
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy
 # of this software and associated documentation files (the "Software"), to deal

scripts/memtest.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8  -*-
 #
-# Copyright (C) 2012-2015 Ben Kurtovic <ben.kurtovic@gmail.com>
+# Copyright (C) 2012-2016 Ben Kurtovic <ben.kurtovic@gmail.com>
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy
 # of this software and associated documentation files (the "Software"), to deal

setup.py

@@ -1,7 +1,7 @@
 #! /usr/bin/env python
 # -*- coding: utf-8  -*-
 #
-# Copyright (C) 2012-2015 Ben Kurtovic <ben.kurtovic@gmail.com>
+# Copyright (C) 2012-2016 Ben Kurtovic <ben.kurtovic@gmail.com>
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy
 # of this software and associated documentation files (the "Software"), to deal
@@ -75,7 +75,7 @@ if fallback:
 # Project-specific part begins here:
 
 tokenizer = Extension("mwparserfromhell.parser._tokenizer",
-                      sources=glob("mwparserfromhell/parser/ctokenizer/*.c"),
+                      sources=sorted(glob("mwparserfromhell/parser/ctokenizer/*.c")),
                       depends=glob("mwparserfromhell/parser/ctokenizer/*.h"))
 
 setup(
@@ -106,6 +106,7 @@ setup(
         "Programming Language :: Python :: 3.3",
         "Programming Language :: Python :: 3.4",
         "Programming Language :: Python :: 3.5",
+        "Programming Language :: Python :: 3.6",
         "Topic :: Text Processing :: Markup"
     ],
 )

tests/_test_tokenizer.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8  -*-
 #
-# Copyright (C) 2012-2015 Ben Kurtovic <ben.kurtovic@gmail.com>
+# Copyright (C) 2012-2016 Ben Kurtovic <ben.kurtovic@gmail.com>
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy
 # of this software and associated documentation files (the "Software"), to deal

tests/_test_tree_equality.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8  -*-
 #
-# Copyright (C) 2012-2015 Ben Kurtovic <ben.kurtovic@gmail.com>
+# Copyright (C) 2012-2016 Ben Kurtovic <ben.kurtovic@gmail.com>
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy
 # of this software and associated documentation files (the "Software"), to deal

tests/test_argument.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8  -*-
 #
-# Copyright (C) 2012-2015 Ben Kurtovic <ben.kurtovic@gmail.com>
+# Copyright (C) 2012-2016 Ben Kurtovic <ben.kurtovic@gmail.com>
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy
 # of this software and associated documentation files (the "Software"), to deal

tests/test_attribute.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8  -*-
 #
-# Copyright (C) 2012-2015 Ben Kurtovic <ben.kurtovic@gmail.com>
+# Copyright (C) 2012-2016 Ben Kurtovic <ben.kurtovic@gmail.com>
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy
 # of this software and associated documentation files (the "Software"), to deal

tests/test_builder.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8  -*-
 #
-# Copyright (C) 2012-2015 Ben Kurtovic <ben.kurtovic@gmail.com>
+# Copyright (C) 2012-2016 Ben Kurtovic <ben.kurtovic@gmail.com>
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy
 # of this software and associated documentation files (the "Software"), to deal

tests/test_comment.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8  -*-
 #
-# Copyright (C) 2012-2015 Ben Kurtovic <ben.kurtovic@gmail.com>
+# Copyright (C) 2012-2016 Ben Kurtovic <ben.kurtovic@gmail.com>
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy
 # of this software and associated documentation files (the "Software"), to deal

tests/test_ctokenizer.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8  -*-
 #
-# Copyright (C) 2012-2015 Ben Kurtovic <ben.kurtovic@gmail.com>
+# Copyright (C) 2012-2016 Ben Kurtovic <ben.kurtovic@gmail.com>
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy
 # of this software and associated documentation files (the "Software"), to deal

tests/test_docs.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8  -*-
 #
-# Copyright (C) 2012-2015 Ben Kurtovic <ben.kurtovic@gmail.com>
+# Copyright (C) 2012-2016 Ben Kurtovic <ben.kurtovic@gmail.com>
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy
 # of this software and associated documentation files (the "Software"), to deal

tests/test_external_link.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8  -*-
 #
-# Copyright (C) 2012-2015 Ben Kurtovic <ben.kurtovic@gmail.com>
+# Copyright (C) 2012-2016 Ben Kurtovic <ben.kurtovic@gmail.com>
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy
 # of this software and associated documentation files (the "Software"), to deal

tests/test_heading.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8  -*-
 #
-# Copyright (C) 2012-2015 Ben Kurtovic <ben.kurtovic@gmail.com>
+# Copyright (C) 2012-2016 Ben Kurtovic <ben.kurtovic@gmail.com>
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy
 # of this software and associated documentation files (the "Software"), to deal

tests/test_html_entity.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8  -*-
 #
-# Copyright (C) 2012-2015 Ben Kurtovic <ben.kurtovic@gmail.com>
+# Copyright (C) 2012-2016 Ben Kurtovic <ben.kurtovic@gmail.com>
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy
 # of this software and associated documentation files (the "Software"), to deal

tests/test_parameter.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8  -*-
 #
-# Copyright (C) 2012-2015 Ben Kurtovic <ben.kurtovic@gmail.com>
+# Copyright (C) 2012-2016 Ben Kurtovic <ben.kurtovic@gmail.com>
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy
 # of this software and associated documentation files (the "Software"), to deal

tests/test_parser.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8  -*-
 #
-# Copyright (C) 2012-2015 Ben Kurtovic <ben.kurtovic@gmail.com>
+# Copyright (C) 2012-2016 Ben Kurtovic <ben.kurtovic@gmail.com>
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy
 # of this software and associated documentation files (the "Software"), to deal

tests/test_pytokenizer.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8  -*-
 #
-# Copyright (C) 2012-2015 Ben Kurtovic <ben.kurtovic@gmail.com>
+# Copyright (C) 2012-2016 Ben Kurtovic <ben.kurtovic@gmail.com>
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy
 # of this software and associated documentation files (the "Software"), to deal

tests/test_roundtripping.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8  -*-
 #
-# Copyright (C) 2012-2015 Ben Kurtovic <ben.kurtovic@gmail.com>
+# Copyright (C) 2012-2016 Ben Kurtovic <ben.kurtovic@gmail.com>
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy
 # of this software and associated documentation files (the "Software"), to deal

tests/test_smart_list.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8  -*-
 #
-# Copyright (C) 2012-2015 Ben Kurtovic <ben.kurtovic@gmail.com>
+# Copyright (C) 2012-2016 Ben Kurtovic <ben.kurtovic@gmail.com>
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy
 # of this software and associated documentation files (the "Software"), to deal

tests/test_string_mixin.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8  -*-
 #
-# Copyright (C) 2012-2015 Ben Kurtovic <ben.kurtovic@gmail.com>
+# Copyright (C) 2012-2016 Ben Kurtovic <ben.kurtovic@gmail.com>
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy
 # of this software and associated documentation files (the "Software"), to deal

tests/test_tag.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8  -*-
 #
-# Copyright (C) 2012-2015 Ben Kurtovic <ben.kurtovic@gmail.com>
+# Copyright (C) 2012-2016 Ben Kurtovic <ben.kurtovic@gmail.com>
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy
 # of this software and associated documentation files (the "Software"), to deal

tests/test_template.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8  -*-
 #
-# Copyright (C) 2012-2015 Ben Kurtovic <ben.kurtovic@gmail.com>
+# Copyright (C) 2012-2016 Ben Kurtovic <ben.kurtovic@gmail.com>
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy
 # of this software and associated documentation files (the "Software"), to deal

tests/test_text.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8  -*-
 #
-# Copyright (C) 2012-2015 Ben Kurtovic <ben.kurtovic@gmail.com>
+# Copyright (C) 2012-2016 Ben Kurtovic <ben.kurtovic@gmail.com>
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy
 # of this software and associated documentation files (the "Software"), to deal

tests/test_tokens.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8  -*-
 #
-# Copyright (C) 2012-2015 Ben Kurtovic <ben.kurtovic@gmail.com>
+# Copyright (C) 2012-2016 Ben Kurtovic <ben.kurtovic@gmail.com>
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy
 # of this software and associated documentation files (the "Software"), to deal

tests/test_utils.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8  -*-
 #
-# Copyright (C) 2012-2015 Ben Kurtovic <ben.kurtovic@gmail.com>
+# Copyright (C) 2012-2016 Ben Kurtovic <ben.kurtovic@gmail.com>
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy
 # of this software and associated documentation files (the "Software"), to deal

tests/test_wikicode.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8  -*-
 #
-# Copyright (C) 2012-2015 Ben Kurtovic <ben.kurtovic@gmail.com>
+# Copyright (C) 2012-2016 Ben Kurtovic <ben.kurtovic@gmail.com>
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy
 # of this software and associated documentation files (the "Software"), to deal

tests/test_wikilink.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8  -*-
 #
-# Copyright (C) 2012-2015 Ben Kurtovic <ben.kurtovic@gmail.com>
+# Copyright (C) 2012-2016 Ben Kurtovic <ben.kurtovic@gmail.com>
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy
 # of this software and associated documentation files (the "Software"), to deal

tests/tokenizer/integration.mwtest

@@ -332,3 +332,17 @@ name:   wikilink_to_external_link_fallback_2
 label:  an external link enclosed in an extra pair of brackets (see issue #120)
 input:  "[[http://example.com]]"
 output: [Text(text="["), ExternalLinkOpen(brackets=True), Text(text="http://example.com"), ExternalLinkClose(), Text(text="]")]
+
+---
+
+name:   tables_in_templates
+label:  catch error handling mistakes when wikitables are inside templates
+input:  "{{hello|test\n{|\n|} }}"
+output: [TemplateOpen(), Text(text="hello"), TemplateParamSeparator(), Text(text="test\n"), TagOpenOpen(wiki_markup="{|"), Text(text="table"), TagCloseOpen(padding="\n"), TagOpenClose(wiki_markup="|}"), Text(text="table"), TagCloseClose(), Text(text=" "), TemplateClose()]
+
+---
+
+name:   tables_in_templates_2
+label:  catch error handling mistakes when wikitables are inside templates
+input:  "{{hello|test\n{|\n| }}"
+output: [TemplateOpen(), Text(text="hello"), TemplateParamSeparator(), Text(text="test\n{"), TemplateParamSeparator(), Text(text="\n"), TemplateParamSeparator(), Text(text=" "), TemplateClose()]

tests/tokenizer/templates.mwtest

@@ -222,6 +222,13 @@ output: [TemplateOpen(), TemplateOpen(), TemplateOpen(), TemplateOpen(), Text(te
 
 ---
 
+name:   nested_two_args
+label:  template whose first parameter is unnamed with two templates, followed by a named parameter
+input:  "{{a|{{b}}{{c}}|d=e}}"
+output: [TemplateOpen(), Text(text="a"), TemplateParamSeparator(), TemplateOpen(), Text(text="b"), TemplateClose(), TemplateOpen(), Text(text="c"), TemplateClose(), TemplateParamSeparator(), Text(text="d"), TemplateParamEquals(), Text(text="e"), TemplateClose()]
+
+---
+
 name:   newlines_start
 label:  a newline at the start of a template name
 input:  "{{\nfoobar}}"
@@ -365,7 +372,7 @@ output: [TemplateOpen(), Text(text="\nfoo\n"), TemplateParamSeparator(), Text(te
 name:   newlines_wildcard_redux_invalid
 label:  a variation of the newlines_wildcard_redux test that is invalid
 input:  "{{\nfoo\n|\n{{\nb\nar\n|\nb\naz\n=\nb\niz\n}}\n=\nb\nuzz\n}}"
-output: [Text(text="{{\nfoo\n|\n{{\nb\nar\n|\nb\naz\n=\nb\niz\n}}\n=\nb\nuzz\n}}")]
+output: [TemplateOpen(), Text(text="\nfoo\n"), TemplateParamSeparator(), Text(text="\n{{\nb\nar\n"), TemplateParamSeparator(), Text(text="\nb\naz\n"), TemplateParamEquals(), Text(text="\nb\niz\n"), TemplateClose(), Text(text="\n=\nb\nuzz\n}}")]
 
 ---