diff --git a/.travis.yml b/.travis.yml index b5d90db..c0233d9 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,6 +6,8 @@ python: - 3.3 - 3.4 - 3.5 + - 3.6 + - nightly sudo: false install: - if [[ $TRAVIS_PYTHON_VERSION == '3.2' ]]; then pip install coverage==3.7.1; fi diff --git a/CHANGELOG b/CHANGELOG index 2629dc6..053b37e 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,3 +1,14 @@ +v0.4.4 (released December 30, 2016): + +- Added support for Python 3.6. +- Fixed parsing bugs involving: + - wikitables nested in templates; + - wikitable error recovery when unable to recurse; + - templates nested in template parameters before other parameters. +- Fixed parsing file-like objects. +- Made builds deterministic. +- Documented caveats. + v0.4.3 (released October 29, 2015): - Added Windows binaries for Python 3.5. diff --git a/LICENSE b/LICENSE index 92f5e42..230bc5c 100644 --- a/LICENSE +++ b/LICENSE @@ -1,4 +1,4 @@ -Copyright (C) 2012-2015 Ben Kurtovic +Copyright (C) 2012-2016 Ben Kurtovic Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/README.rst b/README.rst index c361a56..b7d324c 100644 --- a/README.rst +++ b/README.rst @@ -21,8 +21,7 @@ Installation The easiest way to install the parser is through the `Python Package Index`_; you can install the latest release with ``pip install mwparserfromhell`` -(`get pip`_). On Windows, make sure you have the latest version of pip -installed by running ``pip install --upgrade pip``. +(`get pip`_). Make sure your pip is up-to-date first, especially on Windows. Alternatively, get the latest development version:: @@ -114,6 +113,24 @@ saving the page!) by calling ``str()`` on it:: Likewise, use ``unicode(code)`` in Python 2. +Caveats +------- + +An inherent limitation in wikicode prevents us from generating complete parse +trees in certain cases. For example, the string ``{{echo|''Hello}}, world!''`` +produces the valid output ``Hello, world!`` in MediaWiki, assuming +``{{echo}}`` is a template that returns its first parameter. But since +representing this in mwparserfromhell's node tree would be impossible, we +compromise by treating the first node (i.e., the template) as plain text, +parsing only the italics. + +The current workaround for cases where you are not interested in text +formatting is to pass ``skip_style_tags=True`` to ``mwparserfromhell.parse()``. +This treats ``''`` and ``'''`` like plain text. + +A future version of mwparserfromhell will include multiple parsing modes to get +around this restriction. + Integration ----------- @@ -132,8 +149,8 @@ If you're using Pywikibot_, your code might look like this:: text = page.get() return mwparserfromhell.parse(text) -If you're not using a library, you can parse any page using the following code -(via the API_):: +If you're not using a library, you can parse any page using the following +Python 3 code (via the API_):: import json from urllib.parse import urlencode diff --git a/appveyor.yml b/appveyor.yml index 1432a2b..daec144 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -1,6 +1,6 @@ # This config file is used by appveyor.com to build Windows release binaries -version: 0.4.3-b{build} +version: 0.4.4-b{build} branches: only: @@ -15,7 +15,6 @@ environment: WRAPPER: "cmd /E:ON /V:ON /C .\\scripts\\win_wrapper.cmd" PIP: "%WRAPPER% %PYTHON%\\Scripts\\pip.exe" SETUPPY: "%WRAPPER% %PYTHON%\\python setup.py --with-extension" - PYMOD: "%WRAPPER% %PYTHON%\\python -m" PYPI_USERNAME: "earwigbot" PYPI_PASSWORD: secure: gOIcvPxSC2ujuhwOzwj3v8xjq3CCYd8keFWVnguLM+gcL0e02qshDHy7gwZZwj0+ @@ -67,7 +66,7 @@ after_test: - "%SETUPPY% bdist_wheel" on_success: - - "IF %APPVEYOR_REPO_BRANCH%==master %PYMOD% twine upload dist\\* -u %PYPI_USERNAME% -p %PYPI_PASSWORD%" + - "IF %APPVEYOR_REPO_BRANCH%==master %WRAPPER% %PYTHON%\\python -m twine upload dist\\* -u %PYPI_USERNAME% -p %PYPI_PASSWORD%" artifacts: - path: dist\* diff --git a/docs/caveats.rst b/docs/caveats.rst new file mode 100644 index 0000000..927aa54 --- /dev/null +++ b/docs/caveats.rst @@ -0,0 +1,17 @@ +Caveats +======= + +An inherent limitation in wikicode prevents us from generating complete parse +trees in certain cases. For example, the string ``{{echo|''Hello}}, world!''`` +produces the valid output ``Hello, world!`` in MediaWiki, assuming +``{{echo}}`` is a template that returns its first parameter. But since +representing this in mwparserfromhell's node tree would be impossible, we +compromise by treating the first node (i.e., the template) as plain text, +parsing only the italics. + +The current workaround for cases where you are not interested in text +formatting is to pass *skip_style_tags=True* to :func:`mwparserfromhell.parse`. +This treats ``''`` and ``'''`` like plain text. + +A future version of mwparserfromhell will include multiple parsing modes to get +around this restriction. diff --git a/docs/changelog.rst b/docs/changelog.rst index ef26aa2..43400a9 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -1,6 +1,23 @@ Changelog ========= +v0.4.4 +------ + +`Released December 30, 2016 `_ +(`changes `__): + +- Added support for Python 3.6. +- Fixed parsing bugs involving: + + - wikitables nested in templates; + - wikitable error recovery when unable to recurse; + - templates nested in template parameters before other parameters. + +- Fixed parsing file-like objects. +- Made builds deterministic. +- Documented caveats. + v0.4.3 ------ diff --git a/docs/conf.py b/docs/conf.py index 3f82ea7..8d48dff 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -42,7 +42,7 @@ master_doc = 'index' # General information about the project. project = u'mwparserfromhell' -copyright = u'2012, 2013, 2014, 2015 Ben Kurtovic' +copyright = u'2012, 2013, 2014, 2015, 2016 Ben Kurtovic' # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the diff --git a/docs/index.rst b/docs/index.rst index 9a6c8ab..6593881 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -19,8 +19,7 @@ Installation The easiest way to install the parser is through the `Python Package Index`_; you can install the latest release with ``pip install mwparserfromhell`` -(`get pip`_). On Windows, make sure you have the latest version of pip -installed by running ``pip install --upgrade pip``. +(`get pip`_). Make sure your pip is up-to-date first, especially on Windows. Alternatively, get the latest development version:: @@ -41,6 +40,7 @@ Contents :maxdepth: 2 usage + caveats integration changelog API Reference diff --git a/mwparserfromhell/__init__.py b/mwparserfromhell/__init__.py index 0d90567..1d3c7d7 100644 --- a/mwparserfromhell/__init__.py +++ b/mwparserfromhell/__init__.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # -# Copyright (C) 2012-2015 Ben Kurtovic +# Copyright (C) 2012-2016 Ben Kurtovic # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal @@ -27,9 +27,9 @@ outrageously powerful parser for `MediaWiki `_ wikicode. """ __author__ = "Ben Kurtovic" -__copyright__ = "Copyright (C) 2012, 2013, 2014, 2015 Ben Kurtovic" +__copyright__ = "Copyright (C) 2012, 2013, 2014, 2015, 2016 Ben Kurtovic" __license__ = "MIT License" -__version__ = "0.4.3" +__version__ = "0.4.4" __email__ = "ben.kurtovic@gmail.com" from . import (compat, definitions, nodes, parser, smart_list, string_mixin, diff --git a/mwparserfromhell/definitions.py b/mwparserfromhell/definitions.py index bbfd346..18a06cc 100644 --- a/mwparserfromhell/definitions.py +++ b/mwparserfromhell/definitions.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # -# Copyright (C) 2012-2015 Ben Kurtovic +# Copyright (C) 2012-2016 Ben Kurtovic # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal diff --git a/mwparserfromhell/nodes/__init__.py b/mwparserfromhell/nodes/__init__.py index d0258ca..91678c8 100644 --- a/mwparserfromhell/nodes/__init__.py +++ b/mwparserfromhell/nodes/__init__.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # -# Copyright (C) 2012-2015 Ben Kurtovic +# Copyright (C) 2012-2016 Ben Kurtovic # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal diff --git a/mwparserfromhell/nodes/argument.py b/mwparserfromhell/nodes/argument.py index 39c33ae..9146704 100644 --- a/mwparserfromhell/nodes/argument.py +++ b/mwparserfromhell/nodes/argument.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # -# Copyright (C) 2012-2015 Ben Kurtovic +# Copyright (C) 2012-2016 Ben Kurtovic # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal diff --git a/mwparserfromhell/nodes/comment.py b/mwparserfromhell/nodes/comment.py index 3e82be7..0d141e9 100644 --- a/mwparserfromhell/nodes/comment.py +++ b/mwparserfromhell/nodes/comment.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # -# Copyright (C) 2012-2015 Ben Kurtovic +# Copyright (C) 2012-2016 Ben Kurtovic # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal diff --git a/mwparserfromhell/nodes/external_link.py b/mwparserfromhell/nodes/external_link.py index a07e985..8493a25 100644 --- a/mwparserfromhell/nodes/external_link.py +++ b/mwparserfromhell/nodes/external_link.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # -# Copyright (C) 2012-2015 Ben Kurtovic +# Copyright (C) 2012-2016 Ben Kurtovic # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal diff --git a/mwparserfromhell/nodes/extras/__init__.py b/mwparserfromhell/nodes/extras/__init__.py index 854fa45..2d90b4e 100644 --- a/mwparserfromhell/nodes/extras/__init__.py +++ b/mwparserfromhell/nodes/extras/__init__.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # -# Copyright (C) 2012-2015 Ben Kurtovic +# Copyright (C) 2012-2016 Ben Kurtovic # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal diff --git a/mwparserfromhell/nodes/extras/attribute.py b/mwparserfromhell/nodes/extras/attribute.py index 7c7dd56..0f55a6b 100644 --- a/mwparserfromhell/nodes/extras/attribute.py +++ b/mwparserfromhell/nodes/extras/attribute.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # -# Copyright (C) 2012-2015 Ben Kurtovic +# Copyright (C) 2012-2016 Ben Kurtovic # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal diff --git a/mwparserfromhell/nodes/extras/parameter.py b/mwparserfromhell/nodes/extras/parameter.py index 48f610c..0d21d5b 100644 --- a/mwparserfromhell/nodes/extras/parameter.py +++ b/mwparserfromhell/nodes/extras/parameter.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # -# Copyright (C) 2012-2015 Ben Kurtovic +# Copyright (C) 2012-2016 Ben Kurtovic # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal diff --git a/mwparserfromhell/nodes/heading.py b/mwparserfromhell/nodes/heading.py index 0db56f3..7bba702 100644 --- a/mwparserfromhell/nodes/heading.py +++ b/mwparserfromhell/nodes/heading.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # -# Copyright (C) 2012-2015 Ben Kurtovic +# Copyright (C) 2012-2016 Ben Kurtovic # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal diff --git a/mwparserfromhell/nodes/html_entity.py b/mwparserfromhell/nodes/html_entity.py index e7f1bbc..8b7f270 100644 --- a/mwparserfromhell/nodes/html_entity.py +++ b/mwparserfromhell/nodes/html_entity.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # -# Copyright (C) 2012-2015 Ben Kurtovic +# Copyright (C) 2012-2016 Ben Kurtovic # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal diff --git a/mwparserfromhell/nodes/tag.py b/mwparserfromhell/nodes/tag.py index cf3b4a5..d393e2c 100644 --- a/mwparserfromhell/nodes/tag.py +++ b/mwparserfromhell/nodes/tag.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # -# Copyright (C) 2012-2015 Ben Kurtovic +# Copyright (C) 2012-2016 Ben Kurtovic # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal diff --git a/mwparserfromhell/nodes/template.py b/mwparserfromhell/nodes/template.py index 4ee5f5d..57fec70 100644 --- a/mwparserfromhell/nodes/template.py +++ b/mwparserfromhell/nodes/template.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # -# Copyright (C) 2012-2015 Ben Kurtovic +# Copyright (C) 2012-2016 Ben Kurtovic # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal diff --git a/mwparserfromhell/nodes/text.py b/mwparserfromhell/nodes/text.py index e793c1f..08ac205 100644 --- a/mwparserfromhell/nodes/text.py +++ b/mwparserfromhell/nodes/text.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # -# Copyright (C) 2012-2015 Ben Kurtovic +# Copyright (C) 2012-2016 Ben Kurtovic # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal diff --git a/mwparserfromhell/nodes/wikilink.py b/mwparserfromhell/nodes/wikilink.py index 88eaacc..f71b5f6 100644 --- a/mwparserfromhell/nodes/wikilink.py +++ b/mwparserfromhell/nodes/wikilink.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # -# Copyright (C) 2012-2015 Ben Kurtovic +# Copyright (C) 2012-2016 Ben Kurtovic # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal diff --git a/mwparserfromhell/parser/__init__.py b/mwparserfromhell/parser/__init__.py index cbe58c5..f39fdc4 100644 --- a/mwparserfromhell/parser/__init__.py +++ b/mwparserfromhell/parser/__init__.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # -# Copyright (C) 2012-2015 Ben Kurtovic +# Copyright (C) 2012-2016 Ben Kurtovic # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal diff --git a/mwparserfromhell/parser/builder.py b/mwparserfromhell/parser/builder.py index ad29f4d..c86a923 100644 --- a/mwparserfromhell/parser/builder.py +++ b/mwparserfromhell/parser/builder.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # -# Copyright (C) 2012-2015 Ben Kurtovic +# Copyright (C) 2012-2016 Ben Kurtovic # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal diff --git a/mwparserfromhell/parser/contexts.py b/mwparserfromhell/parser/contexts.py index b676e86..405a027 100644 --- a/mwparserfromhell/parser/contexts.py +++ b/mwparserfromhell/parser/contexts.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # -# Copyright (C) 2012-2015 Ben Kurtovic +# Copyright (C) 2012-2016 Ben Kurtovic # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal diff --git a/mwparserfromhell/parser/ctokenizer/common.h b/mwparserfromhell/parser/ctokenizer/common.h index abade02..3bd22af 100644 --- a/mwparserfromhell/parser/ctokenizer/common.h +++ b/mwparserfromhell/parser/ctokenizer/common.h @@ -1,5 +1,5 @@ /* -Copyright (C) 2012-2015 Ben Kurtovic +Copyright (C) 2012-2016 Ben Kurtovic Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in diff --git a/mwparserfromhell/parser/ctokenizer/contexts.h b/mwparserfromhell/parser/ctokenizer/contexts.h index 4e4a8c7..96afd6c 100644 --- a/mwparserfromhell/parser/ctokenizer/contexts.h +++ b/mwparserfromhell/parser/ctokenizer/contexts.h @@ -1,5 +1,5 @@ /* -Copyright (C) 2012-2015 Ben Kurtovic +Copyright (C) 2012-2016 Ben Kurtovic Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in diff --git a/mwparserfromhell/parser/ctokenizer/definitions.c b/mwparserfromhell/parser/ctokenizer/definitions.c index e5b32da..38482a4 100644 --- a/mwparserfromhell/parser/ctokenizer/definitions.c +++ b/mwparserfromhell/parser/ctokenizer/definitions.c @@ -1,5 +1,5 @@ /* -Copyright (C) 2012-2015 Ben Kurtovic +Copyright (C) 2012-2016 Ben Kurtovic Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in diff --git a/mwparserfromhell/parser/ctokenizer/definitions.h b/mwparserfromhell/parser/ctokenizer/definitions.h index 8f8dc2c..1ae1d09 100644 --- a/mwparserfromhell/parser/ctokenizer/definitions.h +++ b/mwparserfromhell/parser/ctokenizer/definitions.h @@ -1,5 +1,5 @@ /* -Copyright (C) 2012-2015 Ben Kurtovic +Copyright (C) 2012-2016 Ben Kurtovic Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in diff --git a/mwparserfromhell/parser/ctokenizer/tag_data.c b/mwparserfromhell/parser/ctokenizer/tag_data.c index 2f67966..1b73533 100644 --- a/mwparserfromhell/parser/ctokenizer/tag_data.c +++ b/mwparserfromhell/parser/ctokenizer/tag_data.c @@ -1,5 +1,5 @@ /* -Copyright (C) 2012-2015 Ben Kurtovic +Copyright (C) 2012-2016 Ben Kurtovic Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in diff --git a/mwparserfromhell/parser/ctokenizer/tag_data.h b/mwparserfromhell/parser/ctokenizer/tag_data.h index f184081..c2e9303 100644 --- a/mwparserfromhell/parser/ctokenizer/tag_data.h +++ b/mwparserfromhell/parser/ctokenizer/tag_data.h @@ -1,5 +1,5 @@ /* -Copyright (C) 2012-2015 Ben Kurtovic +Copyright (C) 2012-2016 Ben Kurtovic Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in diff --git a/mwparserfromhell/parser/ctokenizer/textbuffer.c b/mwparserfromhell/parser/ctokenizer/textbuffer.c index 0c711c5..3fd129f 100644 --- a/mwparserfromhell/parser/ctokenizer/textbuffer.c +++ b/mwparserfromhell/parser/ctokenizer/textbuffer.c @@ -1,5 +1,5 @@ /* -Copyright (C) 2012-2015 Ben Kurtovic +Copyright (C) 2012-2016 Ben Kurtovic Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in diff --git a/mwparserfromhell/parser/ctokenizer/textbuffer.h b/mwparserfromhell/parser/ctokenizer/textbuffer.h index 123d240..35579fd 100644 --- a/mwparserfromhell/parser/ctokenizer/textbuffer.h +++ b/mwparserfromhell/parser/ctokenizer/textbuffer.h @@ -1,5 +1,5 @@ /* -Copyright (C) 2012-2015 Ben Kurtovic +Copyright (C) 2012-2016 Ben Kurtovic Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in diff --git a/mwparserfromhell/parser/ctokenizer/tok_parse.c b/mwparserfromhell/parser/ctokenizer/tok_parse.c index 5833d01..f4e9606 100644 --- a/mwparserfromhell/parser/ctokenizer/tok_parse.c +++ b/mwparserfromhell/parser/ctokenizer/tok_parse.c @@ -1,5 +1,5 @@ /* -Copyright (C) 2012-2015 Ben Kurtovic +Copyright (C) 2012-2016 Ben Kurtovic Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in @@ -249,7 +249,7 @@ static int Tokenizer_handle_template_param(Tokenizer* self) else if (self->topstack->context & LC_TEMPLATE_PARAM_VALUE) self->topstack->context ^= LC_TEMPLATE_PARAM_VALUE; if (self->topstack->context & LC_TEMPLATE_PARAM_KEY) { - stack = Tokenizer_pop_keeping_context(self); + stack = Tokenizer_pop(self); if (!stack) return -1; if (Tokenizer_emit_all(self, stack)) { @@ -274,7 +274,7 @@ static int Tokenizer_handle_template_param_value(Tokenizer* self) { PyObject *stack; - stack = Tokenizer_pop_keeping_context(self); + stack = Tokenizer_pop(self); if (!stack) return -1; if (Tokenizer_emit_all(self, stack)) { @@ -301,7 +301,7 @@ static PyObject* Tokenizer_handle_template_end(Tokenizer* self) return Tokenizer_fail_route(self); } else if (self->topstack->context & LC_TEMPLATE_PARAM_KEY) { - stack = Tokenizer_pop_keeping_context(self); + stack = Tokenizer_pop(self); if (!stack) return NULL; if (Tokenizer_emit_all(self, stack)) { @@ -2190,7 +2190,7 @@ static PyObject* Tokenizer_handle_table_style(Tokenizer* self, Unicode end_token */ static int Tokenizer_parse_table(Tokenizer* self) { - Py_ssize_t reset = self->head + 1; + Py_ssize_t reset = self->head; PyObject *style, *padding; PyObject *table = NULL; self->head += 2; @@ -2201,7 +2201,7 @@ static int Tokenizer_parse_table(Tokenizer* self) if (BAD_ROUTE) { RESET_ROUTE(); self->head = reset; - if (Tokenizer_emit_text(self, "{|")) + if (Tokenizer_emit_char(self, '{')) return -1; return 0; } @@ -2220,7 +2220,7 @@ static int Tokenizer_parse_table(Tokenizer* self) Py_DECREF(padding); Py_DECREF(style); self->head = reset; - if (Tokenizer_emit_text(self, "{|")) + if (Tokenizer_emit_char(self, '{')) return -1; return 0; } @@ -2689,10 +2689,8 @@ PyObject* Tokenizer_parse(Tokenizer* self, uint64_t context, int push) if (Tokenizer_parse_table(self)) return NULL; } - else if (Tokenizer_emit_char(self, this) || Tokenizer_emit_char(self, next)) + else if (Tokenizer_emit_char(self, this)) return NULL; - else - self->head++; } else if (this_context & LC_TABLE_OPEN) { if (this == '|' && next == '|' && this_context & LC_TABLE_TD_LINE) { diff --git a/mwparserfromhell/parser/ctokenizer/tok_parse.h b/mwparserfromhell/parser/ctokenizer/tok_parse.h index b627ca7..9d98b00 100644 --- a/mwparserfromhell/parser/ctokenizer/tok_parse.h +++ b/mwparserfromhell/parser/ctokenizer/tok_parse.h @@ -1,5 +1,5 @@ /* -Copyright (C) 2012-2015 Ben Kurtovic +Copyright (C) 2012-2016 Ben Kurtovic Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in diff --git a/mwparserfromhell/parser/ctokenizer/tok_support.c b/mwparserfromhell/parser/ctokenizer/tok_support.c index bcd4edf..31c6bb9 100644 --- a/mwparserfromhell/parser/ctokenizer/tok_support.c +++ b/mwparserfromhell/parser/ctokenizer/tok_support.c @@ -1,5 +1,5 @@ /* -Copyright (C) 2012-2015 Ben Kurtovic +Copyright (C) 2012-2016 Ben Kurtovic Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in diff --git a/mwparserfromhell/parser/ctokenizer/tok_support.h b/mwparserfromhell/parser/ctokenizer/tok_support.h index c167c0a..182f9a0 100644 --- a/mwparserfromhell/parser/ctokenizer/tok_support.h +++ b/mwparserfromhell/parser/ctokenizer/tok_support.h @@ -1,5 +1,5 @@ /* -Copyright (C) 2012-2015 Ben Kurtovic +Copyright (C) 2012-2016 Ben Kurtovic Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in diff --git a/mwparserfromhell/parser/ctokenizer/tokenizer.c b/mwparserfromhell/parser/ctokenizer/tokenizer.c index 3d751db..47d2993 100644 --- a/mwparserfromhell/parser/ctokenizer/tokenizer.c +++ b/mwparserfromhell/parser/ctokenizer/tokenizer.c @@ -1,5 +1,5 @@ /* -Copyright (C) 2012-2015 Ben Kurtovic +Copyright (C) 2012-2016 Ben Kurtovic Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in diff --git a/mwparserfromhell/parser/ctokenizer/tokenizer.h b/mwparserfromhell/parser/ctokenizer/tokenizer.h index ac45a45..6050ce0 100644 --- a/mwparserfromhell/parser/ctokenizer/tokenizer.h +++ b/mwparserfromhell/parser/ctokenizer/tokenizer.h @@ -1,5 +1,5 @@ /* -Copyright (C) 2012-2015 Ben Kurtovic +Copyright (C) 2012-2016 Ben Kurtovic Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in diff --git a/mwparserfromhell/parser/ctokenizer/tokens.c b/mwparserfromhell/parser/ctokenizer/tokens.c index f080bab..3cb5f2a 100644 --- a/mwparserfromhell/parser/ctokenizer/tokens.c +++ b/mwparserfromhell/parser/ctokenizer/tokens.c @@ -1,5 +1,5 @@ /* -Copyright (C) 2012-2015 Ben Kurtovic +Copyright (C) 2012-2016 Ben Kurtovic Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in diff --git a/mwparserfromhell/parser/ctokenizer/tokens.h b/mwparserfromhell/parser/ctokenizer/tokens.h index 9551902..705c8af 100644 --- a/mwparserfromhell/parser/ctokenizer/tokens.h +++ b/mwparserfromhell/parser/ctokenizer/tokens.h @@ -1,5 +1,5 @@ /* -Copyright (C) 2012-2015 Ben Kurtovic +Copyright (C) 2012-2016 Ben Kurtovic Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in diff --git a/mwparserfromhell/parser/tokenizer.py b/mwparserfromhell/parser/tokenizer.py index 3a1c775..309d0d3 100644 --- a/mwparserfromhell/parser/tokenizer.py +++ b/mwparserfromhell/parser/tokenizer.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # -# Copyright (C) 2012-2015 Ben Kurtovic +# Copyright (C) 2012-2016 Ben Kurtovic # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal @@ -264,14 +264,14 @@ class Tokenizer(object): elif self._context & contexts.TEMPLATE_PARAM_VALUE: self._context ^= contexts.TEMPLATE_PARAM_VALUE else: - self._emit_all(self._pop(keep_context=True)) + self._emit_all(self._pop()) self._context |= contexts.TEMPLATE_PARAM_KEY self._emit(tokens.TemplateParamSeparator()) self._push(self._context) def _handle_template_param_value(self): """Handle a template parameter's value at the head of the string.""" - self._emit_all(self._pop(keep_context=True)) + self._emit_all(self._pop()) self._context ^= contexts.TEMPLATE_PARAM_KEY self._context |= contexts.TEMPLATE_PARAM_VALUE self._emit(tokens.TemplateParamEquals()) @@ -282,7 +282,7 @@ class Tokenizer(object): if not self._context & (contexts.HAS_TEXT | contexts.HAS_TEMPLATE): self._fail_route() elif self._context & contexts.TEMPLATE_PARAM_KEY: - self._emit_all(self._pop(keep_context=True)) + self._emit_all(self._pop()) self._head += 1 return self._pop() @@ -1074,14 +1074,14 @@ class Tokenizer(object): def _parse_table(self): """Parse a wikicode table by starting with the first line.""" - reset = self._head + 1 + reset = self._head self._head += 2 self._push(contexts.TABLE_OPEN) try: padding = self._handle_table_style("\n") except BadRoute: self._head = reset - self._emit_text("{|") + self._emit_text("{") return style = self._pop() @@ -1090,7 +1090,7 @@ class Tokenizer(object): table = self._parse(contexts.TABLE_OPEN) except BadRoute: self._head = reset - self._emit_text("{|") + self._emit_text("{") return self._emit_table_tag("{|", "table", style, padding, None, table, "|}") @@ -1338,9 +1338,10 @@ class Tokenizer(object): if result is not None: return result elif self._read(-1) in ("\n", self.START) and this in ("#", "*", ";", ":"): - self._handle_list() - elif self._read(-1) in ("\n", self.START) and this == next == self._read(2) == self._read(3) == "-": - self._handle_hr() + self._handle_list() + elif self._read(-1) in ("\n", self.START) and ( + this == next == self._read(2) == self._read(3) == "-"): + self._handle_hr() elif this in ("\n", ":") and self._context & contexts.DL_TERM: self._handle_dl_term() if this == "\n": @@ -1352,7 +1353,7 @@ class Tokenizer(object): if self._can_recurse(): self._parse_table() else: - self._emit_text("{|") + self._emit_text("{") elif self._context & contexts.TABLE_OPEN: if this == next == "|" and self._context & contexts.TABLE_TD_LINE: if self._context & contexts.TABLE_CELL_OPEN: diff --git a/mwparserfromhell/parser/tokens.py b/mwparserfromhell/parser/tokens.py index 4668780..036dc9b 100644 --- a/mwparserfromhell/parser/tokens.py +++ b/mwparserfromhell/parser/tokens.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # -# Copyright (C) 2012-2015 Ben Kurtovic +# Copyright (C) 2012-2016 Ben Kurtovic # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal diff --git a/mwparserfromhell/smart_list.py b/mwparserfromhell/smart_list.py index 1ff1cc2..c59a363 100644 --- a/mwparserfromhell/smart_list.py +++ b/mwparserfromhell/smart_list.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # -# Copyright (C) 2012-2015 Ben Kurtovic +# Copyright (C) 2012-2016 Ben Kurtovic # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal diff --git a/mwparserfromhell/string_mixin.py b/mwparserfromhell/string_mixin.py index 01809a7..b5ba5a4 100644 --- a/mwparserfromhell/string_mixin.py +++ b/mwparserfromhell/string_mixin.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # -# Copyright (C) 2012-2015 Ben Kurtovic +# Copyright (C) 2012-2016 Ben Kurtovic # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal diff --git a/mwparserfromhell/utils.py b/mwparserfromhell/utils.py index 28823fc..7387420 100644 --- a/mwparserfromhell/utils.py +++ b/mwparserfromhell/utils.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # -# Copyright (C) 2012-2015 Ben Kurtovic +# Copyright (C) 2012-2016 Ben Kurtovic # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal @@ -62,6 +62,8 @@ def parse_anything(value, context=0, skip_style_tags=False): return Parser().parse(str(value), context, skip_style_tags) elif value is None: return Wikicode(SmartList()) + elif hasattr(value, "read"): + return parse_anything(value.read(), context, skip_style_tags) try: nodelist = SmartList() for item in value: diff --git a/mwparserfromhell/wikicode.py b/mwparserfromhell/wikicode.py index c623971..e3f6b92 100644 --- a/mwparserfromhell/wikicode.py +++ b/mwparserfromhell/wikicode.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # -# Copyright (C) 2012-2015 Ben Kurtovic +# Copyright (C) 2012-2016 Ben Kurtovic # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal diff --git a/scripts/memtest.py b/scripts/memtest.py index 824d992..823560d 100644 --- a/scripts/memtest.py +++ b/scripts/memtest.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # -# Copyright (C) 2012-2015 Ben Kurtovic +# Copyright (C) 2012-2016 Ben Kurtovic # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal diff --git a/setup.py b/setup.py index 1bca436..ee5fd50 100644 --- a/setup.py +++ b/setup.py @@ -1,7 +1,7 @@ #! /usr/bin/env python # -*- coding: utf-8 -*- # -# Copyright (C) 2012-2015 Ben Kurtovic +# Copyright (C) 2012-2016 Ben Kurtovic # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal @@ -75,7 +75,7 @@ if fallback: # Project-specific part begins here: tokenizer = Extension("mwparserfromhell.parser._tokenizer", - sources=glob("mwparserfromhell/parser/ctokenizer/*.c"), + sources=sorted(glob("mwparserfromhell/parser/ctokenizer/*.c")), depends=glob("mwparserfromhell/parser/ctokenizer/*.h")) setup( @@ -106,6 +106,7 @@ setup( "Programming Language :: Python :: 3.3", "Programming Language :: Python :: 3.4", "Programming Language :: Python :: 3.5", + "Programming Language :: Python :: 3.6", "Topic :: Text Processing :: Markup" ], ) diff --git a/tests/_test_tokenizer.py b/tests/_test_tokenizer.py index cacf166..d025625 100644 --- a/tests/_test_tokenizer.py +++ b/tests/_test_tokenizer.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # -# Copyright (C) 2012-2015 Ben Kurtovic +# Copyright (C) 2012-2016 Ben Kurtovic # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal diff --git a/tests/_test_tree_equality.py b/tests/_test_tree_equality.py index 086f113..fe626ce 100644 --- a/tests/_test_tree_equality.py +++ b/tests/_test_tree_equality.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # -# Copyright (C) 2012-2015 Ben Kurtovic +# Copyright (C) 2012-2016 Ben Kurtovic # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal diff --git a/tests/test_argument.py b/tests/test_argument.py index 70d8006..de12eab 100644 --- a/tests/test_argument.py +++ b/tests/test_argument.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # -# Copyright (C) 2012-2015 Ben Kurtovic +# Copyright (C) 2012-2016 Ben Kurtovic # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal diff --git a/tests/test_attribute.py b/tests/test_attribute.py index b3e325d..7fe5772 100644 --- a/tests/test_attribute.py +++ b/tests/test_attribute.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # -# Copyright (C) 2012-2015 Ben Kurtovic +# Copyright (C) 2012-2016 Ben Kurtovic # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal diff --git a/tests/test_builder.py b/tests/test_builder.py index 9af4f21..eed5861 100644 --- a/tests/test_builder.py +++ b/tests/test_builder.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # -# Copyright (C) 2012-2015 Ben Kurtovic +# Copyright (C) 2012-2016 Ben Kurtovic # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal diff --git a/tests/test_comment.py b/tests/test_comment.py index ad13f4a..97a6503 100644 --- a/tests/test_comment.py +++ b/tests/test_comment.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # -# Copyright (C) 2012-2015 Ben Kurtovic +# Copyright (C) 2012-2016 Ben Kurtovic # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal diff --git a/tests/test_ctokenizer.py b/tests/test_ctokenizer.py index 0d37485..27ff237 100644 --- a/tests/test_ctokenizer.py +++ b/tests/test_ctokenizer.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # -# Copyright (C) 2012-2015 Ben Kurtovic +# Copyright (C) 2012-2016 Ben Kurtovic # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal diff --git a/tests/test_docs.py b/tests/test_docs.py index 1c94130..398be4c 100644 --- a/tests/test_docs.py +++ b/tests/test_docs.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # -# Copyright (C) 2012-2015 Ben Kurtovic +# Copyright (C) 2012-2016 Ben Kurtovic # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal diff --git a/tests/test_external_link.py b/tests/test_external_link.py index 5137247..3432ae1 100644 --- a/tests/test_external_link.py +++ b/tests/test_external_link.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # -# Copyright (C) 2012-2015 Ben Kurtovic +# Copyright (C) 2012-2016 Ben Kurtovic # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal diff --git a/tests/test_heading.py b/tests/test_heading.py index effc03b..cb7ac8b 100644 --- a/tests/test_heading.py +++ b/tests/test_heading.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # -# Copyright (C) 2012-2015 Ben Kurtovic +# Copyright (C) 2012-2016 Ben Kurtovic # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal diff --git a/tests/test_html_entity.py b/tests/test_html_entity.py index a13fd71..4aa176f 100644 --- a/tests/test_html_entity.py +++ b/tests/test_html_entity.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # -# Copyright (C) 2012-2015 Ben Kurtovic +# Copyright (C) 2012-2016 Ben Kurtovic # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal diff --git a/tests/test_parameter.py b/tests/test_parameter.py index 71b298c..44c30af 100644 --- a/tests/test_parameter.py +++ b/tests/test_parameter.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # -# Copyright (C) 2012-2015 Ben Kurtovic +# Copyright (C) 2012-2016 Ben Kurtovic # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal diff --git a/tests/test_parser.py b/tests/test_parser.py index 6885c37..d586ecd 100644 --- a/tests/test_parser.py +++ b/tests/test_parser.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # -# Copyright (C) 2012-2015 Ben Kurtovic +# Copyright (C) 2012-2016 Ben Kurtovic # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal diff --git a/tests/test_pytokenizer.py b/tests/test_pytokenizer.py index f009c14..f7f26b8 100644 --- a/tests/test_pytokenizer.py +++ b/tests/test_pytokenizer.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # -# Copyright (C) 2012-2015 Ben Kurtovic +# Copyright (C) 2012-2016 Ben Kurtovic # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal diff --git a/tests/test_roundtripping.py b/tests/test_roundtripping.py index 5c64535..a217e21 100644 --- a/tests/test_roundtripping.py +++ b/tests/test_roundtripping.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # -# Copyright (C) 2012-2015 Ben Kurtovic +# Copyright (C) 2012-2016 Ben Kurtovic # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal diff --git a/tests/test_smart_list.py b/tests/test_smart_list.py index 4a27a04..0330aed 100644 --- a/tests/test_smart_list.py +++ b/tests/test_smart_list.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # -# Copyright (C) 2012-2015 Ben Kurtovic +# Copyright (C) 2012-2016 Ben Kurtovic # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal diff --git a/tests/test_string_mixin.py b/tests/test_string_mixin.py index 09e2e63..08d5b9e 100644 --- a/tests/test_string_mixin.py +++ b/tests/test_string_mixin.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # -# Copyright (C) 2012-2015 Ben Kurtovic +# Copyright (C) 2012-2016 Ben Kurtovic # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal diff --git a/tests/test_tag.py b/tests/test_tag.py index 0f0040a..0ac75a9 100644 --- a/tests/test_tag.py +++ b/tests/test_tag.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # -# Copyright (C) 2012-2015 Ben Kurtovic +# Copyright (C) 2012-2016 Ben Kurtovic # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal diff --git a/tests/test_template.py b/tests/test_template.py index e990818..c306b60 100644 --- a/tests/test_template.py +++ b/tests/test_template.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # -# Copyright (C) 2012-2015 Ben Kurtovic +# Copyright (C) 2012-2016 Ben Kurtovic # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal diff --git a/tests/test_text.py b/tests/test_text.py index 9093824..d890323 100644 --- a/tests/test_text.py +++ b/tests/test_text.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # -# Copyright (C) 2012-2015 Ben Kurtovic +# Copyright (C) 2012-2016 Ben Kurtovic # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal diff --git a/tests/test_tokens.py b/tests/test_tokens.py index 98f9a56..b33c2f1 100644 --- a/tests/test_tokens.py +++ b/tests/test_tokens.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # -# Copyright (C) 2012-2015 Ben Kurtovic +# Copyright (C) 2012-2016 Ben Kurtovic # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal diff --git a/tests/test_utils.py b/tests/test_utils.py index a9d4119..342cfd7 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # -# Copyright (C) 2012-2015 Ben Kurtovic +# Copyright (C) 2012-2016 Ben Kurtovic # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal diff --git a/tests/test_wikicode.py b/tests/test_wikicode.py index d97830c..d0c11fd 100644 --- a/tests/test_wikicode.py +++ b/tests/test_wikicode.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # -# Copyright (C) 2012-2015 Ben Kurtovic +# Copyright (C) 2012-2016 Ben Kurtovic # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal diff --git a/tests/test_wikilink.py b/tests/test_wikilink.py index e95cd84..965d8d5 100644 --- a/tests/test_wikilink.py +++ b/tests/test_wikilink.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # -# Copyright (C) 2012-2015 Ben Kurtovic +# Copyright (C) 2012-2016 Ben Kurtovic # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal diff --git a/tests/tokenizer/integration.mwtest b/tests/tokenizer/integration.mwtest index 5b8ff25..831f4d0 100644 --- a/tests/tokenizer/integration.mwtest +++ b/tests/tokenizer/integration.mwtest @@ -332,3 +332,17 @@ name: wikilink_to_external_link_fallback_2 label: an external link enclosed in an extra pair of brackets (see issue #120) input: "[[http://example.com]]" output: [Text(text="["), ExternalLinkOpen(brackets=True), Text(text="http://example.com"), ExternalLinkClose(), Text(text="]")] + +--- + +name: tables_in_templates +label: catch error handling mistakes when wikitables are inside templates +input: "{{hello|test\n{|\n|} }}" +output: [TemplateOpen(), Text(text="hello"), TemplateParamSeparator(), Text(text="test\n"), TagOpenOpen(wiki_markup="{|"), Text(text="table"), TagCloseOpen(padding="\n"), TagOpenClose(wiki_markup="|}"), Text(text="table"), TagCloseClose(), Text(text=" "), TemplateClose()] + +--- + +name: tables_in_templates_2 +label: catch error handling mistakes when wikitables are inside templates +input: "{{hello|test\n{|\n| }}" +output: [TemplateOpen(), Text(text="hello"), TemplateParamSeparator(), Text(text="test\n{"), TemplateParamSeparator(), Text(text="\n"), TemplateParamSeparator(), Text(text=" "), TemplateClose()] diff --git a/tests/tokenizer/templates.mwtest b/tests/tokenizer/templates.mwtest index 1913f5d..dccee37 100644 --- a/tests/tokenizer/templates.mwtest +++ b/tests/tokenizer/templates.mwtest @@ -222,6 +222,13 @@ output: [TemplateOpen(), TemplateOpen(), TemplateOpen(), TemplateOpen(), Text(te --- +name: nested_two_args +label: template whose first parameter is unnamed with two templates, followed by a named parameter +input: "{{a|{{b}}{{c}}|d=e}}" +output: [TemplateOpen(), Text(text="a"), TemplateParamSeparator(), TemplateOpen(), Text(text="b"), TemplateClose(), TemplateOpen(), Text(text="c"), TemplateClose(), TemplateParamSeparator(), Text(text="d"), TemplateParamEquals(), Text(text="e"), TemplateClose()] + +--- + name: newlines_start label: a newline at the start of a template name input: "{{\nfoobar}}" @@ -365,7 +372,7 @@ output: [TemplateOpen(), Text(text="\nfoo\n"), TemplateParamSeparator(), Text(te name: newlines_wildcard_redux_invalid label: a variation of the newlines_wildcard_redux test that is invalid input: "{{\nfoo\n|\n{{\nb\nar\n|\nb\naz\n=\nb\niz\n}}\n=\nb\nuzz\n}}" -output: [Text(text="{{\nfoo\n|\n{{\nb\nar\n|\nb\naz\n=\nb\niz\n}}\n=\nb\nuzz\n}}")] +output: [TemplateOpen(), Text(text="\nfoo\n"), TemplateParamSeparator(), Text(text="\n{{\nb\nar\n"), TemplateParamSeparator(), Text(text="\nb\naz\n"), TemplateParamEquals(), Text(text="\nb\niz\n"), TemplateClose(), Text(text="\n=\nb\nuzz\n}}")] ---