Some clean up for Python 2.6 support.

* Removed unittest2 dependency on Python >2.6.
* Moved discover_tests.py into tests/.
* tokenizer.c: Fixed errors noted by -Wshorten-64-to-32.

discover_tests.py

@@ -1,23 +0,0 @@
-# -*- coding: UTF-8 -*-
-
-
-"""
-Discover tests using ``unittest2`.
-
-It appears the default distutils test suite doesn't play nice with
-``setUpClass`` thereby making some tests fail. Using ``unittest2``
-to load tests seems to work around that issue.
-
-http://stackoverflow.com/a/17004409/753501
-"""
-
-
-# Standard:
-import os.path
-
-# External:
-import unittest2
-
-
-def additional_tests():
-    return unittest2.defaultTestLoader.discover(os.path.dirname(__file__))

mwparserfromhell/compat.py

@@ -10,6 +10,7 @@ types are meant to be imported directly from within the parser's modules.
 
 import sys
 
+py26 = (sys.version_info[0] == 2) and (sys.version_info[1] == 6)
 py3k = (sys.version_info[0] == 3)
 py32 = py3k and (sys.version_info[1] == 2)
 

mwparserfromhell/parser/tokenizer.c

@@ -440,7 +440,7 @@ static int
 Tokenizer_emit_textbuffer(Tokenizer* self, Textbuffer* buffer, int reverse)
 {
     Textbuffer *original = buffer;
-    int i;
+    long i;
 
     if (reverse) {
         do {
@@ -940,7 +940,8 @@ static int Tokenizer_parse_free_uri_scheme(Tokenizer* self)
     Textbuffer *scheme_buffer = Textbuffer_new(), *temp_buffer;
     PyObject *scheme;
     Py_UNICODE chunk;
-    int slashes, i, j;
+    long i;
+    int slashes, j;
 
     if (!scheme_buffer)
         return -1;
@@ -1296,8 +1297,8 @@ static int Tokenizer_parse_heading(Tokenizer* self)
 */
 static HeadingData* Tokenizer_handle_heading_end(Tokenizer* self)
 {
-    Py_ssize_t reset = self->head, best;
-    int i, current, level, diff;
+    Py_ssize_t reset = self->head;
+    int best, i, current, level, diff;
     HeadingData *after, *heading;
     PyObject *stack;
 

mwparserfromhell/parser/tokenizer.py

@@ -620,7 +620,8 @@ class Tokenizer(object):
         self._emit_first(tokens.TagAttrStart(pad_first=buf["first"],
             pad_before_eq=buf["before_eq"], pad_after_eq=buf["after_eq"]))
         self._emit_all(self._pop())
-        data.padding_buffer = dict((key, "") for key in data.padding_buffer)
+        for key in data.padding_buffer:
+            data.padding_buffer[key] = ""
 
     def _handle_tag_space(self, data, text):
         """Handle whitespace (*text*) inside of an HTML open tag."""

mwparserfromhell/string_mixin.py

@@ -28,7 +28,7 @@ interface for the ``unicode`` type (``str`` on py3k) in a dynamic manner.
 from __future__ import unicode_literals
 from sys import getdefaultencoding
 
-from .compat import bytes, py3k, str
+from .compat import bytes, py26, py3k, str
 
 __all__ = ["StringMixIn"]
 
@@ -108,20 +108,20 @@ class StringMixIn(object):
     def __contains__(self, item):
         return str(item) in self.__unicode__()
 
-    @inheritdoc
-    def encode(self, encoding=None, errors=None):
-        if encoding is None:
-            encoding = getdefaultencoding()
-        args = [encoding]
-        if errors is not None:
-            args.append(errors)
-        return self.__unicode__().encode(*args)
-
     def __getattr__(self, attr):
         return getattr(self.__unicode__(), attr)
 
     if py3k:
         maketrans = str.maketrans  # Static method can't rely on __getattr__
 
+    if py26:
+        @inheritdoc
+        def encode(self, encoding=None, errors=None):
+            if encoding is None:
+                encoding = getdefaultencoding()
+            if errors is not None:
+                return self.__unicode__().encode(encoding, errors)
+            return self.__unicode__().encode(encoding)
+
 
 del inheritdoc

setup.py

@@ -30,7 +30,7 @@ if (sys.version_info[0] == 2 and sys.version_info[1] < 6) or \
 from setuptools import setup, find_packages, Extension
 
 from mwparserfromhell import __version__
-from mwparserfromhell.compat import py3k
+from mwparserfromhell.compat import py26, py3k
 
 with open("README.rst") as fp:
     long_docs = fp.read()
@@ -42,8 +42,8 @@ setup(
     name = "mwparserfromhell",
     packages = find_packages(exclude=("tests",)),
     ext_modules = [tokenizer],
-    tests_require = ['unittest2py3k' if py3k else 'unittest2'],
-    test_suite = "discover_tests",
+    tests_require = ["unittest2"] if py26 else [],
+    test_suite = "tests.discover",
     version = __version__,
     author = "Ben Kurtovic",
     author_email = "ben.kurtovic@gmail.com",

tests/_test_tokenizer.py

@@ -35,7 +35,7 @@ class _TestParseError(Exception):
 class TokenizerTestCase(object):
     """A base test case for tokenizers, whose tests are loaded dynamically.
 
-    Subclassed along with unittest2.TestCase to form TestPyTokenizer and
+    Subclassed along with unittest.TestCase to form TestPyTokenizer and
     TestCTokenizer. Tests are loaded dynamically from files in the 'tokenizer'
     directory.
     """
@@ -121,7 +121,7 @@ class TokenizerTestCase(object):
         if len(sys.argv) > 2 and sys.argv[1] == "--use":
             for name in sys.argv[2:]:
                 load_file(path.join(directory, name + extension))
-            sys.argv = [sys.argv[0]]  # So unittest2 doesn't try to load these
+            sys.argv = [sys.argv[0]]  # So unittest doesn't try to load these
             cls.skip_others = True
         else:
             for filename in listdir(directory):

tests/_test_tree_equality.py

@@ -21,7 +21,11 @@
 # SOFTWARE.
 
 from __future__ import unicode_literals
-from unittest2 import TestCase
+
+try:
+    from unittest2 import TestCase
+except ImportError:
+    from unittest import TestCase
 
 from mwparserfromhell.compat import range
 from mwparserfromhell.nodes import (Argument, Comment, Heading, HTMLEntity,

tests/discover.py

@@ -0,0 +1,24 @@
+# -*- coding: utf-8 -*-
+
+"""
+Discover tests using ``unittest2` for Python 2.6.
+
+It appears the default distutils test suite doesn't play nice with
+``setUpClass`` thereby making some tests fail. Using ``unittest2`` to load
+tests seems to work around that issue.
+
+http://stackoverflow.com/a/17004409/753501
+"""
+
+import os.path
+
+from mwparserfromhell.compat import py26
+
+if py26:
+    import unittest2 as unittest
+else:
+    import unittest
+
+def additional_tests():
+    project_root = os.path.split(os.path.dirname(__file__))[0]
+    return unittest.defaultTestLoader.discover(project_root)

tests/test_argument.py

@@ -21,7 +21,11 @@
 # SOFTWARE.
 
 from __future__ import unicode_literals
-import unittest2
+
+try:
+    import unittest2 as unittest
+except ImportError:
+    import unittest
 
 from mwparserfromhell.compat import str
 from mwparserfromhell.nodes import Argument, Text
@@ -99,4 +103,4 @@ class TestArgument(TreeEqualityTestCase):
         self.assertIs(None, node2.default)
 
 if __name__ == "__main__":
-    unittest2.main(verbosity=2)
+    unittest.main(verbosity=2)

tests/test_attribute.py

@@ -21,7 +21,11 @@
 # SOFTWARE.
 
 from __future__ import unicode_literals
-import unittest2
+
+try:
+    import unittest2 as unittest
+except ImportError:
+    import unittest
 
 from mwparserfromhell.compat import str
 from mwparserfromhell.nodes import Template
@@ -86,4 +90,4 @@ class TestAttribute(TreeEqualityTestCase):
             self.assertRaises(ValueError, setattr, node, pad, True)
 
 if __name__ == "__main__":
-    unittest2.main(verbosity=2)
+    unittest.main(verbosity=2)

tests/test_builder.py

@@ -21,7 +21,11 @@
 # SOFTWARE.
 
 from __future__ import unicode_literals
-import unittest2
+
+try:
+    import unittest2 as unittest
+except ImportError:
+    import unittest
 
 from mwparserfromhell.nodes import (Argument, Comment, ExternalLink, Heading,
                                     HTMLEntity, Tag, Template, Text, Wikilink)
@@ -417,4 +421,4 @@ class TestBuilder(TreeEqualityTestCase):
         self.assertWikicodeEqual(valid, self.builder.build(test))
 
 if __name__ == "__main__":
-    unittest2.main(verbosity=2)
+    unittest.main(verbosity=2)

tests/test_comment.py

@@ -21,7 +21,11 @@
 # SOFTWARE.
 
 from __future__ import unicode_literals
-import unittest2
+
+try:
+    import unittest2 as unittest
+except ImportError:
+    import unittest
 
 from mwparserfromhell.compat import str
 from mwparserfromhell.nodes import Comment
@@ -64,4 +68,4 @@ class TestComment(TreeEqualityTestCase):
         self.assertEqual("barfoo", node.contents)
 
 if __name__ == "__main__":
-    unittest2.main(verbosity=2)
+    unittest.main(verbosity=2)

tests/test_ctokenizer.py

@@ -21,7 +21,11 @@
 # SOFTWARE.
 
 from __future__ import unicode_literals
-import unittest2
+
+try:
+    import unittest2 as unittest
+except ImportError:
+    import unittest
 
 try:
     from mwparserfromhell.parser._tokenizer import CTokenizer
@@ -30,8 +34,8 @@ except ImportError:
 
 from ._test_tokenizer import TokenizerTestCase
 
-@unittest2.skipUnless(CTokenizer, "C tokenizer not available")
-class TestCTokenizer(TokenizerTestCase, unittest2.TestCase):
+@unittest.skipUnless(CTokenizer, "C tokenizer not available")
+class TestCTokenizer(TokenizerTestCase, unittest.TestCase):
     """Test cases for the C tokenizer."""
 
     @classmethod
@@ -45,4 +49,4 @@ class TestCTokenizer(TokenizerTestCase, unittest2.TestCase):
             self.assertTrue(CTokenizer().USES_C)
 
 if __name__ == "__main__":
-    unittest2.main(verbosity=2)
+    unittest.main(verbosity=2)

tests/test_docs.py

@@ -22,14 +22,18 @@
 
 from __future__ import print_function, unicode_literals
 import json
-import unittest2
+
+try:
+    import unittest2 as unittest
+except ImportError:
+    import unittest
 
 import mwparserfromhell
 from mwparserfromhell.compat import py3k, str
 
 from .compat import StringIO, urlencode, urlopen
 
-class TestDocs(unittest2.TestCase):
+class TestDocs(unittest.TestCase):
     """Integration test cases for mwparserfromhell's documentation."""
 
     def assertPrint(self, input, output):
@@ -128,4 +132,4 @@ class TestDocs(unittest2.TestCase):
         self.assertEqual(expected, actual)
 
 if __name__ == "__main__":
-    unittest2.main(verbosity=2)
+    unittest.main(verbosity=2)

tests/test_external_link.py

@@ -21,7 +21,11 @@
 # SOFTWARE.
 
 from __future__ import unicode_literals
-import unittest2
+
+try:
+    import unittest2 as unittest
+except ImportError:
+    import unittest
 
 from mwparserfromhell.compat import str
 from mwparserfromhell.nodes import ExternalLink, Text
@@ -122,4 +126,4 @@ class TestExternalLink(TreeEqualityTestCase):
         self.assertEqual("http://example.com/", str(node2))
 
 if __name__ == "__main__":
-    unittest2.main(verbosity=2)
+    unittest.main(verbosity=2)

tests/test_heading.py

@@ -21,7 +21,11 @@
 # SOFTWARE.
 
 from __future__ import unicode_literals
-import unittest2
+
+try:
+    import unittest2 as unittest
+except ImportError:
+    import unittest
 
 from mwparserfromhell.compat import str
 from mwparserfromhell.nodes import Heading, Text
@@ -85,4 +89,4 @@ class TestHeading(TreeEqualityTestCase):
         self.assertRaises(ValueError, setattr, node, "level", False)
 
 if __name__ == "__main__":
-    unittest2.main(verbosity=2)
+    unittest.main(verbosity=2)

tests/test_html_entity.py

@@ -21,7 +21,11 @@
 # SOFTWARE.
 
 from __future__ import unicode_literals
-import unittest2
+
+try:
+    import unittest2 as unittest
+except ImportError:
+    import unittest
 
 from mwparserfromhell.compat import str
 from mwparserfromhell.nodes import HTMLEntity
@@ -165,4 +169,4 @@ class TestHTMLEntity(TreeEqualityTestCase):
         self.assertEqual("\U0001F648", node4.normalize())
 
 if __name__ == "__main__":
-    unittest2.main(verbosity=2)
+    unittest.main(verbosity=2)

tests/test_parameter.py

@@ -21,7 +21,11 @@
 # SOFTWARE.
 
 from __future__ import unicode_literals
-import unittest2
+
+try:
+    import unittest2 as unittest
+except ImportError:
+    import unittest
 
 from mwparserfromhell.compat import str
 from mwparserfromhell.nodes import Text
@@ -72,4 +76,4 @@ class TestParameter(TreeEqualityTestCase):
         self.assertFalse(node2.showkey)
 
 if __name__ == "__main__":
-    unittest2.main(verbosity=2)
+    unittest.main(verbosity=2)

tests/test_parser.py

@@ -21,7 +21,11 @@
 # SOFTWARE.
 
 from __future__ import unicode_literals
-import unittest2
+
+try:
+    import unittest2 as unittest
+except ImportError:
+    import unittest
 
 from mwparserfromhell import parser
 from mwparserfromhell.compat import range
@@ -86,4 +90,4 @@ class TestParser(TreeEqualityTestCase):
         parser.use_c = restore
 
 if __name__ == "__main__":
-    unittest2.main(verbosity=2)
+    unittest.main(verbosity=2)

tests/test_pytokenizer.py

@@ -21,13 +21,17 @@
 # SOFTWARE.
 
 from __future__ import unicode_literals
-import unittest2
+
+try:
+    import unittest2 as unittest
+except ImportError:
+    import unittest
 
 from mwparserfromhell.parser.tokenizer import Tokenizer
 
 from ._test_tokenizer import TokenizerTestCase
 
-class TestPyTokenizer(TokenizerTestCase, unittest2.TestCase):
+class TestPyTokenizer(TokenizerTestCase, unittest.TestCase):
     """Test cases for the Python tokenizer."""
 
     @classmethod
@@ -41,4 +45,4 @@ class TestPyTokenizer(TokenizerTestCase, unittest2.TestCase):
             self.assertFalse(Tokenizer().USES_C)
 
 if __name__ == "__main__":
-    unittest2.main(verbosity=2)
+    unittest.main(verbosity=2)

tests/test_smart_list.py

@@ -21,12 +21,16 @@
 # SOFTWARE.
 
 from __future__ import unicode_literals
-import unittest2
+
+try:
+    import unittest2 as unittest
+except ImportError:
+    import unittest
 
 from mwparserfromhell.compat import py3k, range
 from mwparserfromhell.smart_list import SmartList, _ListProxy
 
-class TestSmartList(unittest2.TestCase):
+class TestSmartList(unittest.TestCase):
     """Test cases for the SmartList class and its child, _ListProxy."""
 
     def _test_get_set_del_item(self, builder):
@@ -387,4 +391,4 @@ class TestSmartList(unittest2.TestCase):
         self.assertEqual([4, 3, 2, 1.9, 1.8], child2)
 
 if __name__ == "__main__":
-    unittest2.main(verbosity=2)
+    unittest.main(verbosity=2)

tests/test_string_mixin.py

@@ -23,7 +23,11 @@
 from __future__ import unicode_literals
 from sys import getdefaultencoding
 from types import GeneratorType
-import unittest2
+
+try:
+    import unittest2 as unittest
+except ImportError:
+    import unittest
 
 from mwparserfromhell.compat import bytes, py3k, py32, range, str
 from mwparserfromhell.string_mixin import StringMixIn
@@ -36,7 +40,7 @@ class _FakeString(StringMixIn):
         return self._data
 
 
-class TestStringMixIn(unittest2.TestCase):
+class TestStringMixIn(unittest.TestCase):
     """Test cases for the StringMixIn class."""
 
     def test_docs(self):
@@ -432,4 +436,4 @@ class TestStringMixIn(unittest2.TestCase):
         self.assertEqual("000123", str12.zfill(6))
 
 if __name__ == "__main__":
-    unittest2.main(verbosity=2)
+    unittest.main(verbosity=2)

tests/test_tag.py

@@ -21,7 +21,11 @@
 # SOFTWARE.
 
 from __future__ import unicode_literals
-import unittest2
+
+try:
+    import unittest2 as unittest
+except ImportError:
+    import unittest
 
 from mwparserfromhell.compat import str
 from mwparserfromhell.nodes import Tag, Template, Text
@@ -305,4 +309,4 @@ class TestTag(TreeEqualityTestCase):
         self.assertEqual('<div/>', node)
 
 if __name__ == "__main__":
-    unittest2.main(verbosity=2)
+    unittest.main(verbosity=2)

tests/test_template.py

@@ -21,7 +21,11 @@
 # SOFTWARE.
 
 from __future__ import unicode_literals
-import unittest2
+
+try:
+    import unittest2 as unittest
+except ImportError:
+    import unittest
 
 from mwparserfromhell.compat import str
 from mwparserfromhell.nodes import HTMLEntity, Template, Text
@@ -428,4 +432,4 @@ class TestTemplate(TreeEqualityTestCase):
         self.assertEqual("{{foo|a=b|c=d|e=f|a=|a=b}}", node26)
 
 if __name__ == "__main__":
-    unittest2.main(verbosity=2)
+    unittest.main(verbosity=2)

tests/test_text.py

@@ -21,12 +21,16 @@
 # SOFTWARE.
 
 from __future__ import unicode_literals
-import unittest2
+
+try:
+    import unittest2 as unittest
+except ImportError:
+    import unittest
 
 from mwparserfromhell.compat import str
 from mwparserfromhell.nodes import Text
 
-class TestText(unittest2.TestCase):
+class TestText(unittest.TestCase):
     """Test cases for the Text node."""
 
     def test_unicode(self):
@@ -71,4 +75,4 @@ class TestText(unittest2.TestCase):
         self.assertIsInstance(node.value, str)
 
 if __name__ == "__main__":
-    unittest2.main(verbosity=2)
+    unittest.main(verbosity=2)

tests/test_tokens.py

@@ -21,12 +21,16 @@
 # SOFTWARE.
 
 from __future__ import unicode_literals
-import unittest2
+
+try:
+    import unittest2 as unittest
+except ImportError:
+    import unittest
 
 from mwparserfromhell.compat import py3k
 from mwparserfromhell.parser import tokens
 
-class TestTokens(unittest2.TestCase):
+class TestTokens(unittest.TestCase):
     """Test cases for the Token class and its subclasses."""
 
     def test_issubclass(self):
@@ -105,4 +109,4 @@ class TestTokens(unittest2.TestCase):
             self.assertEqual(token, eval(repr(token), vars(tokens)))
 
 if __name__ == "__main__":
-    unittest2.main(verbosity=2)
+    unittest.main(verbosity=2)

tests/test_utils.py

@@ -21,7 +21,11 @@
 # SOFTWARE.
 
 from __future__ import unicode_literals
-import unittest2
+
+try:
+    import unittest2 as unittest
+except ImportError:
+    import unittest
 
 from mwparserfromhell.nodes import Template, Text
 from mwparserfromhell.utils import parse_anything
@@ -59,4 +63,4 @@ class TestUtils(TreeEqualityTestCase):
         self.assertRaises(ValueError, parse_anything, ["foo", [object]])
 
 if __name__ == "__main__":
-    unittest2.main(verbosity=2)
+    unittest.main(verbosity=2)

tests/test_wikicode.py

@@ -24,7 +24,11 @@ from __future__ import unicode_literals
 from functools import partial
 import re
 from types import GeneratorType
-import unittest2
+
+try:
+    import unittest2 as unittest
+except ImportError:
+    import unittest
 
 from mwparserfromhell.compat import py3k, str
 from mwparserfromhell.nodes import (Argument, Comment, Heading, HTMLEntity,
@@ -432,4 +436,4 @@ class TestWikicode(TreeEqualityTestCase):
         self.assertEqual(expected.expandtabs(4), code.get_tree())
 
 if __name__ == "__main__":
-    unittest2.main(verbosity=2)
+    unittest.main(verbosity=2)

tests/test_wikilink.py

@@ -21,7 +21,11 @@
 # SOFTWARE.
 
 from __future__ import unicode_literals
-import unittest2
+
+try:
+    import unittest2 as unittest
+except ImportError:
+    import unittest
 
 from mwparserfromhell.compat import str
 from mwparserfromhell.nodes import Text, Wikilink
@@ -99,4 +103,4 @@ class TestWikilink(TreeEqualityTestCase):
         self.assertIs(None, node2.text)
 
 if __name__ == "__main__":
-    unittest2.main(verbosity=2)
+    unittest.main(verbosity=2)