Merge branch 'valhallasw-winbuild' into develop

Windows build setup (fixes #68)

.gitignore

@@ -1,4 +1,5 @@
 *.pyc
+*.pyd
 *.so
 *.dll
 *.egg
@@ -8,3 +9,4 @@ __pycache__
 build
 dist
 docs/_build
+scripts/*.log

CHANGELOG

@@ -1,5 +1,7 @@
 v0.4 (unreleased):
 
+- The parser is now distributed with Windows binaries, fixing an issue that
+  prevented Windows users from using the C tokenizer.
 - Added a script to test for memory leaks in scripts/memtest.py.
 - Added a script to do releases in scripts/release.sh.
 - skip_style_tags can now be passed to mwparserfromhell.parse() (previously,

README.rst

@@ -15,19 +15,17 @@ Full documentation is available on ReadTheDocs_. Development occurs on GitHub_.
 Installation
 ------------
 
-The easiest way to install the parser is through the `Python Package Index`_,
-so you can install the latest release with ``pip install mwparserfromhell``
-(`get pip`_). Alternatively, get the latest development version::
+The easiest way to install the parser is through the `Python Package Index`_;
+you can install the latest release with ``pip install mwparserfromhell``
+(`get pip`_). On Windows, make sure you have the latest version of pip
+installed by running ``pip install --upgrade pip``.
+
+Alternatively, get the latest development version::
 
     git clone https://github.com/earwig/mwparserfromhell.git
     cd mwparserfromhell
     python setup.py install
 
-If you get ``error: Unable to find vcvarsall.bat`` while installing, this is
-because Windows can't find the compiler for C extensions. Consult this
-`StackOverflow question`_ for help. You can also set ``ext_modules`` in
-``setup.py`` to an empty list to prevent the extension from building.
-
 You can run the comprehensive unit testing suite with
 ``python setup.py test -q``.
 

docs/changelog.rst

@@ -7,6 +7,8 @@ v0.4
 Unreleased
 (`changes <https://github.com/earwig/mwparserfromhell/compare/v0.3.3...develop>`__):
 
+- The parser is now distributed with Windows binaries, fixing an issue that
+  prevented Windows users from using the C tokenizer.
 - Added a script to test for memory leaks in :file:`scripts/memtest.py`.
 - Added a script to do releases in :file:`scripts/release.sh`.
 - *skip_style_tags* can now be passed to :py:func:`mwparserfromhell.parse()

scripts/win_build.py

@@ -0,0 +1,36 @@
+from __future__ import print_function
+import os
+from subprocess import call, STDOUT
+
+ENVIRONMENTS = ["26", "27", "32", "33", "34"]
+
+def run(pyver, cmds):
+    cmd = [r"C:\Python%s\Python.exe" % pyver, "setup.py"] + cmds
+    print(" ".join(cmd), end=" ")
+
+    with open("%s%s.log" % (cmds[0], pyver), "w") as logfile:
+        retval = call(cmd, stdout=logfile, stderr=STDOUT, cwd="..")
+    if not retval:
+        print("[OK]")
+    else:
+        print("[FAILED (%i)]" % retval)
+    return retval
+
+def main():
+    path = os.path.split(__file__)[0]
+    if path:
+        os.chdir(path)
+
+    print("Building Windows wheels for Python %s:" % ", ".join(ENVIRONMENTS))
+    for pyver in ENVIRONMENTS:
+        print()
+        try:
+            os.unlink("mwparserfromhell/parser/_tokenizer.pyd")
+        except OSError:
+            pass
+
+        if run(pyver, ["test"]) == 0:
+            run(pyver, ["bdist_wheel", "upload"])
+
+if __name__ == "__main__":
+    main()

setup.py

@@ -25,7 +25,7 @@ import sys
 
 if (sys.version_info[0] == 2 and sys.version_info[1] < 6) or \
    (sys.version_info[1] == 3 and sys.version_info[1] < 2):
-    raise Exception('mwparserfromhell needs Python 2.6+ or 3.2+')
+    raise Exception("mwparserfromhell needs Python 2.6+ or 3.2+")
 
 from setuptools import setup, find_packages, Extension
 
@@ -36,8 +36,8 @@ with open("README.rst") as fp:
     long_docs = fp.read()
 
 tokenizer = Extension("mwparserfromhell.parser._tokenizer",
-                      sources = ["mwparserfromhell/parser/tokenizer.c"],
-                      depends = ["mwparserfromhell/parser/tokenizer.h"])
+                      sources=["mwparserfromhell/parser/tokenizer.c"],
+                      depends=["mwparserfromhell/parser/tokenizer.h"])
 
 setup(
     name = "mwparserfromhell",

tests/_test_tokenizer.py

@@ -21,6 +21,7 @@
 # SOFTWARE.
 
 from __future__ import print_function, unicode_literals
+import codecs
 from os import listdir, path
 import sys
 
@@ -109,10 +110,8 @@ class TokenizerTestCase(object):
     def build(cls):
         """Load and install all tests from the 'tokenizer' directory."""
         def load_file(filename):
-            with open(filename, "rU") as fp:
+            with codecs.open(filename, "rU", encoding="utf8") as fp:
                 text = fp.read()
-                if not py3k:
-                    text = text.decode("utf8")
                 name = path.split(filename)[1][:0-len(extension)]
                 cls._load_tests(filename, name, text)