From 3dd29097e4d436f4fa7a01e2c4213c528168b242 Mon Sep 17 00:00:00 2001 From: Merlijn van Deen Date: Fri, 27 Jun 2014 15:21:16 -0700 Subject: [PATCH 1/6] _test_tokenizer: force utf-8 file encoding --- tests/_test_tokenizer.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/_test_tokenizer.py b/tests/_test_tokenizer.py index 7487241..313b959 100644 --- a/tests/_test_tokenizer.py +++ b/tests/_test_tokenizer.py @@ -25,6 +25,8 @@ from os import listdir, path import sys from mwparserfromhell.compat import py3k +if not py3k: + from codecs import open from mwparserfromhell.parser import tokens class _TestParseError(Exception): @@ -109,10 +111,8 @@ class TokenizerTestCase(object): def build(cls): """Load and install all tests from the 'tokenizer' directory.""" def load_file(filename): - with open(filename, "rU") as fp: + with open(filename, "rU", encoding='utf8') as fp: text = fp.read() - if not py3k: - text = text.decode("utf8") name = path.split(filename)[1][:0-len(extension)] cls._load_tests(filename, name, text) From b135e8e473837909c6847f8a52711527409b5224 Mon Sep 17 00:00:00 2001 From: Merlijn van Deen Date: Fri, 27 Jun 2014 15:21:37 -0700 Subject: [PATCH 2/6] Add windows build tools --- tools/build_mwpfh.py | 43 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) create mode 100644 tools/build_mwpfh.py diff --git a/tools/build_mwpfh.py b/tools/build_mwpfh.py new file mode 100644 index 0000000..a090b4d --- /dev/null +++ b/tools/build_mwpfh.py @@ -0,0 +1,43 @@ +from __future__ import print_function + +import subprocess +import sys +import os + +path = os.path.split(__file__)[0] +if path: + os.chdir(path) + +environments = ['26', '27', '32', '33', '34'] + +target = "pypi" if "--push" in sys.argv else "test" + +returnvalues = {} + +def run(pyver, cmds, target=None): + cmd = [r"C:\Python%s\Python.exe" % pyver, "setup.py"] + cmds + if target: + cmd += ["-r", target] + + print(" ".join(cmd), end=" ") + retval = subprocess.call(cmd, stdout=open("%s%s.log" % (cmds[0], pyver), 'w'), stderr=subprocess.STDOUT, cwd="..") + if not retval: + print("[OK]") + else: + print("[FAILED (%i)]" % retval) + return retval + +run("27", ["register"], target) + +if 'failed' in open('register27.log').read(): + raise Exception + +for pyver in environments: + print() + try: + os.unlink('mwparserfromhell/parser/_tokenizer.pyd') + except WindowsError: + pass + + if run(pyver, ["test"]) == 0: + run(pyver, ["bdist_wheel", "upload"], target) \ No newline at end of file From 05d048762f2e0c81a9e425425269dcdde4bec251 Mon Sep 17 00:00:00 2001 From: Merlijn van Deen Date: Fri, 27 Jun 2014 15:21:45 -0700 Subject: [PATCH 3/6] Improve .gitignore --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index 4068716..8790182 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ *.pyc +*.pyd *.so *.dll *.egg @@ -8,3 +9,4 @@ __pycache__ build dist docs/_build +tools/*.log From 581ca9a2213d6329a45d3b927873febe9e5ad479 Mon Sep 17 00:00:00 2001 From: Merlijn van Deen Date: Sat, 28 Jun 2014 00:35:50 +0200 Subject: [PATCH 4/6] Update README.rst for the new Windows wheels --- README.rst | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 5b4cfe1..d0f67c9 100644 --- a/README.rst +++ b/README.rst @@ -17,7 +17,10 @@ Installation The easiest way to install the parser is through the `Python Package Index`_, so you can install the latest release with ``pip install mwparserfromhell`` -(`get pip`_). Alternatively, get the latest development version:: +(`get pip`_). On Windows, make sure you have the latest version of pip +installed by running `pip install --upgrade pip`. + +Alternatively, get the latest development version:: git clone https://github.com/earwig/mwparserfromhell.git cd mwparserfromhell From 5e9930b8a060d2ad80713809e312eae913ce7a4f Mon Sep 17 00:00:00 2001 From: Merlijn van Deen Date: Sat, 28 Jun 2014 00:37:31 +0200 Subject: [PATCH 5/6] Fix tabs in update_mwpfh.py --- tools/build_mwpfh.py | 42 +++++++++++++++++++++--------------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/tools/build_mwpfh.py b/tools/build_mwpfh.py index a090b4d..4a86241 100644 --- a/tools/build_mwpfh.py +++ b/tools/build_mwpfh.py @@ -6,7 +6,7 @@ import os path = os.path.split(__file__)[0] if path: - os.chdir(path) + os.chdir(path) environments = ['26', '27', '32', '33', '34'] @@ -15,29 +15,29 @@ target = "pypi" if "--push" in sys.argv else "test" returnvalues = {} def run(pyver, cmds, target=None): - cmd = [r"C:\Python%s\Python.exe" % pyver, "setup.py"] + cmds - if target: - cmd += ["-r", target] - - print(" ".join(cmd), end=" ") - retval = subprocess.call(cmd, stdout=open("%s%s.log" % (cmds[0], pyver), 'w'), stderr=subprocess.STDOUT, cwd="..") - if not retval: - print("[OK]") - else: - print("[FAILED (%i)]" % retval) - return retval +cmd = [r"C:\Python%s\Python.exe" % pyver, "setup.py"] + cmds +if target: + cmd += ["-r", target] + +print(" ".join(cmd), end=" ") +retval = subprocess.call(cmd, stdout=open("%s%s.log" % (cmds[0], pyver), 'w'), stderr=subprocess.STDOUT, cwd="..") +if not retval: + print("[OK]") +else: + print("[FAILED (%i)]" % retval) +return retval run("27", ["register"], target) if 'failed' in open('register27.log').read(): - raise Exception + raise Exception for pyver in environments: - print() - try: - os.unlink('mwparserfromhell/parser/_tokenizer.pyd') - except WindowsError: - pass - - if run(pyver, ["test"]) == 0: - run(pyver, ["bdist_wheel", "upload"], target) \ No newline at end of file + print() + try: + os.unlink('mwparserfromhell/parser/_tokenizer.pyd') + except WindowsError: + pass + + if run(pyver, ["test"]) == 0: + run(pyver, ["bdist_wheel", "upload"], target) From d8adb62454f464f39b59c179f52ddb17621a2e18 Mon Sep 17 00:00:00 2001 From: Ben Kurtovic Date: Fri, 27 Jun 2014 19:56:18 -0400 Subject: [PATCH 6/6] Some tweaks, but no change in functionality. --- .gitignore | 2 +- README.rst | 11 +++-------- scripts/win_build.py | 36 ++++++++++++++++++++++++++++++++++++ setup.py | 4 ++-- tests/_test_tokenizer.py | 5 ++--- tools/build_mwpfh.py | 43 ------------------------------------------- 6 files changed, 44 insertions(+), 57 deletions(-) create mode 100644 scripts/win_build.py delete mode 100644 tools/build_mwpfh.py diff --git a/.gitignore b/.gitignore index 8790182..f7f7bd9 100644 --- a/.gitignore +++ b/.gitignore @@ -9,4 +9,4 @@ __pycache__ build dist docs/_build -tools/*.log +scripts/*.log diff --git a/README.rst b/README.rst index d0f67c9..c112afd 100644 --- a/README.rst +++ b/README.rst @@ -15,10 +15,10 @@ Full documentation is available on ReadTheDocs_. Development occurs on GitHub_. Installation ------------ -The easiest way to install the parser is through the `Python Package Index`_, -so you can install the latest release with ``pip install mwparserfromhell`` +The easiest way to install the parser is through the `Python Package Index`_; +you can install the latest release with ``pip install mwparserfromhell`` (`get pip`_). On Windows, make sure you have the latest version of pip -installed by running `pip install --upgrade pip`. +installed by running ``pip install --upgrade pip``. Alternatively, get the latest development version:: @@ -26,11 +26,6 @@ Alternatively, get the latest development version:: cd mwparserfromhell python setup.py install -If you get ``error: Unable to find vcvarsall.bat`` while installing, this is -because Windows can't find the compiler for C extensions. Consult this -`StackOverflow question`_ for help. You can also set ``ext_modules`` in -``setup.py`` to an empty list to prevent the extension from building. - You can run the comprehensive unit testing suite with ``python setup.py test -q``. diff --git a/scripts/win_build.py b/scripts/win_build.py new file mode 100644 index 0000000..c70dedc --- /dev/null +++ b/scripts/win_build.py @@ -0,0 +1,36 @@ +from __future__ import print_function +import os +from subprocess import call, STDOUT + +ENVIRONMENTS = ["26", "27", "32", "33", "34"] + +def run(pyver, cmds): + cmd = [r"C:\Python%s\Python.exe" % pyver, "setup.py"] + cmds + print(" ".join(cmd), end=" ") + + with open("%s%s.log" % (cmds[0], pyver), "w") as logfile: + retval = call(cmd, stdout=logfile, stderr=STDOUT, cwd="..") + if not retval: + print("[OK]") + else: + print("[FAILED (%i)]" % retval) + return retval + +def main(): + path = os.path.split(__file__)[0] + if path: + os.chdir(path) + + print("Building Windows wheels for Python %s:" % ", ".join(ENVIRONMENTS)) + for pyver in ENVIRONMENTS: + print() + try: + os.unlink("mwparserfromhell/parser/_tokenizer.pyd") + except OSError: + pass + + if run(pyver, ["test"]) == 0: + run(pyver, ["bdist_wheel", "upload"]) + +if __name__ == "__main__": + main() diff --git a/setup.py b/setup.py index 5a45902..07fb330 100644 --- a/setup.py +++ b/setup.py @@ -25,7 +25,7 @@ import sys if (sys.version_info[0] == 2 and sys.version_info[1] < 6) or \ (sys.version_info[1] == 3 and sys.version_info[1] < 2): - raise Exception('mwparserfromhell needs Python 2.6+ or 3.2+') + raise Exception("mwparserfromhell needs Python 2.6+ or 3.2+") from setuptools import setup, find_packages, Extension @@ -36,7 +36,7 @@ with open("README.rst") as fp: long_docs = fp.read() tokenizer = Extension("mwparserfromhell.parser._tokenizer", - sources = ["mwparserfromhell/parser/tokenizer.c"]) + sources=["mwparserfromhell/parser/tokenizer.c"]) setup( name = "mwparserfromhell", diff --git a/tests/_test_tokenizer.py b/tests/_test_tokenizer.py index 313b959..bfd4857 100644 --- a/tests/_test_tokenizer.py +++ b/tests/_test_tokenizer.py @@ -21,12 +21,11 @@ # SOFTWARE. from __future__ import print_function, unicode_literals +import codecs from os import listdir, path import sys from mwparserfromhell.compat import py3k -if not py3k: - from codecs import open from mwparserfromhell.parser import tokens class _TestParseError(Exception): @@ -111,7 +110,7 @@ class TokenizerTestCase(object): def build(cls): """Load and install all tests from the 'tokenizer' directory.""" def load_file(filename): - with open(filename, "rU", encoding='utf8') as fp: + with codecs.open(filename, "rU", encoding="utf8") as fp: text = fp.read() name = path.split(filename)[1][:0-len(extension)] cls._load_tests(filename, name, text) diff --git a/tools/build_mwpfh.py b/tools/build_mwpfh.py deleted file mode 100644 index 4a86241..0000000 --- a/tools/build_mwpfh.py +++ /dev/null @@ -1,43 +0,0 @@ -from __future__ import print_function - -import subprocess -import sys -import os - -path = os.path.split(__file__)[0] -if path: - os.chdir(path) - -environments = ['26', '27', '32', '33', '34'] - -target = "pypi" if "--push" in sys.argv else "test" - -returnvalues = {} - -def run(pyver, cmds, target=None): -cmd = [r"C:\Python%s\Python.exe" % pyver, "setup.py"] + cmds -if target: - cmd += ["-r", target] - -print(" ".join(cmd), end=" ") -retval = subprocess.call(cmd, stdout=open("%s%s.log" % (cmds[0], pyver), 'w'), stderr=subprocess.STDOUT, cwd="..") -if not retval: - print("[OK]") -else: - print("[FAILED (%i)]" % retval) -return retval - -run("27", ["register"], target) - -if 'failed' in open('register27.log').read(): - raise Exception - -for pyver in environments: - print() - try: - os.unlink('mwparserfromhell/parser/_tokenizer.pyd') - except WindowsError: - pass - - if run(pyver, ["test"]) == 0: - run(pyver, ["bdist_wheel", "upload"], target)