From 067baef238602cbcd1afe46f388e035a408c1530 Mon Sep 17 00:00:00 2001 From: Ben Kurtovic Date: Sun, 29 Jun 2025 23:51:38 -0400 Subject: [PATCH] Support $WITH_EXTENSION to control building the C tokenizer --- CHANGELOG | 4 ++- README.rst | 9 ++++- docs/changelog.rst | 4 ++- docs/index.rst | 15 +++++++- pyproject.toml | 29 ++-------------- setup.py | 100 +++++++++++++++++++++++++++++++++++++++++++++++++++++ uv.lock | 11 ++++++ 7 files changed, 141 insertions(+), 31 deletions(-) create mode 100644 setup.py diff --git a/CHANGELOG b/CHANGELOG index 7759a47..bea6875 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,6 +1,8 @@ v0.7.1 (unreleased): -- Modernized packaging. +- Modernize packaging. +- Explicitly support $WITH_EXTENSION/$WITHOUT_EXTENSION to control whether the + C tokenizer should be built. v0.7.0 (released June 28, 2025): diff --git a/README.rst b/README.rst index 9105c40..020467e 100644 --- a/README.rst +++ b/README.rst @@ -18,7 +18,13 @@ Installation The easiest way to install the parser is from `PyPI`_; you can install the latest release with ``pip install mwparserfromhell``. -Alternatively, get the latest development version: +Prebuilt wheels are available on PyPI with a fast, compiled C tokenizer +extension for most environments (Linux x86_64 and arm64, macOS x86_64 and +arm64, Windows x86 and x86_64). If building from source and the C tokenizer +cannot be built, you can fall back to the slower pure-Python implementation by +setting the environment variable ``WITH_EXTENSION=0`` when installing. + +To get the latest development version (with `uv`_): .. code-block:: sh @@ -223,6 +229,7 @@ Python 3 code (using the API_ and the requests_ library): .. _Legoktm: https://en.wikipedia.org/wiki/User:Legoktm .. _GitHub: https://github.com/earwig/mwparserfromhell .. _PyPI: https://pypi.org/project/mwparserfromhell/ +.. _uv: https://docs.astral.sh/uv/ .. _pytest: https://docs.pytest.org/ .. _Word-ending links: https://www.mediawiki.org/wiki/Help:Links#linktrail .. _EarwigBot: https://github.com/earwig/earwigbot diff --git a/docs/changelog.rst b/docs/changelog.rst index ada2d86..22f4bca 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -7,7 +7,9 @@ v0.7.1 Unreleased (`changes `__): -- Modernized packaging. +- Modernize packaging. +- Explicitly support ``$WITH_EXTENSION`` / ``$WITHOUT_EXTENSION`` to control + whether the C tokenizer should be built. v0.7.0 ------ diff --git a/docs/index.rst b/docs/index.rst index 7f07933..44a6d93 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -20,7 +20,13 @@ Installation The easiest way to install the parser is from `PyPI`_; you can install the latest release with ``pip install mwparserfromhell``. -Alternatively, get the latest development version:: +Prebuilt wheels are available on PyPI with a fast, compiled C tokenizer +extension for most environments (Linux x86_64 and arm64, macOS x86_64 and +arm64, Windows x86 and x86_64). If building from source and the C tokenizer +cannot be built, you can fall back to the slower pure-Python implementation by +setting the environment variable ``WITH_EXTENSION=0`` when installing. + +To get the latest development version (with `uv`_):: git clone https://github.com/earwig/mwparserfromhell.git cd mwparserfromhell @@ -32,7 +38,14 @@ The comprehensive test suite can be run with ``pytest``. If using ``uv``, pass uv run --reinstall-package mwparserfromhell pytest +.. note:: + + To see if the fast C tokenizer is being used, check the value of + ``mwparserfromhell.parser.use_c``. If ``True``, it's being used; if + ``False``, the Python fallback is being used. + .. _PyPI: https://pypi.org/project/mwparserfromhell/ +.. _uv: https://docs.astral.sh/uv/ .. _pytest: https://docs.pytest.org/ Contents diff --git a/pyproject.toml b/pyproject.toml index 9d83472..565e720 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -38,6 +38,7 @@ dev = [ "psutil", "pytest", "pytest-cov", + "setuptools", "sphinx", ] @@ -45,36 +46,10 @@ dev = [ requires = ["setuptools >= 77.0.3", "setuptools-scm>=8"] build-backend = "setuptools.build_meta" -[[tool.setuptools.ext-modules]] -name = "mwparserfromhell.parser._tokenizer" -sources = [ - "src/mwparserfromhell/parser/ctokenizer/avl_tree.c", - "src/mwparserfromhell/parser/ctokenizer/definitions.c", - "src/mwparserfromhell/parser/ctokenizer/tag_data.c", - "src/mwparserfromhell/parser/ctokenizer/textbuffer.c", - "src/mwparserfromhell/parser/ctokenizer/tok_parse.c", - "src/mwparserfromhell/parser/ctokenizer/tok_support.c", - "src/mwparserfromhell/parser/ctokenizer/tokenizer.c", - "src/mwparserfromhell/parser/ctokenizer/tokens.c", -] -depends = [ - "src/mwparserfromhell/parser/ctokenizer/avl_tree.h", - "src/mwparserfromhell/parser/ctokenizer/common.h", - "src/mwparserfromhell/parser/ctokenizer/contexts.h", - "src/mwparserfromhell/parser/ctokenizer/definitions.h", - "src/mwparserfromhell/parser/ctokenizer/tag_data.h", - "src/mwparserfromhell/parser/ctokenizer/textbuffer.h", - "src/mwparserfromhell/parser/ctokenizer/tok_parse.h", - "src/mwparserfromhell/parser/ctokenizer/tok_support.h", - "src/mwparserfromhell/parser/ctokenizer/tokenizer.h", - "src/mwparserfromhell/parser/ctokenizer/tokens.h", -] -include-dirs = ["lib"] -optional = true - [tool.setuptools_scm] [tool.cibuildwheel] +environment = { WITH_EXTENSION = "1" } environment-pass = ["RUNNER_OS"] test-groups = ["dev"] test-command = [ diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..5cfc667 --- /dev/null +++ b/setup.py @@ -0,0 +1,100 @@ +#! /usr/bin/env python +# +# Copyright (C) 2012-2025 Ben Kurtovic +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +from __future__ import annotations + +import glob +import os +import sys +from enum import Enum + +from setuptools import Extension, setup +from setuptools.command.build_ext import build_ext + + +class UseExtension(Enum): + REQUIRED = 1 + OPTIONAL = 2 + IGNORED = 3 + + +if "WITH_EXTENSION" in os.environ: + if "WITHOUT_EXTENSION" in os.environ: + raise RuntimeError("Cannot set both $WITH_EXTENSION and $WITHOUT_EXTENSION") + value = os.environ["WITH_EXTENSION"].lower() + if value in ("1", "true", "yes", "required"): + use_extension = UseExtension.REQUIRED + elif value == "optional": + use_extension = UseExtension.OPTIONAL + elif value in ("0", "false", "no"): + use_extension = UseExtension.IGNORED + else: + raise RuntimeError( + f"Unknown value for $WITH_EXTENSION; should be '1', '0', or 'optional', " + f"but found {value!r}" + ) +elif "WITHOUT_EXTENSION" in os.environ: + value = os.environ["WITHOUT_EXTENSION"].lower() + if value in ("1", "true", "yes"): + use_extension = UseExtension.IGNORED + elif value in ("0", "false", "no"): + use_extension = UseExtension.REQUIRED + else: + raise RuntimeError( + f"Unknown value for $WITHOUT_EXTENSION; should be '1', or '0', " + f"but found {value!r}" + ) +else: + use_extension = UseExtension.REQUIRED + +if use_extension == UseExtension.IGNORED: + ext_modules = [] +else: + tokenizer = Extension( + "mwparserfromhell.parser._tokenizer", + sources=sorted(glob.glob("src/mwparserfromhell/parser/ctokenizer/*.c")), + depends=sorted(glob.glob("src/mwparserfromhell/parser/ctokenizer/*.h")), + optional=use_extension == UseExtension.OPTIONAL, + ) + ext_modules = [tokenizer] + + +def build_ext_patched(self): + try: + build_ext_original(self) + except Exception: + print( + """ +********** +Note: To avoid building the C tokenizer extension, set the environment variable \ +`WITH_EXTENSION=0`. +This will fall back to a pure-Python tokenizer. +********** +""", + file=sys.stderr, + ) + raise + + +build_ext.run, build_ext_original = build_ext_patched, build_ext.run + +setup(ext_modules=ext_modules) diff --git a/uv.lock b/uv.lock index 23e5bd0..b8d61b1 100644 --- a/uv.lock +++ b/uv.lock @@ -361,6 +361,7 @@ dev = [ { name = "psutil" }, { name = "pytest" }, { name = "pytest-cov" }, + { name = "setuptools" }, { name = "sphinx", version = "7.4.7", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, { name = "sphinx", version = "8.1.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.10.*'" }, { name = "sphinx", version = "8.2.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, @@ -373,6 +374,7 @@ dev = [ { name = "psutil" }, { name = "pytest" }, { name = "pytest-cov" }, + { name = "setuptools" }, { name = "sphinx" }, ] @@ -475,6 +477,15 @@ wheels = [ ] [[package]] +name = "setuptools" +version = "80.9.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/18/5d/3bf57dcd21979b887f014ea83c24ae194cfcd12b9e0fda66b957c69d1fca/setuptools-80.9.0.tar.gz", hash = "sha256:f36b47402ecde768dbfafc46e8e4207b4360c654f1f3bb84475f0a28628fb19c", size = 1319958, upload-time = "2025-05-27T00:56:51.443Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a3/dc/17031897dae0efacfea57dfd3a82fdd2a2aeb58e0ff71b77b87e44edc772/setuptools-80.9.0-py3-none-any.whl", hash = "sha256:062d34222ad13e0cc312a4c02d73f059e86a4acbfbdea8f8f76b28c99f306922", size = 1201486, upload-time = "2025-05-27T00:56:49.664Z" }, +] + +[[package]] name = "snowballstemmer" version = "3.0.1" source = { registry = "https://pypi.org/simple" }