Support $WITH_EXTENSION to control building the C tokenizer

CHANGELOG

@@ -1,6 +1,8 @@
 v0.7.1 (unreleased):
 
-- Modernized packaging.
+- Modernize packaging.
+- Explicitly support $WITH_EXTENSION/$WITHOUT_EXTENSION to control whether the
+  C tokenizer should be built.
 
 v0.7.0 (released June 28, 2025):
 

README.rst

@@ -18,7 +18,13 @@ Installation
 The easiest way to install the parser is from `PyPI`_; you can install the
 latest release with ``pip install mwparserfromhell``.
 
-Alternatively, get the latest development version:
+Prebuilt wheels are available on PyPI with a fast, compiled C tokenizer
+extension for most environments (Linux x86_64 and arm64, macOS x86_64 and
+arm64, Windows x86 and x86_64). If building from source and the C tokenizer
+cannot be built, you can fall back to the slower pure-Python implementation by
+setting the environment variable ``WITH_EXTENSION=0`` when installing.
+
+To get the latest development version (with `uv`_):
 
 .. code-block:: sh
 
@@ -223,6 +229,7 @@ Python 3 code (using the API_ and the requests_ library):
 .. _Legoktm:                https://en.wikipedia.org/wiki/User:Legoktm
 .. _GitHub:                 https://github.com/earwig/mwparserfromhell
 .. _PyPI:                   https://pypi.org/project/mwparserfromhell/
+.. _uv:                     https://docs.astral.sh/uv/
 .. _pytest:                 https://docs.pytest.org/
 .. _Word-ending links:      https://www.mediawiki.org/wiki/Help:Links#linktrail
 .. _EarwigBot:              https://github.com/earwig/earwigbot

docs/changelog.rst

@@ -7,7 +7,9 @@ v0.7.1
 Unreleased
 (`changes <https://github.com/earwig/mwparserfromhell/compare/v0.7.0...main>`__):
 
-- Modernized packaging.
+- Modernize packaging.
+- Explicitly support ``$WITH_EXTENSION`` / ``$WITHOUT_EXTENSION`` to control
+  whether the C tokenizer should be built.
 
 v0.7.0
 ------

docs/index.rst

@@ -20,7 +20,13 @@ Installation
 The easiest way to install the parser is from `PyPI`_; you can install the
 latest release with ``pip install mwparserfromhell``.
 
-Alternatively, get the latest development version::
+Prebuilt wheels are available on PyPI with a fast, compiled C tokenizer
+extension for most environments (Linux x86_64 and arm64, macOS x86_64 and
+arm64, Windows x86 and x86_64). If building from source and the C tokenizer
+cannot be built, you can fall back to the slower pure-Python implementation by
+setting the environment variable ``WITH_EXTENSION=0`` when installing.
+
+To get the latest development version (with `uv`_)::
 
     git clone https://github.com/earwig/mwparserfromhell.git
     cd mwparserfromhell
@@ -32,7 +38,14 @@ The comprehensive test suite can be run with ``pytest``. If using ``uv``, pass
 
     uv run --reinstall-package mwparserfromhell pytest
 
+.. note::
+
+    To see if the fast C tokenizer is being used, check the value of
+    ``mwparserfromhell.parser.use_c``. If ``True``, it's being used; if
+    ``False``, the Python fallback is being used.
+
 .. _PyPI:                   https://pypi.org/project/mwparserfromhell/
+.. _uv:                     https://docs.astral.sh/uv/
 .. _pytest:                 https://docs.pytest.org/
 
 Contents

pyproject.toml

@@ -38,6 +38,7 @@ dev = [
     "psutil",
     "pytest",
     "pytest-cov",
+    "setuptools",
     "sphinx",
 ]
 
@@ -45,36 +46,10 @@ dev = [
 requires = ["setuptools >= 77.0.3", "setuptools-scm>=8"]
 build-backend = "setuptools.build_meta"
 
-[[tool.setuptools.ext-modules]]
-name = "mwparserfromhell.parser._tokenizer"
-sources = [
-    "src/mwparserfromhell/parser/ctokenizer/avl_tree.c",
-    "src/mwparserfromhell/parser/ctokenizer/definitions.c",
-    "src/mwparserfromhell/parser/ctokenizer/tag_data.c",
-    "src/mwparserfromhell/parser/ctokenizer/textbuffer.c",
-    "src/mwparserfromhell/parser/ctokenizer/tok_parse.c",
-    "src/mwparserfromhell/parser/ctokenizer/tok_support.c",
-    "src/mwparserfromhell/parser/ctokenizer/tokenizer.c",
-    "src/mwparserfromhell/parser/ctokenizer/tokens.c",
-]
-depends = [
-    "src/mwparserfromhell/parser/ctokenizer/avl_tree.h",
-    "src/mwparserfromhell/parser/ctokenizer/common.h",
-    "src/mwparserfromhell/parser/ctokenizer/contexts.h",
-    "src/mwparserfromhell/parser/ctokenizer/definitions.h",
-    "src/mwparserfromhell/parser/ctokenizer/tag_data.h",
-    "src/mwparserfromhell/parser/ctokenizer/textbuffer.h",
-    "src/mwparserfromhell/parser/ctokenizer/tok_parse.h",
-    "src/mwparserfromhell/parser/ctokenizer/tok_support.h",
-    "src/mwparserfromhell/parser/ctokenizer/tokenizer.h",
-    "src/mwparserfromhell/parser/ctokenizer/tokens.h",
-]
-include-dirs = ["lib"]
-optional = true
-
 [tool.setuptools_scm]
 
 [tool.cibuildwheel]
+environment = { WITH_EXTENSION = "1" }
 environment-pass = ["RUNNER_OS"]
 test-groups = ["dev"]
 test-command = [

setup.py

@@ -0,0 +1,100 @@
+#! /usr/bin/env python
+#
+# Copyright (C) 2012-2025 Ben Kurtovic <ben.kurtovic@gmail.com>
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+from __future__ import annotations
+
+import glob
+import os
+import sys
+from enum import Enum
+
+from setuptools import Extension, setup
+from setuptools.command.build_ext import build_ext
+
+
+class UseExtension(Enum):
+    REQUIRED = 1
+    OPTIONAL = 2
+    IGNORED = 3
+
+
+if "WITH_EXTENSION" in os.environ:
+    if "WITHOUT_EXTENSION" in os.environ:
+        raise RuntimeError("Cannot set both $WITH_EXTENSION and $WITHOUT_EXTENSION")
+    value = os.environ["WITH_EXTENSION"].lower()
+    if value in ("1", "true", "yes", "required"):
+        use_extension = UseExtension.REQUIRED
+    elif value == "optional":
+        use_extension = UseExtension.OPTIONAL
+    elif value in ("0", "false", "no"):
+        use_extension = UseExtension.IGNORED
+    else:
+        raise RuntimeError(
+            f"Unknown value for $WITH_EXTENSION; should be '1', '0', or 'optional', "
+            f"but found {value!r}"
+        )
+elif "WITHOUT_EXTENSION" in os.environ:
+    value = os.environ["WITHOUT_EXTENSION"].lower()
+    if value in ("1", "true", "yes"):
+        use_extension = UseExtension.IGNORED
+    elif value in ("0", "false", "no"):
+        use_extension = UseExtension.REQUIRED
+    else:
+        raise RuntimeError(
+            f"Unknown value for $WITHOUT_EXTENSION; should be '1', or '0', "
+            f"but found {value!r}"
+        )
+else:
+    use_extension = UseExtension.REQUIRED
+
+if use_extension == UseExtension.IGNORED:
+    ext_modules = []
+else:
+    tokenizer = Extension(
+        "mwparserfromhell.parser._tokenizer",
+        sources=sorted(glob.glob("src/mwparserfromhell/parser/ctokenizer/*.c")),
+        depends=sorted(glob.glob("src/mwparserfromhell/parser/ctokenizer/*.h")),
+        optional=use_extension == UseExtension.OPTIONAL,
+    )
+    ext_modules = [tokenizer]
+
+
+def build_ext_patched(self):
+    try:
+        build_ext_original(self)
+    except Exception:
+        print(
+            """
+**********
+Note: To avoid building the C tokenizer extension, set the environment variable \
+`WITH_EXTENSION=0`.
+This will fall back to a pure-Python tokenizer.
+**********
+""",
+            file=sys.stderr,
+        )
+        raise
+
+
+build_ext.run, build_ext_original = build_ext_patched, build_ext.run
+
+setup(ext_modules=ext_modules)

uv.lock

@@ -361,6 +361,7 @@ dev = [
     { name = "psutil" },
     { name = "pytest" },
     { name = "pytest-cov" },
+    { name = "setuptools" },
     { name = "sphinx", version = "7.4.7", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" },
     { name = "sphinx", version = "8.1.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.10.*'" },
     { name = "sphinx", version = "8.2.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" },
@@ -373,6 +374,7 @@ dev = [
     { name = "psutil" },
     { name = "pytest" },
     { name = "pytest-cov" },
+    { name = "setuptools" },
     { name = "sphinx" },
 ]
 
@@ -475,6 +477,15 @@ wheels = [
 ]
 
 [[package]]
+name = "setuptools"
+version = "80.9.0"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/18/5d/3bf57dcd21979b887f014ea83c24ae194cfcd12b9e0fda66b957c69d1fca/setuptools-80.9.0.tar.gz", hash = "sha256:f36b47402ecde768dbfafc46e8e4207b4360c654f1f3bb84475f0a28628fb19c", size = 1319958, upload-time = "2025-05-27T00:56:51.443Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/a3/dc/17031897dae0efacfea57dfd3a82fdd2a2aeb58e0ff71b77b87e44edc772/setuptools-80.9.0-py3-none-any.whl", hash = "sha256:062d34222ad13e0cc312a4c02d73f059e86a4acbfbdea8f8f76b28c99f306922", size = 1201486, upload-time = "2025-05-27T00:56:49.664Z" },
+]
+
+[[package]]
 name = "snowballstemmer"
 version = "3.0.1"
 source = { registry = "https://pypi.org/simple" }