Fix pyright issues in src/mwparserfromhell

scripts/memtest.py

@@ -131,16 +131,16 @@ class MemoryTest:
             parent, child = Pipe()
             p = Process(target=_runner, args=(text, child))
             p.start()
+            proc = psutil.Process(p.pid)
             try:
-                proc = psutil.Process(p.pid)
                 parent.recv()
                 parent.send("OK")
                 parent.recv()
-                info1 = proc.get_memory_info()
+                info1 = proc.memory_info
                 sys.stdout.flush()
                 parent.send("OK")
                 parent.recv()
-                info2 = proc.get_memory_info()
+                info2 = proc.memory_info
                 self._print_results(info1, info2)
                 sys.stdout.flush()
                 parent.send("OK")

src/mwparserfromhell/nodes/comment.py

@@ -20,17 +20,20 @@
 
 from __future__ import annotations
 
-from typing import Any
+from typing import TYPE_CHECKING, Any
 
 from ._base import Node
 
+if TYPE_CHECKING:
+    from ..wikicode import Wikicode
+
 __all__ = ["Comment"]
 
 
 class Comment(Node):
     """Represents a hidden HTML comment, like ``<!-- foobar -->``."""
 
-    def __init__(self, contents: str):
+    def __init__(self, contents: Wikicode | str):
         super().__init__()
         self.contents = contents
 

src/mwparserfromhell/nodes/extras/attribute.py

@@ -72,7 +72,7 @@ class Attribute(StringMixIn):
         return result
 
     @staticmethod
-    def _value_needs_quotes(value: Wikicode) -> str | None:
+    def _value_needs_quotes(value: Wikicode | None) -> str | None:
         """Return valid quotes for the given value, or None if unneeded."""
         if not value:
             return None

src/mwparserfromhell/nodes/template.py

@@ -135,6 +135,7 @@ class Template(Node):
             before, after = "", sval
         else:
             match = re.search(r"^(\s*).*?(\s*)$", sval, FLAGS)
+            assert match, sval
             before, after = match.group(1), match.group(2)
         value.nodes = [Text(before), Text(after)]
 
@@ -157,6 +158,7 @@ class Template(Node):
             else:
                 component = str(param.value)
             match = re.search(r"^(\s*).*?(\s*)$", component, FLAGS)
+            assert match, component
             before, after = match.group(1), match.group(2)
             if not use_names and component.isspace() and "\n" in before:
                 # If the value is empty, we expect newlines in the whitespace
@@ -314,7 +316,7 @@ class Template(Node):
                 existing.showkey = showkey
             if not existing.showkey:
                 self._surface_escape(value, "=")
-            nodes = existing.value.nodes
+            nodes: list[Node | None] = list(existing.value.nodes)
             if preserve_spacing and existing.showkey:
                 for i in range(2):  # Ignore empty text nodes
                     if not nodes[i]:

src/mwparserfromhell/parser/builder.py

@@ -110,6 +110,7 @@ class Builder:
     @_add_handler(tokens.TemplateOpen)
     def _handle_template(self, token):
         """Handle a case where a template is at the head of the tokens."""
+        name = None
         params = []
         default = 1
         self._push()
@@ -125,6 +126,7 @@ class Builder:
             elif isinstance(token, tokens.TemplateClose):
                 if not params:
                     name = self._pop()
+                assert name is not None
                 return Template(name, params)
             else:
                 self._write(self._handle_token(token))
@@ -272,6 +274,8 @@ class Builder:
     def _handle_tag(self, token):
         """Handle a case where a tag is at the head of the tokens."""
         close_tokens = (tokens.TagCloseSelfclose, tokens.TagCloseClose)
+        tag = None
+        padding = None
         implicit, attrs, contents, closing_tag = False, [], None, None
         wiki_markup, invalid = token.wiki_markup, token.invalid or False
         wiki_style_separator, closing_wiki_markup = None, wiki_markup
@@ -299,6 +303,8 @@ class Builder:
                 else:
                     self_closing = False
                     closing_tag = self._pop()
+                assert tag is not None
+                assert padding is not None
                 return Tag(
                     tag,
                     contents,

src/mwparserfromhell/parser/tokenizer.py

@@ -20,9 +20,11 @@
 
 from __future__ import annotations
 
-import html.entities as htmlentities
+import html.entities
+import math
 import re
-from math import log
+from enum import Enum
+from typing import Literal, cast, overload
 
 from ..definitions import (
     get_html_tag,
@@ -64,13 +66,20 @@ class _TagOpenData:
         self.reset = 0
 
 
+class Sentinel(Enum):
+    START = 0
+    END = 1
+
+
+START = Sentinel.START
+END = Sentinel.END
+
+
 class Tokenizer:
     """Creates a list of tokens from a string of wikicode."""
 
     USES_C = False
-    START = object()
-    END = object()
-    MARKERS = [
+    MARKERS: list[str | Sentinel] = [
         "{",
         "}",
         "[",
@@ -99,8 +108,8 @@ class Tokenizer:
     tag_splitter = re.compile(r"([\s\"\'\\]+)")
 
     def __init__(self):
-        self._text = None
-        self._head = 0
+        self._text: list[str] = []
+        self._head: int = 0
         self._stacks = []
         self._global = 0
         self._depth = 0
@@ -222,27 +231,44 @@ class Tokenizer:
             self._emit_all(stack)
         self._head -= 1
 
-    def _read(self, delta=0, wrap=False, strict=False):
+    @overload
+    def _read(
+        self, *, strict: Literal[False] = False
+    ) -> str | Literal[Sentinel.END]: ...
+
+    @overload
+    def _read(self, *, strict: Literal[True]) -> str: ...
+
+    @overload
+    def _read(
+        self, delta: int = 0, *, strict: Literal[False] = False
+    ) -> str | Literal[Sentinel.START, Sentinel.END]: ...
+
+    @overload
+    def _read(
+        self, delta: int = 0, *, strict: Literal[True]
+    ) -> str | Literal[Sentinel.START]: ...
+
+    def _read(
+        self, delta: int = 0, *, strict: bool = False
+    ) -> str | Literal[Sentinel.START, Sentinel.END]:
         """Read the value at a relative point in the wikicode.
 
         The value is read from :attr:`self._head <_head>` plus the value of
-        *delta* (which can be negative). If *wrap* is ``False``, we will not
-        allow attempts to read from the end of the string if ``self._head +
-        delta`` is negative. If *strict* is ``True``, the route will be failed
-        (with :meth:`_fail_route`) if we try to read from past the end of the
-        string; otherwise, :attr:`self.END <END>` is returned. If we try to
-        read from before the start of the string, :attr:`self.START <START>` is
-        returned.
+        *delta* (which can be negative). If *strict* is ``True``, the route
+        will be failed (with :meth:`_fail_route`) if we try to read from past
+        the end of the string; otherwise, ``END`` is returned. If we try to
+        read from before the start of the string, ``START`` is returned.
         """
         index = self._head + delta
-        if index < 0 and (not wrap or abs(index) > len(self._text)):
-            return self.START
+        if index < 0:
+            return START
         try:
             return self._text[index]
         except IndexError:
             if strict:
                 self._fail_route()
-            return self.END
+            return END
 
     def _parse_template(self, has_content):
         """Parse a template at the head of the wikicode string."""
@@ -399,12 +425,12 @@ class Tokenizer:
             self._head += 2
         else:
 
-            def all_valid():
-                return all(char in self.URISCHEME for char in self._read())
+            def all_valid(this: str):
+                return all(char in self.URISCHEME for char in this)
 
             scheme = ""
-            while self._read() is not self.END and all_valid():
-                scheme += self._read()
+            while (this := self._read()) is not END and all_valid(this):
+                scheme += this
                 self._emit_text(self._read())
                 self._head += 1
             if self._read() != ":":
@@ -472,7 +498,7 @@ class Tokenizer:
         # Built from _parse()'s end sentinels:
         after, ctx = self._read(2), self._context
         return (
-            this in (self.END, "\n", "[", "]", "<", ">", '"')
+            this in (END, "\n", "[", "]", "<", ">", '"')
             or " " in this
             or this == nxt == "'"
             or (this == "|" and ctx & contexts.TEMPLATE)
@@ -491,7 +517,7 @@ class Tokenizer:
             self._parse_free_uri_scheme()
             invalid = ("\n", " ", "[", "]")
             punct = tuple(",;\\.:!?)")
-        if self._read() is self.END or self._read()[0] in invalid:
+        if (this := self._read()) is END or this[0] in invalid:
             self._fail_route()
         tail = ""
         while True:
@@ -512,7 +538,7 @@ class Tokenizer:
                     tail = ""
                 self._parse_template_or_argument()
             elif brackets:
-                if this is self.END or this == "\n":
+                if this is END or this == "\n":
                     self._fail_route()
                 if this == "]":
                     return self._pop(), None
@@ -534,7 +560,7 @@ class Tokenizer:
                 self._emit_text(this)
             else:
                 if self._is_uri_end(this, nxt):
-                    if this is not self.END and " " in this:
+                    if this is not END and " " in this:
                         before, after = this.split(" ", 1)
                         punct, tail = self._handle_free_link_text(punct, tail, before)
                         tail += " " + after
@@ -616,7 +642,7 @@ class Tokenizer:
         while self._read() == "=":
             best += 1
             self._head += 1
-        current = int(log(self._context / contexts.HEADING_LEVEL_1, 2)) + 1
+        current = int(math.log(self._context / contexts.HEADING_LEVEL_1, 2)) + 1
         level = min(current, min(best, 6))
 
         try:  # Try to check for a heading closure after this one
@@ -667,7 +693,7 @@ class Tokenizer:
             if test < 1 or test > 0x10FFFF:
                 self._fail_route()
         else:
-            if this not in htmlentities.entitydefs:
+            if this not in html.entities.entitydefs:
                 self._fail_route()
 
         self._emit(tokens.Text(text=this))
@@ -692,7 +718,7 @@ class Tokenizer:
         self._push()
         while True:
             this = self._read()
-            if this == self.END:
+            if this == END:
                 self._pop()
                 self._head = reset
                 self._emit_text("<!--")
@@ -854,7 +880,7 @@ class Tokenizer:
 
         while True:
             this, nxt = self._read(), self._read(1)
-            if this is self.END:
+            if this is END:
                 self._fail_route()
             elif this == "<" and nxt == "/":
                 self._head += 3
@@ -917,7 +943,7 @@ class Tokenizer:
                 not data.context & (data.CX_QUOTED | data.CX_NAME)
                 or data.context & data.CX_NOTE_SPACE
             )
-            if this is self.END:
+            if this is END:
                 if self._context & contexts.TAG_ATTR:
                     if data.context & data.CX_QUOTED:
                         # Unclosed attribute quote: reset, don't die
@@ -948,7 +974,8 @@ class Tokenizer:
         reset = self._head + 1
         self._head += 2
         try:
-            if not is_single_only(self.tag_splitter.split(self._read())[0]):
+            assert (this := self._read()) is not END
+            if not is_single_only(self.tag_splitter.split(this)[0]):
                 raise BadRoute()
             tag = self._really_parse_tag()
         except BadRoute:
@@ -1100,6 +1127,7 @@ class Tokenizer:
     def _handle_list_marker(self):
         """Handle a list marker at the head (``#``, ``*``, ``;``, ``:``)."""
         markup = self._read()
+        assert markup is not END
         if markup == ";":
             self._context |= contexts.DL_TERM
         self._emit(tokens.TagOpenOpen(wiki_markup=markup))
@@ -1159,7 +1187,7 @@ class Tokenizer:
         self._emit_text(tag)
         self._emit(tokens.TagCloseClose())
 
-    def _handle_table_style(self, end_token):
+    def _handle_table_style(self, end_token: str):
         """Handle style attributes for a table until ``end_token``."""
         data = _TagOpenData()
         data.context = _TagOpenData.CX_ATTR_READY
@@ -1169,12 +1197,13 @@ class Tokenizer:
                 not data.context & data.CX_QUOTED or data.context & data.CX_NOTE_SPACE
             )
             if this == end_token and can_exit:
+                assert this is not END
                 if data.context & (data.CX_ATTR_NAME | data.CX_ATTR_VALUE):
                     self._push_tag_buffer(data)
                 if this.isspace():
                     data.padding_buffer["first"] += this
                 return data.padding_buffer["first"]
-            if this is self.END or this == end_token:
+            if this is END or this == end_token:
                 if self._context & contexts.TAG_ATTR:
                     if data.context & data.CX_QUOTED:
                         # Unclosed attribute quote: reset, don't die
@@ -1343,11 +1372,11 @@ class Tokenizer:
                 return True
             if context & contexts.HAS_TEXT:
                 if context & contexts.FAIL_ON_TEXT:
-                    if this is self.END or not this.isspace():
+                    if this is END or not this.isspace():
                         return False
                 elif this == "\n":
                     self._context |= contexts.FAIL_ON_TEXT
-            elif this is self.END or not this.isspace():
+            elif this is END or not this.isspace():
                 self._context |= contexts.HAS_TEXT
             return True
         if context & contexts.TAG_CLOSE:
@@ -1389,7 +1418,7 @@ class Tokenizer:
                 self._emit_text(this)
                 self._head += 1
                 continue
-            if this is self.END:
+            if this is END:
                 return self._handle_end()
             nxt = self._read(1)
             if this == nxt == "{":
@@ -1402,7 +1431,7 @@ class Tokenizer:
             elif this == "=" and self._context & contexts.TEMPLATE_PARAM_KEY:
                 if (
                     not self._global & contexts.GL_HEADING
-                    and self._read(-1) in ("\n", self.START)
+                    and self._read(-1) in ("\n", START)
                     and nxt == "="
                 ):
                     self._parse_heading()
@@ -1439,7 +1468,7 @@ class Tokenizer:
                 and not self._global & contexts.GL_HEADING
                 and not self._context & contexts.TEMPLATE
             ):
-                if self._read(-1) in ("\n", self.START):
+                if self._read(-1) in ("\n", START):
                     self._parse_heading()
                 else:
                     self._emit_text("=")
@@ -1454,7 +1483,7 @@ class Tokenizer:
                     self._parse_comment()
                 else:
                     self._emit_text(this)
-            elif this == "<" and nxt == "/" and self._read(2) is not self.END:
+            elif this == "<" and nxt == "/" and self._read(2) is not END:
                 if self._context & contexts.TAG_BODY:
                     self._handle_tag_open_close()
                 else:
@@ -1470,9 +1499,9 @@ class Tokenizer:
                 result = self._parse_style()
                 if result is not None:
                     return result
-            elif self._read(-1) in ("\n", self.START) and this in ("#", "*", ";", ":"):
+            elif self._read(-1) in ("\n", START) and this in ("#", "*", ";", ":"):
                 self._handle_list()
-            elif self._read(-1) in ("\n", self.START) and (
+            elif self._read(-1) in ("\n", START) and (
                 this == nxt == self._read(2) == self._read(3) == "-"
             ):
                 self._handle_hr()
@@ -1486,10 +1515,10 @@ class Tokenizer:
                 this == "{"
                 and nxt == "|"
                 and (
-                    self._read(-1) in ("\n", self.START)
+                    self._read(-1) in ("\n", START)
                     or (
-                        self._read(-2) in ("\n", self.START)
-                        and self._read(-1).isspace()
+                        self._read(-2) in ("\n", START)
+                        and cast(str, self._read(-1)).isspace()
                     )
                 )
             ):
@@ -1516,8 +1545,9 @@ class Tokenizer:
                 elif this == "\n" and self._context & contexts.TABLE_CELL_LINE_CONTEXTS:
                     self._context &= ~contexts.TABLE_CELL_LINE_CONTEXTS
                     self._emit_text(this)
-                elif self._read(-1) in ("\n", self.START) or (
-                    self._read(-2) in ("\n", self.START) and self._read(-1).isspace()
+                elif self._read(-1) in ("\n", START) or (
+                    self._read(-2) in ("\n", START)
+                    and cast(str, self._read(-1)).isspace()
                 ):
                     if this == "|" and nxt == "}":
                         if self._context & contexts.TABLE_CELL_OPEN:
@@ -1548,7 +1578,7 @@ class Tokenizer:
                 self._emit_text(this)
             self._head += 1
 
-    def tokenize(self, text, context=0, skip_style_tags=False):
+    def tokenize(self, text: str, context=0, skip_style_tags=False):
         """Build a list of tokens from a string of wikicode and return it."""
         split = self.regex.split(text)
         self._text = [segment for segment in split if segment]

src/mwparserfromhell/smart_list/list_proxy.py

@@ -22,6 +22,7 @@
 from __future__ import annotations
 
 import weakref
+from typing import Any
 
 from .utils import _SliceNormalizerMixIn, inheritdoc
 
@@ -41,7 +42,7 @@ class ListProxy(_SliceNormalizerMixIn, list):
         self._parent = parent
         self._sliceinfo = sliceinfo
 
-    def __reduce_ex__(self, protocol: int) -> tuple:
+    def __reduce_ex__(self, protocol: Any) -> tuple:
         return (ListProxy, (self._parent, self._sliceinfo), ())
 
     def __setstate__(self, state: tuple) -> None:
@@ -232,7 +233,7 @@ class ListProxy(_SliceNormalizerMixIn, list):
         self._parent[self._start : self._stop : self._step] = item
 
     @inheritdoc
-    def sort(self, key=None, reverse=None):
+    def sort(self, *, key=None, reverse=None):
         item = self._render()
         kwargs = {}
         if key is not None:

src/mwparserfromhell/smart_list/smart_list.py

@@ -21,13 +21,14 @@
 
 from __future__ import annotations
 
+from typing import Any
 from weakref import ref
 
 from .list_proxy import ListProxy
 from .utils import _SliceNormalizerMixIn, inheritdoc
 
 
-class SmartList(_SliceNormalizerMixIn, list):
+class SmartList(list, _SliceNormalizerMixIn):
     """Implements the ``list`` interface with special handling of sublists.
 
     When a sublist is created (by ``list[i:j]``), any changes made to this
@@ -58,7 +59,7 @@ class SmartList(_SliceNormalizerMixIn, list):
         obj._children = {}
         return obj
 
-    def __reduce_ex__(self, protocol: int) -> tuple:
+    def __reduce_ex__(self, protocol: Any) -> tuple:
         # Detach children when pickling
         return (SmartList, (), None, iter(self))
 
@@ -154,7 +155,7 @@ class SmartList(_SliceNormalizerMixIn, list):
         super().reverse()
 
     @inheritdoc
-    def sort(self, key=None, reverse=None):
+    def sort(self, *, key=None, reverse=None):
         self._detach_children()
         kwargs = {}
         if key is not None:

src/mwparserfromhell/smart_list/utils.py

@@ -21,7 +21,9 @@
 
 from __future__ import annotations
 
-from sys import maxsize
+import sys
+from collections.abc import Sized
+from typing import Literal, overload
 
 __all__ = []
 
@@ -36,18 +38,32 @@ def inheritdoc(method):
     return method
 
 
-class _SliceNormalizerMixIn:
+class _SliceNormalizerMixIn(Sized):
     """MixIn that provides a private method to normalize slices."""
 
     __slots__ = ()
 
-    def _normalize_slice(self, key, clamp=False):
+    @overload
+    def _normalize_slice(
+        self,
+        key: slice[int | None, int | None, int | None],
+        clamp: Literal[False] = False,
+    ) -> slice[int, int | None, int]: ...
+
+    @overload
+    def _normalize_slice(
+        self, key: slice[int | None, int | None, int | None], clamp: Literal[True]
+    ) -> slice[int, int, int]: ...
+
+    def _normalize_slice(
+        self, key: slice[int | None, int | None, int | None], clamp: bool = False
+    ) -> slice[int, int | None, int]:
         """Return a slice equivalent to the input *key*, standardized."""
         if key.start is None:
             start = 0
         else:
             start = (len(self) + key.start) if key.start < 0 else key.start
-        if key.stop is None or key.stop == maxsize:
+        if key.stop is None or key.stop == sys.maxsize:
             stop = len(self) if clamp else None
         else:
             stop = (len(self) + key.stop) if key.stop < 0 else key.stop

src/mwparserfromhell/wikicode.py

@@ -23,7 +23,7 @@ from __future__ import annotations
 import re
 from collections.abc import Generator, Iterable
 from itertools import chain
-from typing import Any, Callable, cast, overload
+from typing import Any, Callable, TypeVar, cast, overload
 
 from .nodes import (
     Argument,
@@ -45,6 +45,8 @@ __all__ = ["Wikicode"]
 
 FLAGS = re.IGNORECASE | re.DOTALL | re.UNICODE
 
+N = TypeVar("N", bound=Node)
+
 
 class Wikicode(StringMixIn):
     """A ``Wikicode`` is a container for nodes that operates like a string.
@@ -91,11 +93,9 @@ class Wikicode(StringMixIn):
 
     @staticmethod
     def _build_matcher(
-        matches: (
-            Callable[[Node], bool | re.Match[str] | None] | re.Pattern | str | None
-        ),
+        matches: Callable[[N], bool | re.Match[str] | None] | re.Pattern | str | None,
         flags: int,
-    ) -> Callable[[Node], bool | re.Match[str] | None]:
+    ) -> Callable[[N], bool | re.Match[str] | None]:
         """Helper for :meth:`_indexed_ifilter` and others.
 
         If *matches* is a function, return it. If it's a regex, return a
@@ -105,16 +105,18 @@ class Wikicode(StringMixIn):
         if matches:
             if callable(matches):
                 return matches
-            return lambda obj: re.search(matches, str(obj), flags)
-        return lambda obj: True
+            else:
+                return lambda obj: re.search(matches, str(obj), flags)
+        else:
+            return lambda obj: True
 
     def _indexed_ifilter(
         self,
         recursive: bool = True,
-        matches: Callable[[Node], bool] | re.Pattern | str | None = None,
+        matches: Callable[[N], bool] | re.Pattern | str | None = None,
         flags: int = FLAGS,
-        forcetype: type | None = None,
-    ) -> Generator[tuple[int, Node]]:
+        forcetype: type[N] | None = None,
+    ) -> Generator[tuple[int, N]]:
         """Iterate over nodes and their corresponding indices in the node list.
 
         The arguments are interpreted as for :meth:`ifilter`. For each tuple
@@ -135,8 +137,10 @@ class Wikicode(StringMixIn):
         else:
             inodes = enumerate(self.nodes)
         for i, node in inodes:
-            if (not forcetype or isinstance(node, forcetype)) and match(node):
-                yield (i, node)
+            if (forcetype is None or isinstance(node, forcetype)) and match(
+                cast(N, node)
+            ):
+                yield (i, cast(N, node))
 
     def _is_child_wikicode(self, obj: Wikicode, recursive: bool = True) -> bool:
         """Return whether the given :class:`.Wikicode` is a descendant."""
@@ -264,8 +268,8 @@ class Wikicode(StringMixIn):
             else:
                 lines.append(" " * 6 * indent + " ".join(args))
 
-        def get(code):
-            return self._get_tree(code, lines, marker, indent + 1)
+        def get(code: Wikicode):
+            self._get_tree(code, lines, marker, indent + 1)
 
         def mark():
             return lines.append(marker)