diff --git a/scripts/memtest.py b/scripts/memtest.py index c3bb60f..15d4a1a 100644 --- a/scripts/memtest.py +++ b/scripts/memtest.py @@ -131,16 +131,16 @@ class MemoryTest: parent, child = Pipe() p = Process(target=_runner, args=(text, child)) p.start() + proc = psutil.Process(p.pid) try: - proc = psutil.Process(p.pid) parent.recv() parent.send("OK") parent.recv() - info1 = proc.get_memory_info() + info1 = proc.memory_info sys.stdout.flush() parent.send("OK") parent.recv() - info2 = proc.get_memory_info() + info2 = proc.memory_info self._print_results(info1, info2) sys.stdout.flush() parent.send("OK") diff --git a/src/mwparserfromhell/nodes/comment.py b/src/mwparserfromhell/nodes/comment.py index 5afdebc..f6b80f8 100644 --- a/src/mwparserfromhell/nodes/comment.py +++ b/src/mwparserfromhell/nodes/comment.py @@ -20,17 +20,20 @@ from __future__ import annotations -from typing import Any +from typing import TYPE_CHECKING, Any from ._base import Node +if TYPE_CHECKING: + from ..wikicode import Wikicode + __all__ = ["Comment"] class Comment(Node): """Represents a hidden HTML comment, like ````.""" - def __init__(self, contents: str): + def __init__(self, contents: Wikicode | str): super().__init__() self.contents = contents diff --git a/src/mwparserfromhell/nodes/extras/attribute.py b/src/mwparserfromhell/nodes/extras/attribute.py index 91514da..0f284e7 100644 --- a/src/mwparserfromhell/nodes/extras/attribute.py +++ b/src/mwparserfromhell/nodes/extras/attribute.py @@ -72,7 +72,7 @@ class Attribute(StringMixIn): return result @staticmethod - def _value_needs_quotes(value: Wikicode) -> str | None: + def _value_needs_quotes(value: Wikicode | None) -> str | None: """Return valid quotes for the given value, or None if unneeded.""" if not value: return None diff --git a/src/mwparserfromhell/nodes/template.py b/src/mwparserfromhell/nodes/template.py index 4a4ea88..2fd35cd 100644 --- a/src/mwparserfromhell/nodes/template.py +++ b/src/mwparserfromhell/nodes/template.py @@ -135,6 +135,7 @@ class Template(Node): before, after = "", sval else: match = re.search(r"^(\s*).*?(\s*)$", sval, FLAGS) + assert match, sval before, after = match.group(1), match.group(2) value.nodes = [Text(before), Text(after)] @@ -157,6 +158,7 @@ class Template(Node): else: component = str(param.value) match = re.search(r"^(\s*).*?(\s*)$", component, FLAGS) + assert match, component before, after = match.group(1), match.group(2) if not use_names and component.isspace() and "\n" in before: # If the value is empty, we expect newlines in the whitespace @@ -314,7 +316,7 @@ class Template(Node): existing.showkey = showkey if not existing.showkey: self._surface_escape(value, "=") - nodes = existing.value.nodes + nodes: list[Node | None] = list(existing.value.nodes) if preserve_spacing and existing.showkey: for i in range(2): # Ignore empty text nodes if not nodes[i]: diff --git a/src/mwparserfromhell/parser/builder.py b/src/mwparserfromhell/parser/builder.py index e660118..ab2cb9a 100644 --- a/src/mwparserfromhell/parser/builder.py +++ b/src/mwparserfromhell/parser/builder.py @@ -110,6 +110,7 @@ class Builder: @_add_handler(tokens.TemplateOpen) def _handle_template(self, token): """Handle a case where a template is at the head of the tokens.""" + name = None params = [] default = 1 self._push() @@ -125,6 +126,7 @@ class Builder: elif isinstance(token, tokens.TemplateClose): if not params: name = self._pop() + assert name is not None return Template(name, params) else: self._write(self._handle_token(token)) @@ -272,6 +274,8 @@ class Builder: def _handle_tag(self, token): """Handle a case where a tag is at the head of the tokens.""" close_tokens = (tokens.TagCloseSelfclose, tokens.TagCloseClose) + tag = None + padding = None implicit, attrs, contents, closing_tag = False, [], None, None wiki_markup, invalid = token.wiki_markup, token.invalid or False wiki_style_separator, closing_wiki_markup = None, wiki_markup @@ -299,6 +303,8 @@ class Builder: else: self_closing = False closing_tag = self._pop() + assert tag is not None + assert padding is not None return Tag( tag, contents, diff --git a/src/mwparserfromhell/parser/tokenizer.py b/src/mwparserfromhell/parser/tokenizer.py index a1da548..9231080 100644 --- a/src/mwparserfromhell/parser/tokenizer.py +++ b/src/mwparserfromhell/parser/tokenizer.py @@ -20,9 +20,11 @@ from __future__ import annotations -import html.entities as htmlentities +import html.entities +import math import re -from math import log +from enum import Enum +from typing import Literal, cast, overload from ..definitions import ( get_html_tag, @@ -64,13 +66,20 @@ class _TagOpenData: self.reset = 0 +class Sentinel(Enum): + START = 0 + END = 1 + + +START = Sentinel.START +END = Sentinel.END + + class Tokenizer: """Creates a list of tokens from a string of wikicode.""" USES_C = False - START = object() - END = object() - MARKERS = [ + MARKERS: list[str | Sentinel] = [ "{", "}", "[", @@ -99,8 +108,8 @@ class Tokenizer: tag_splitter = re.compile(r"([\s\"\'\\]+)") def __init__(self): - self._text = None - self._head = 0 + self._text: list[str] = [] + self._head: int = 0 self._stacks = [] self._global = 0 self._depth = 0 @@ -222,27 +231,44 @@ class Tokenizer: self._emit_all(stack) self._head -= 1 - def _read(self, delta=0, wrap=False, strict=False): + @overload + def _read( + self, *, strict: Literal[False] = False + ) -> str | Literal[Sentinel.END]: ... + + @overload + def _read(self, *, strict: Literal[True]) -> str: ... + + @overload + def _read( + self, delta: int = 0, *, strict: Literal[False] = False + ) -> str | Literal[Sentinel.START, Sentinel.END]: ... + + @overload + def _read( + self, delta: int = 0, *, strict: Literal[True] + ) -> str | Literal[Sentinel.START]: ... + + def _read( + self, delta: int = 0, *, strict: bool = False + ) -> str | Literal[Sentinel.START, Sentinel.END]: """Read the value at a relative point in the wikicode. The value is read from :attr:`self._head <_head>` plus the value of - *delta* (which can be negative). If *wrap* is ``False``, we will not - allow attempts to read from the end of the string if ``self._head + - delta`` is negative. If *strict* is ``True``, the route will be failed - (with :meth:`_fail_route`) if we try to read from past the end of the - string; otherwise, :attr:`self.END ` is returned. If we try to - read from before the start of the string, :attr:`self.START ` is - returned. + *delta* (which can be negative). If *strict* is ``True``, the route + will be failed (with :meth:`_fail_route`) if we try to read from past + the end of the string; otherwise, ``END`` is returned. If we try to + read from before the start of the string, ``START`` is returned. """ index = self._head + delta - if index < 0 and (not wrap or abs(index) > len(self._text)): - return self.START + if index < 0: + return START try: return self._text[index] except IndexError: if strict: self._fail_route() - return self.END + return END def _parse_template(self, has_content): """Parse a template at the head of the wikicode string.""" @@ -399,12 +425,12 @@ class Tokenizer: self._head += 2 else: - def all_valid(): - return all(char in self.URISCHEME for char in self._read()) + def all_valid(this: str): + return all(char in self.URISCHEME for char in this) scheme = "" - while self._read() is not self.END and all_valid(): - scheme += self._read() + while (this := self._read()) is not END and all_valid(this): + scheme += this self._emit_text(self._read()) self._head += 1 if self._read() != ":": @@ -472,7 +498,7 @@ class Tokenizer: # Built from _parse()'s end sentinels: after, ctx = self._read(2), self._context return ( - this in (self.END, "\n", "[", "]", "<", ">", '"') + this in (END, "\n", "[", "]", "<", ">", '"') or " " in this or this == nxt == "'" or (this == "|" and ctx & contexts.TEMPLATE) @@ -491,7 +517,7 @@ class Tokenizer: self._parse_free_uri_scheme() invalid = ("\n", " ", "[", "]") punct = tuple(",;\\.:!?)") - if self._read() is self.END or self._read()[0] in invalid: + if (this := self._read()) is END or this[0] in invalid: self._fail_route() tail = "" while True: @@ -512,7 +538,7 @@ class Tokenizer: tail = "" self._parse_template_or_argument() elif brackets: - if this is self.END or this == "\n": + if this is END or this == "\n": self._fail_route() if this == "]": return self._pop(), None @@ -534,7 +560,7 @@ class Tokenizer: self._emit_text(this) else: if self._is_uri_end(this, nxt): - if this is not self.END and " " in this: + if this is not END and " " in this: before, after = this.split(" ", 1) punct, tail = self._handle_free_link_text(punct, tail, before) tail += " " + after @@ -616,7 +642,7 @@ class Tokenizer: while self._read() == "=": best += 1 self._head += 1 - current = int(log(self._context / contexts.HEADING_LEVEL_1, 2)) + 1 + current = int(math.log(self._context / contexts.HEADING_LEVEL_1, 2)) + 1 level = min(current, min(best, 6)) try: # Try to check for a heading closure after this one @@ -667,7 +693,7 @@ class Tokenizer: if test < 1 or test > 0x10FFFF: self._fail_route() else: - if this not in htmlentities.entitydefs: + if this not in html.entities.entitydefs: self._fail_route() self._emit(tokens.Text(text=this)) @@ -692,7 +718,7 @@ class Tokenizer: self._push() while True: this = self._read() - if this == self.END: + if this == END: self._pop() self._head = reset self._emit_text("