From 0ee505b5a506cfc1c0530935bb01933b94aa14dc Mon Sep 17 00:00:00 2001 From: Ben Kurtovic Date: Thu, 24 Jan 2013 01:24:06 -0500 Subject: [PATCH] Docstrings for new tokenizer methods. --- mwparserfromhell/parser/tokenizer.py | 41 ++++++++++++++++++++++++++++++------ mwparserfromhell/tag_defs.py | 2 +- 2 files changed, 36 insertions(+), 7 deletions(-) diff --git a/mwparserfromhell/parser/tokenizer.py b/mwparserfromhell/parser/tokenizer.py index e83ec5d..8ec3355 100644 --- a/mwparserfromhell/parser/tokenizer.py +++ b/mwparserfromhell/parser/tokenizer.py @@ -434,6 +434,10 @@ class Tokenizer(object): self._write_all(tokens) def _get_tag_type_from_stack(self, stack=None): + """Return the tag type based on the text in *stack*. + + If *stack* is ``None``, we will use the current, topmost one. + """ if stack is None: stack = self._stack self._push_textbuffer() @@ -447,6 +451,13 @@ class Tokenizer(object): self._fail_route() def _actually_close_tag_opening(self): + """Handle cleanup at the end of a opening tag. + + The current context will be updated and the + :py:class:`~.tokens.TagOpenOpen` token will be written. Returns the + opening tag's padding to be used in the + :py:class:`~.tokens.TagOpenClose` token. + """ if self._context & contexts.TAG_OPEN_ATTR: if self._context & contexts.TAG_OPEN_ATTR_NAME: self._context ^= contexts.TAG_OPEN_ATTR_NAME @@ -463,6 +474,11 @@ class Tokenizer(object): return "" def _actually_handle_chunk(self, chunks, is_new): + """Actually handle a chunk of code within a tag's attributes. + + Called by :py:meth:`_handle_tag_chunk` and + :py:meth:`_handle_tag_attribute_body`. + """ if is_new and not self._context & contexts.TAG_OPEN_ATTR_QUOTED: padding = 0 while chunks: @@ -495,6 +511,12 @@ class Tokenizer(object): self._write_text(chunk) def _handle_tag_chunk(self, text): + """Handle a chunk of code within a tag's attributes. + + This is called by :py:meth:`_parse`, which intercepts parsing of + wikicode when we're inside of an opening tag and no :py:attr:`MARKERS` + are present. + """ if " " not in text: self._write_text(text) return @@ -517,6 +539,12 @@ class Tokenizer(object): return self._pop() def _handle_tag_attribute_body(self): + """Handle the body, or value, of a tag attribute. + + Attribute bodies can usually be handled at once, but sometimes a new + stack must be created to keep track of "rich" attribute values that + contain, for example, templates. + """ self._context ^= contexts.TAG_OPEN_ATTR_NAME self._context |= contexts.TAG_OPEN_ATTR_BODY self._write(tokens.TagAttrEquals()) @@ -552,21 +580,25 @@ class Tokenizer(object): self._actually_handle_chunk(chunks, True) def _handle_tag_close_open(self): + """Handle the ending of an open tag (````).""" padding = self._actually_close_tag_opening() self._write(tokens.TagCloseOpen(padding=padding)) def _handle_tag_selfclose(self): + """Handle the ending of an tag that closes itself (````).""" padding = self._actually_close_tag_opening() self._write(tokens.TagCloseSelfclose(padding=padding)) self._head += 1 return self._pop() def _handle_tag_open_close(self): + """Handle the opening of a closing tag (````).""" self._write(tokens.TagOpenClose()) self._push(contexts.TAG_CLOSE) self._head += 1 def _handle_tag_close_close(self): + """Handle the ending of a closing tag (````).""" closing = self._pop() tag = self._get_tag_type_from_stack(closing) if tag != self._stack[0].type: @@ -653,8 +685,7 @@ class Tokenizer(object): elif this == "<" and next != "/" and ( not self._context & (contexts.TAG ^ contexts.TAG_BODY)): self._parse_tag() - elif self._context & ( - contexts.TAG_OPEN ^ contexts.TAG_OPEN_ATTR_QUOTED): + elif self._context & (contexts.TAG_OPEN ^ contexts.TAG_OPEN_ATTR_QUOTED): if this == "\n": if self._context & contexts.TAG_CLOSE: self._pop() @@ -663,11 +694,9 @@ class Tokenizer(object): self._handle_tag_close_open() elif this == "/" and next == ">": return self._handle_tag_selfclose() - elif this == "=" and ( - self._context & contexts.TAG_OPEN_ATTR_NAME): + elif this == "=" and self._context & contexts.TAG_OPEN_ATTR_NAME: self._handle_tag_attribute_body() - elif this == "<" and next == "/" and ( - self._context & contexts.TAG_BODY): + elif this == "<" and next == "/" and self._context & contexts.TAG_BODY: self._handle_tag_open_close() elif this == ">" and self._context & contexts.TAG_CLOSE: return self._handle_tag_close_close() diff --git a/mwparserfromhell/tag_defs.py b/mwparserfromhell/tag_defs.py index 74d3a81..b2ee90d 100644 --- a/mwparserfromhell/tag_defs.py +++ b/mwparserfromhell/tag_defs.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # -# Copyright (C) 2012 Ben Kurtovic +# Copyright (C) 2012-2013 Ben Kurtovic # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal