More unicode fixes and refactoring.

earwigbot/commands/langcode.py

@@ -44,8 +44,9 @@ class Langcode(Command):
         for site in matrix.itervalues():
             if site["code"] == code:
                 name = site["name"].encode("utf8")
-                if name != site["localname"]:
-                    name += " ({0})".format(site["localname"].encode("utf8"))
+                localname = site["localname"].encode("utf8")
+                if name != localname:
+                    name += " ({0})".format(localname)
                 sites = ", ".join([s["url"] for s in site["site"]])
                 msg = "\x0302{0}\x0301 is {1} ({2})".format(code, name, sites)
                 self.reply(data, msg)

earwigbot/wiki/__init__.py

@@ -43,6 +43,26 @@ of your :py:class:`~earwigbot.wiki.site.Site` (and thus,
 :py:class:`~earwigbot.wiki.user.User`) needs.
 """
 
+from urllib import quote_plus
+
+# Some helper functions:
+
+def unicodeify(value, encoding="utf8"):
+    """Return input as unicode if it's not unicode to begin with."""
+    if isinstance(value, unicode):
+        return value
+    return unicode(value, encoding)
+
+def urlencode_utf8(params):
+    """Implement urllib.urlencode() with support for unicode input."""
+    enc = lambda s: s.encode("utf8") if isinstance(s, unicode) else str(s)
+    args = []
+    for key, val in params.iteritems():
+        key = quote_plus(enc(key))
+        val = quote_plus(enc(val))
+        args.append(key + "=" + val)
+    return "&".join(args)
+
 from earwigbot.wiki.category import *
 from earwigbot.wiki.constants import *
 from earwigbot.wiki.page import *

earwigbot/wiki/page.py

@@ -31,6 +31,7 @@ except ImportError:
     mwparserfromhell = None
 
 from earwigbot import exceptions
+from earwigbot.wiki import unicodeify
 from earwigbot.wiki.copyright import CopyrightMixIn
 
 __all__ = ["Page"]
@@ -149,7 +150,7 @@ class Page(CopyrightMixIn):
         contains "[") it will always be invalid, and cannot be edited.
         """
         if self._exists == self.PAGE_INVALID:
-            e = u"Page '{0}' is invalid.".format(self._title)
+            e = u"Page '{0}' is invalid.".format(unicodeify(self._title))
             raise exceptions.InvalidPageError(e)
 
     def _assert_existence(self):
@@ -161,7 +162,7 @@ class Page(CopyrightMixIn):
         """
         self._assert_validity()
         if self._exists == self.PAGE_MISSING:
-            e = u"Page '{0}' does not exist.".format(self._title)
+            e = u"Page '{0}' does not exist.".format(unicodeify(self._title))
             raise exceptions.PageNotFoundError(e)
 
     def _load(self):

earwigbot/wiki/site.py

@@ -29,7 +29,6 @@ from re import escape as re_escape, match as re_match
 from StringIO import StringIO
 from threading import Lock
 from time import sleep, time
-from urllib import quote_plus
 from urllib2 import build_opener, HTTPCookieProcessor, URLError
 from urlparse import urlparse
 
@@ -39,7 +38,7 @@ except ImportError:
     oursql = None
 
 from earwigbot import exceptions
-from earwigbot.wiki import constants
+from earwigbot.wiki import constants, unicodeify, urlencode_utf8
 from earwigbot.wiki.category import Category
 from earwigbot.wiki.page import Page
 from earwigbot.wiki.user import User
@@ -185,22 +184,6 @@ class Site(object):
         res = "<Site {0} ({1}:{2}) at {3}>"
         return res.format(self.name, self.project, self.lang, self.domain)
 
-    def _unicodeify(self, value, encoding="utf8"):
-        """Return input as unicode if it's not unicode to begin with."""
-        if isinstance(value, unicode):
-            return value
-        return unicode(value, encoding)
-
-    def _urlencode_utf8(self, params):
-        """Implement urllib.urlencode() with support for unicode input."""
-        enc = lambda s: s.encode("utf8") if isinstance(s, unicode) else str(s)
-        args = []
-        for key, val in params.iteritems():
-            key = quote_plus(enc(key))
-            val = quote_plus(enc(val))
-            args.append(key + "=" + val)
-        return "&".join(args)
-
     def _api_query(self, params, tries=0, wait=5):
         """Do an API query with *params* as a dict of parameters.
 
@@ -251,7 +234,7 @@ class Site(object):
         if self._maxlag:  # If requested, don't overload the servers
             params["maxlag"] = self._maxlag
 
-        data = self._urlencode_utf8(params)
+        data = urlencode_utf8(params)
         return url, data
 
     def _handle_api_query_result(self, result, params, tries, wait):
@@ -707,7 +690,7 @@ class Site(object):
         redirect-following: :py:class:`~earwigbot.wiki.page.Page`'s methods
         provide that.
         """
-        title = self._unicodeify(title)
+        title = unicodeify(title)
         prefixes = self.namespace_id_to_name(constants.NS_CATEGORY, all=True)
         prefix = title.split(":", 1)[0]
         if prefix != title:  # Avoid a page that is simply "Category"
@@ -722,7 +705,7 @@ class Site(object):
         really just shorthand for :py:meth:`get_page("Category:" + catname)
         <get_page>`.
         """
-        catname = self._unicodeify(catname)
+        catname = unicodeify(catname)
         name = name if isinstance(name, unicode) else name.decode("utf8")
         prefix = self.namespace_id_to_name(constants.NS_CATEGORY)
         pagename = u':'.join((prefix, catname))
@@ -735,7 +718,7 @@ class Site(object):
         :py:class:`~earwigbot.wiki.user.User` object representing the currently
         logged-in (or anonymous!) user is returned.
         """
-        username = self._unicodeify(username)
+        username = unicodeify(username)
         if not username:
             username = self._get_username()
         return User(self, username)

earwigbot/wiki/user.py

@@ -23,7 +23,7 @@
 from time import gmtime, strptime
 
 from earwigbot.exceptions import UserNotFoundError
-from earwigbot.wiki import constants
+from earwigbot.wiki import constants, unicodeify
 from earwigbot.wiki.page import Page
 
 __all__ = ["User"]
@@ -96,7 +96,7 @@ class User(object):
         if not hasattr(self, attr):
             self._load_attributes()
         if not self._exists:
-            e = u"User '{0}' does not exist.".format(self._name)
+            e = u"User '{0}' does not exist.".format(unicodeify(self._name))
             raise UserNotFoundError(e)
         return getattr(self, attr)