Implement lazy importing of root-level modules and packages.

- Simplify all imports
- Update dependency version in setup.py
- Change waitTime default from three seconds to two

earwigbot/__init__.py

@@ -51,12 +51,18 @@ if not __release__:
     finally:
         del _get_git_commit_id
 
-from earwigbot import bot
-from earwigbot import commands
-from earwigbot import config
-from earwigbot import exceptions
-from earwigbot import irc
-from earwigbot import managers
-from earwigbot import tasks
-from earwigbot import util
-from earwigbot import wiki
+from earwigbot import lazy
+
+importer = lazy.LazyImporter()
+
+bot = importer.new("earwigbot.bot")
+commands = importer.new("earwigbot.commands")
+config = importer.new("earwigbot.config")
+exceptions = importer.new("earwigbot.exceptions")
+irc = importer.new("earwigbot.irc")
+managers = importer.new("earwigbot.managers")
+tasks = importer.new("earwigbot.tasks")
+util = importer.new("earwigbot.util")
+wiki = importer.new("earwigbot.wiki")
+
+del importer

earwigbot/commands/time_command.py

@@ -24,10 +24,7 @@ from datetime import datetime
 from math import floor
 from time import time
 
-try:
-    import pytz
-except ImportError:
-    pytz = None
+import pytz
 
 from earwigbot.commands import Command
 

earwigbot/config/__init__.py

@@ -28,20 +28,9 @@ import logging.handlers
 from os import mkdir, path
 import stat
 
-try:
-    from Crypto.Cipher import Blowfish
-except ImportError:
-    Blowfish = None
-
-try:
-    import bcrypt
-except ImportError:
-    bcrypt = None
-
-try:
-    import yaml
-except ImportError:
-    yaml = None
+from Crypto.Cipher import Blowfish
+import bcrypt
+import yaml
 
 from earwigbot.config.formatter import BotFormatter
 from earwigbot.config.node import ConfigNode

earwigbot/config/ordered_yaml.py

@@ -29,15 +29,11 @@ with modifications.
 
 from collections import OrderedDict
 
-try:
-    import yaml
-    from yaml import Loader, SafeDumper
-except ImportError:
-    yaml = Loader = SafeDumper = None
+import yaml
 
 __all__ = ["OrderedLoader", "OrderedDumper"]
 
-class OrderedLoader(Loader):
+class OrderedLoader(yaml.Loader):
     """A YAML loader that loads mappings into ordered dictionaries."""
 
     def __init__(self, *args, **kwargs):
@@ -75,7 +71,7 @@ class OrderedLoader(Loader):
         return mapping
 
 
-class OrderedDumper(SafeDumper):
+class OrderedDumper(yaml.SafeDumper):
     """A YAML dumper that dumps ordered dictionaries into mappings."""
 
     def __init__(self, *args, **kwargs):

earwigbot/config/script.py

@@ -29,20 +29,9 @@ import stat
 import sys
 from textwrap import fill, wrap
 
-try:
-    from Crypto.Cipher import Blowfish
-except ImportError:
-    Blowfish = None
-
-try:
-    import bcrypt
-except ImportError:
-    bcrypt = None
-
-try:
-    import yaml
-except ImportError:
-    yaml = None
+from Crypto.Cipher import Blowfish
+import bcrypt
+import yaml
 
 from earwigbot import exceptions
 from earwigbot.config.ordered_yaml import OrderedDumper
@@ -271,7 +260,7 @@ class ConfigScript(object):
         self.data["wiki"]["useHTTPS"] = True
         self.data["wiki"]["assert"] = "user"
         self.data["wiki"]["maxlag"] = 10
-        self.data["wiki"]["waitTime"] = 3
+        self.data["wiki"]["waitTime"] = 2
         self.data["wiki"]["defaultSite"] = self._login(kwargs).name
         self.data["wiki"]["sql"] = {}
 

earwigbot/lazy.py

@@ -0,0 +1,81 @@
+# -*- coding: utf-8  -*-
+#
+# Copyright (C) 2009-2012 Ben Kurtovic <ben.kurtovic@verizon.net>
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+"""
+Implements a hierarchy of importing classes as defined in PEP 302 to load
+modules in a safe yet lazy manner.
+"""
+
+from imp import acquire_lock, release_lock
+import sys
+from types import ModuleType
+
+__all__ = ["LazyImporter"]
+
+def _getattribute(self, attr):
+    _load(self)
+    return self.__getattribute__(attr)
+
+def _setattr(self, attr, value):
+    _load(self)
+    self.__setattr__(attr, value)
+
+def _load(self):
+    type(self).__getattribute__ = ModuleType.__getattribute__
+    type(self).__setattr__ = ModuleType.__setattr__
+    reload(self)
+
+
+class _LazyModule(type):
+    def __new__(cls, name):
+        acquire_lock()
+        try:
+            if name not in sys.modules:
+                attributes = {
+                    "__name__": name,
+                    "__getattribute__": _getattribute,
+                    "__setattr__": _setattr
+                }
+                parents = (ModuleType,)
+                klass = type.__new__(cls, "module", parents, attributes)
+                sys.modules[name] = klass(name)
+            return sys.modules[name]
+        finally:
+            release_lock()
+
+
+class LazyImporter(object):
+    def __init__(self):
+        self._modules = {}
+        sys.meta_path.append(self)
+
+    def new(self, name):
+        module = _LazyModule(name)
+        self._modules[name] = module
+        return module
+
+    def find_module(self, fullname, path=None):
+        if fullname in self._modules and fullname not in sys.modules:
+            return self
+
+    def load_module(self, fullname):
+        return self._modules.pop(fullname)

earwigbot/wiki/copyvios/__init__.py

@@ -26,10 +26,7 @@ from StringIO import StringIO
 from time import sleep, time
 from urllib2 import build_opener, URLError
 
-try:
-    import oauth2 as oauth
-except ImportError:
-    oauth = None
+import oauth2 as oauth
 
 from earwigbot import exceptions
 from earwigbot.wiki.copyvios.markov import MarkovChain, MarkovChainIntersection

earwigbot/wiki/copyvios/parsers.py

@@ -22,20 +22,9 @@
 
 from os import path
 
-try:
-    import bs4
-except ImportError:
-    bs4 = None
-
-try:
-    import mwparserfromhell
-except ImportError:
-    mwparserfromhell = None
-
-try:
-    import nltk
-except ImportError:
-    nltk = None
+import bs4
+import mwparserfromhell
+import nltk
 
 __all__ = ["BaseTextParser", "ArticleTextParser", "HTMLTextParser"]
 

earwigbot/wiki/copyvios/search.py

@@ -23,10 +23,7 @@
 from json import loads
 from urllib import quote_plus, urlencode
 
-try:
-    import oauth2 as oauth
-except ImportError:
-    oauth = None
+import oauth2 as oauth
 
 from earwigbot.exceptions import SearchQueryError
 

earwigbot/wiki/page.py

@@ -26,10 +26,7 @@ import re
 from time import gmtime, strftime
 from urllib import quote
 
-try:
-    import mwparserfromhell
-except ImportError:
-    mwparserfromhell = None
+import mwparserfromhell
 
 from earwigbot import exceptions
 from earwigbot.wiki.copyvios import CopyvioMixIn

earwigbot/wiki/site.py

@@ -32,10 +32,7 @@ from urllib import quote_plus, unquote_plus
 from urllib2 import build_opener, HTTPCookieProcessor, URLError
 from urlparse import urlparse
 
-try:
-    import oursql
-except ImportError:
-    oursql = None
+import oursql
 
 from earwigbot import exceptions
 from earwigbot.wiki import constants
@@ -90,7 +87,7 @@ class Site(object):
                  article_path=None, script_path=None, sql=None,
                  namespaces=None, login=(None, None), cookiejar=None,
                  user_agent=None, use_https=False, assert_edit=None,
-                 maxlag=None, wait_between_queries=3, logger=None,
+                 maxlag=None, wait_between_queries=2, logger=None,
                  search_config=None):
         """Constructor for new Site instances.
 

earwigbot/wiki/sitesdb.py

@@ -191,7 +191,7 @@ class SitesDB(object):
         use_https = config.wiki.get("useHTTPS", False)
         assert_edit = config.wiki.get("assert")
         maxlag = config.wiki.get("maxlag")
-        wait_between_queries = config.wiki.get("waitTime", 3)
+        wait_between_queries = config.wiki.get("waitTime", 2)
         logger = self._logger.getChild(name)
         search_config = config.wiki.get("search", OrderedDict()).copy()
 
@@ -390,7 +390,7 @@ class SitesDB(object):
         use_https = config.wiki.get("useHTTPS", True)
         assert_edit = config.wiki.get("assert")
         maxlag = config.wiki.get("maxlag")
-        wait_between_queries = config.wiki.get("waitTime", 3)
+        wait_between_queries = config.wiki.get("waitTime", 2)
 
         if user_agent:
             user_agent = user_agent.replace("$1", __version__)

setup.py

@@ -34,14 +34,14 @@ dependencies = [
     "GitPython >= 0.3.2.RC1",  # Interfacing with git for !git and __version__
     "PyYAML >= 3.10",  # Parsing config files
     "beautifulsoup4 >= 4.1.1",  # Parsing/scraping HTML for copyvios
-    "lxml >= 2.3.4",  # Faster parser for BeautifulSoup
+    "lxml >= 2.3.5",  # Faster parser for BeautifulSoup
     "mwparserfromhell >= 0.1",  # Parsing wikicode for manipulation
     "nltk >= 2.0.2",  # Parsing sentences to split article content for copyvios
-    "oursql >= 0.9.3",  # Interfacing with MediaWiki databases
     "oauth2 >= 1.5.211",  # Interfacing with Yahoo! BOSS Search for copyvios
+    "oursql >= 0.9.3.1",  # Interfacing with MediaWiki databases
     "py-bcrypt >= 0.2",  # Hashing the bot key in the config file
-    "pycrypto >= 2.5",  # Storing bot passwords and keys in the config file
-    "pytz >= 2012c",  # Handling timezones for the !time IRC command
+    "pycrypto >= 2.6",  # Storing bot passwords and keys in the config file
+    "pytz >= 2012d",  # Handling timezones for the !time IRC command
 ]
 
 with open("README.rst") as fp: