From 4baab6f57cf3e29cc8cbc5e160dd6240df0c7739 Mon Sep 17 00:00:00 2001 From: Ben Kurtovic Date: Sat, 11 Aug 2012 22:04:24 -0400 Subject: [PATCH] Implement lazy importing of root-level modules and packages. - Simplify all imports - Update dependency version in setup.py - Change waitTime default from three seconds to two --- earwigbot/__init__.py | 24 ++++++----- earwigbot/commands/time_command.py | 5 +-- earwigbot/config/__init__.py | 17 ++------ earwigbot/config/ordered_yaml.py | 10 ++--- earwigbot/config/script.py | 19 ++------- earwigbot/lazy.py | 81 +++++++++++++++++++++++++++++++++++++ earwigbot/wiki/copyvios/__init__.py | 5 +-- earwigbot/wiki/copyvios/parsers.py | 17 ++------ earwigbot/wiki/copyvios/search.py | 5 +-- earwigbot/wiki/page.py | 5 +-- earwigbot/wiki/site.py | 7 +--- earwigbot/wiki/sitesdb.py | 4 +- setup.py | 8 ++-- 13 files changed, 121 insertions(+), 86 deletions(-) create mode 100644 earwigbot/lazy.py diff --git a/earwigbot/__init__.py b/earwigbot/__init__.py index 6fd26a8..96e227f 100644 --- a/earwigbot/__init__.py +++ b/earwigbot/__init__.py @@ -51,12 +51,18 @@ if not __release__: finally: del _get_git_commit_id -from earwigbot import bot -from earwigbot import commands -from earwigbot import config -from earwigbot import exceptions -from earwigbot import irc -from earwigbot import managers -from earwigbot import tasks -from earwigbot import util -from earwigbot import wiki +from earwigbot import lazy + +importer = lazy.LazyImporter() + +bot = importer.new("earwigbot.bot") +commands = importer.new("earwigbot.commands") +config = importer.new("earwigbot.config") +exceptions = importer.new("earwigbot.exceptions") +irc = importer.new("earwigbot.irc") +managers = importer.new("earwigbot.managers") +tasks = importer.new("earwigbot.tasks") +util = importer.new("earwigbot.util") +wiki = importer.new("earwigbot.wiki") + +del importer diff --git a/earwigbot/commands/time_command.py b/earwigbot/commands/time_command.py index 94a85e6..a172a43 100644 --- a/earwigbot/commands/time_command.py +++ b/earwigbot/commands/time_command.py @@ -24,10 +24,7 @@ from datetime import datetime from math import floor from time import time -try: - import pytz -except ImportError: - pytz = None +import pytz from earwigbot.commands import Command diff --git a/earwigbot/config/__init__.py b/earwigbot/config/__init__.py index d21cc39..b0d7fae 100644 --- a/earwigbot/config/__init__.py +++ b/earwigbot/config/__init__.py @@ -28,20 +28,9 @@ import logging.handlers from os import mkdir, path import stat -try: - from Crypto.Cipher import Blowfish -except ImportError: - Blowfish = None - -try: - import bcrypt -except ImportError: - bcrypt = None - -try: - import yaml -except ImportError: - yaml = None +from Crypto.Cipher import Blowfish +import bcrypt +import yaml from earwigbot.config.formatter import BotFormatter from earwigbot.config.node import ConfigNode diff --git a/earwigbot/config/ordered_yaml.py b/earwigbot/config/ordered_yaml.py index c95f7ee..e3fa806 100644 --- a/earwigbot/config/ordered_yaml.py +++ b/earwigbot/config/ordered_yaml.py @@ -29,15 +29,11 @@ with modifications. from collections import OrderedDict -try: - import yaml - from yaml import Loader, SafeDumper -except ImportError: - yaml = Loader = SafeDumper = None +import yaml __all__ = ["OrderedLoader", "OrderedDumper"] -class OrderedLoader(Loader): +class OrderedLoader(yaml.Loader): """A YAML loader that loads mappings into ordered dictionaries.""" def __init__(self, *args, **kwargs): @@ -75,7 +71,7 @@ class OrderedLoader(Loader): return mapping -class OrderedDumper(SafeDumper): +class OrderedDumper(yaml.SafeDumper): """A YAML dumper that dumps ordered dictionaries into mappings.""" def __init__(self, *args, **kwargs): diff --git a/earwigbot/config/script.py b/earwigbot/config/script.py index f25231d..9d953e9 100644 --- a/earwigbot/config/script.py +++ b/earwigbot/config/script.py @@ -29,20 +29,9 @@ import stat import sys from textwrap import fill, wrap -try: - from Crypto.Cipher import Blowfish -except ImportError: - Blowfish = None - -try: - import bcrypt -except ImportError: - bcrypt = None - -try: - import yaml -except ImportError: - yaml = None +from Crypto.Cipher import Blowfish +import bcrypt +import yaml from earwigbot import exceptions from earwigbot.config.ordered_yaml import OrderedDumper @@ -271,7 +260,7 @@ class ConfigScript(object): self.data["wiki"]["useHTTPS"] = True self.data["wiki"]["assert"] = "user" self.data["wiki"]["maxlag"] = 10 - self.data["wiki"]["waitTime"] = 3 + self.data["wiki"]["waitTime"] = 2 self.data["wiki"]["defaultSite"] = self._login(kwargs).name self.data["wiki"]["sql"] = {} diff --git a/earwigbot/lazy.py b/earwigbot/lazy.py new file mode 100644 index 0000000..555f039 --- /dev/null +++ b/earwigbot/lazy.py @@ -0,0 +1,81 @@ +# -*- coding: utf-8 -*- +# +# Copyright (C) 2009-2012 Ben Kurtovic +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +""" +Implements a hierarchy of importing classes as defined in PEP 302 to load +modules in a safe yet lazy manner. +""" + +from imp import acquire_lock, release_lock +import sys +from types import ModuleType + +__all__ = ["LazyImporter"] + +def _getattribute(self, attr): + _load(self) + return self.__getattribute__(attr) + +def _setattr(self, attr, value): + _load(self) + self.__setattr__(attr, value) + +def _load(self): + type(self).__getattribute__ = ModuleType.__getattribute__ + type(self).__setattr__ = ModuleType.__setattr__ + reload(self) + + +class _LazyModule(type): + def __new__(cls, name): + acquire_lock() + try: + if name not in sys.modules: + attributes = { + "__name__": name, + "__getattribute__": _getattribute, + "__setattr__": _setattr + } + parents = (ModuleType,) + klass = type.__new__(cls, "module", parents, attributes) + sys.modules[name] = klass(name) + return sys.modules[name] + finally: + release_lock() + + +class LazyImporter(object): + def __init__(self): + self._modules = {} + sys.meta_path.append(self) + + def new(self, name): + module = _LazyModule(name) + self._modules[name] = module + return module + + def find_module(self, fullname, path=None): + if fullname in self._modules and fullname not in sys.modules: + return self + + def load_module(self, fullname): + return self._modules.pop(fullname) diff --git a/earwigbot/wiki/copyvios/__init__.py b/earwigbot/wiki/copyvios/__init__.py index ee0bc9e..cd643f3 100644 --- a/earwigbot/wiki/copyvios/__init__.py +++ b/earwigbot/wiki/copyvios/__init__.py @@ -26,10 +26,7 @@ from StringIO import StringIO from time import sleep, time from urllib2 import build_opener, URLError -try: - import oauth2 as oauth -except ImportError: - oauth = None +import oauth2 as oauth from earwigbot import exceptions from earwigbot.wiki.copyvios.markov import MarkovChain, MarkovChainIntersection diff --git a/earwigbot/wiki/copyvios/parsers.py b/earwigbot/wiki/copyvios/parsers.py index 0df925b..798b2a7 100644 --- a/earwigbot/wiki/copyvios/parsers.py +++ b/earwigbot/wiki/copyvios/parsers.py @@ -22,20 +22,9 @@ from os import path -try: - import bs4 -except ImportError: - bs4 = None - -try: - import mwparserfromhell -except ImportError: - mwparserfromhell = None - -try: - import nltk -except ImportError: - nltk = None +import bs4 +import mwparserfromhell +import nltk __all__ = ["BaseTextParser", "ArticleTextParser", "HTMLTextParser"] diff --git a/earwigbot/wiki/copyvios/search.py b/earwigbot/wiki/copyvios/search.py index 0ccd62e..0834885 100644 --- a/earwigbot/wiki/copyvios/search.py +++ b/earwigbot/wiki/copyvios/search.py @@ -23,10 +23,7 @@ from json import loads from urllib import quote_plus, urlencode -try: - import oauth2 as oauth -except ImportError: - oauth = None +import oauth2 as oauth from earwigbot.exceptions import SearchQueryError diff --git a/earwigbot/wiki/page.py b/earwigbot/wiki/page.py index 17274bc..1496a2d 100644 --- a/earwigbot/wiki/page.py +++ b/earwigbot/wiki/page.py @@ -26,10 +26,7 @@ import re from time import gmtime, strftime from urllib import quote -try: - import mwparserfromhell -except ImportError: - mwparserfromhell = None +import mwparserfromhell from earwigbot import exceptions from earwigbot.wiki.copyvios import CopyvioMixIn diff --git a/earwigbot/wiki/site.py b/earwigbot/wiki/site.py index 1bc6dcf..6043675 100644 --- a/earwigbot/wiki/site.py +++ b/earwigbot/wiki/site.py @@ -32,10 +32,7 @@ from urllib import quote_plus, unquote_plus from urllib2 import build_opener, HTTPCookieProcessor, URLError from urlparse import urlparse -try: - import oursql -except ImportError: - oursql = None +import oursql from earwigbot import exceptions from earwigbot.wiki import constants @@ -90,7 +87,7 @@ class Site(object): article_path=None, script_path=None, sql=None, namespaces=None, login=(None, None), cookiejar=None, user_agent=None, use_https=False, assert_edit=None, - maxlag=None, wait_between_queries=3, logger=None, + maxlag=None, wait_between_queries=2, logger=None, search_config=None): """Constructor for new Site instances. diff --git a/earwigbot/wiki/sitesdb.py b/earwigbot/wiki/sitesdb.py index d8c2e3b..5b37f78 100644 --- a/earwigbot/wiki/sitesdb.py +++ b/earwigbot/wiki/sitesdb.py @@ -191,7 +191,7 @@ class SitesDB(object): use_https = config.wiki.get("useHTTPS", False) assert_edit = config.wiki.get("assert") maxlag = config.wiki.get("maxlag") - wait_between_queries = config.wiki.get("waitTime", 3) + wait_between_queries = config.wiki.get("waitTime", 2) logger = self._logger.getChild(name) search_config = config.wiki.get("search", OrderedDict()).copy() @@ -390,7 +390,7 @@ class SitesDB(object): use_https = config.wiki.get("useHTTPS", True) assert_edit = config.wiki.get("assert") maxlag = config.wiki.get("maxlag") - wait_between_queries = config.wiki.get("waitTime", 3) + wait_between_queries = config.wiki.get("waitTime", 2) if user_agent: user_agent = user_agent.replace("$1", __version__) diff --git a/setup.py b/setup.py index 6809ac2..716f1c8 100644 --- a/setup.py +++ b/setup.py @@ -34,14 +34,14 @@ dependencies = [ "GitPython >= 0.3.2.RC1", # Interfacing with git for !git and __version__ "PyYAML >= 3.10", # Parsing config files "beautifulsoup4 >= 4.1.1", # Parsing/scraping HTML for copyvios - "lxml >= 2.3.4", # Faster parser for BeautifulSoup + "lxml >= 2.3.5", # Faster parser for BeautifulSoup "mwparserfromhell >= 0.1", # Parsing wikicode for manipulation "nltk >= 2.0.2", # Parsing sentences to split article content for copyvios - "oursql >= 0.9.3", # Interfacing with MediaWiki databases "oauth2 >= 1.5.211", # Interfacing with Yahoo! BOSS Search for copyvios + "oursql >= 0.9.3.1", # Interfacing with MediaWiki databases "py-bcrypt >= 0.2", # Hashing the bot key in the config file - "pycrypto >= 2.5", # Storing bot passwords and keys in the config file - "pytz >= 2012c", # Handling timezones for the !time IRC command + "pycrypto >= 2.6", # Storing bot passwords and keys in the config file + "pytz >= 2012d", # Handling timezones for the !time IRC command ] with open("README.rst") as fp: