From 117eccc35de9b0418f6e25fabe32a27c874e681d Mon Sep 17 00:00:00 2001 From: Ben Kurtovic Date: Thu, 5 Apr 2012 22:04:05 -0400 Subject: [PATCH 01/26] Beginning work (#16) --- .gitignore | 6 - bot.py | 70 --------- earwigbot/__init__.py | 4 +- earwigbot/bot.py | 113 +++++++++++++++ earwigbot/commands/restart.py | 2 +- earwigbot/commands/threads.py | 16 +-- earwigbot/config.py | 213 ++++++++++++++-------------- earwigbot/irc/connection.py | 33 ++++- earwigbot/irc/frontend.py | 12 +- earwigbot/irc/watcher.py | 12 +- earwigbot/main.py | 132 ----------------- earwigbot/runner.py | 65 --------- earwigbot/tasks/__init__.py | 4 - earwigbot/util.py | 50 +++++++ setup.py | 40 ++++++ {earwigbot/tests => tests}/__init__.py | 3 +- {earwigbot/tests => tests}/test_blowfish.py | 0 {earwigbot/tests => tests}/test_calc.py | 0 {earwigbot/tests => tests}/test_test.py | 0 19 files changed, 353 insertions(+), 422 deletions(-) delete mode 100755 bot.py create mode 100644 earwigbot/bot.py delete mode 100644 earwigbot/main.py delete mode 100644 earwigbot/runner.py create mode 100755 earwigbot/util.py create mode 100644 setup.py rename {earwigbot/tests => tests}/__init__.py (98%) rename {earwigbot/tests => tests}/test_blowfish.py (100%) rename {earwigbot/tests => tests}/test_calc.py (100%) rename {earwigbot/tests => tests}/test_test.py (100%) diff --git a/.gitignore b/.gitignore index d2b75fb..91e9551 100644 --- a/.gitignore +++ b/.gitignore @@ -1,9 +1,3 @@ -# Ignore bot-specific files: -logs/ -config.yml -sites.db -.cookies - # Ignore python bytecode: *.pyc diff --git a/bot.py b/bot.py deleted file mode 100755 index d8f2d21..0000000 --- a/bot.py +++ /dev/null @@ -1,70 +0,0 @@ -#! /usr/bin/env python -# -*- coding: utf-8 -*- -# -# Copyright (C) 2009-2012 by Ben Kurtovic -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. - -""" -EarwigBot - -This is a thin wrapper for EarwigBot's main bot code, specified by bot_script. -The wrapper will automatically restart the bot when it shuts down (from -!restart, for example). It requests the bot's password at startup and reuses it -every time the bot restarts internally, so you do not need to re-enter the -password after using !restart. - -For information about the bot as a whole, see the attached README.md file (in -markdown format!), the docs/ directory, and the LICENSE file for licensing -information. EarwigBot is released under the MIT license. -""" -from getpass import getpass -from subprocess import Popen, PIPE -from os import path -from sys import executable -from time import sleep - -import earwigbot - -bot_script = path.join(earwigbot.__path__[0], "runner.py") - -def main(): - print "EarwigBot v{0}\n".format(earwigbot.__version__) - - is_encrypted = earwigbot.config.config.load() - if is_encrypted: # Passwords in the config file are encrypted - key = getpass("Enter key to unencrypt bot passwords: ") - else: - key = None - - while 1: - bot = Popen([executable, bot_script], stdin=PIPE) - print >> bot.stdin, path.dirname(path.abspath(__file__)) - if is_encrypted: - print >> bot.stdin, key - return_code = bot.wait() - if return_code == 1: - exit() # Let critical exceptions in the subprocess cause us to - # exit as well - else: - sleep(5) # Sleep between bot runs following a non-critical - # subprocess exit - -if __name__ == "__main__": - main() diff --git a/earwigbot/__init__.py b/earwigbot/__init__.py index 4dab7da..a78194c 100644 --- a/earwigbot/__init__.py +++ b/earwigbot/__init__.py @@ -31,6 +31,4 @@ __license__ = "MIT License" __version__ = "0.1.dev" __email__ = "ben.kurtovic@verizon.net" -from earwigbot import ( - blowfish, commands, config, irc, main, runner, tasks, tests, wiki -) +from earwigbot import blowfish, bot, commands, config, irc, tasks, util, wiki diff --git a/earwigbot/bot.py b/earwigbot/bot.py new file mode 100644 index 0000000..65c5442 --- /dev/null +++ b/earwigbot/bot.py @@ -0,0 +1,113 @@ +# -*- coding: utf-8 -*- +# +# Copyright (C) 2009-2012 by Ben Kurtovic +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +import threading +from time import sleep, time + +from earwigbot.config import BotConfig +from earwigbot.irc import Frontend, Watcher +from earwigbot.tasks import task_manager + +class Bot(object): + """ + The Bot class is the core of EarwigBot, essentially responsible for + starting the various bot components and making sure they are all happy. An + explanation of the different components follows: + + EarwigBot has three components that can run independently of each other: an + IRC front-end, an IRC watcher, and a wiki scheduler. + * The IRC front-end runs on a normal IRC server and expects users to + interact with it/give it commands. + * The IRC watcher runs on a wiki recent-changes server and listens for + edits. Users cannot interact with this part of the bot. + * The wiki scheduler runs wiki-editing bot tasks in separate threads at + user-defined times through a cron-like interface. + """ + + def __init__(self, root_dir): + self.config = BotConfig(root_dir) + self.logger = logging.getLogger("earwigbot") + self.frontend = None + self.watcher = None + + self._keep_scheduling = True + self._lock = threading.Lock() + + def _start_thread(self, name, target): + thread = threading.Thread(name=name, target=target) + thread.start() + + def _wiki_scheduler(self): + while self._keep_scheduling: + time_start = time() + task_manager.schedule() + time_end = time() + time_diff = time_start - time_end + if time_diff < 60: # Sleep until the next minute + sleep(60 - time_diff) + + def _start_components(self): + if self.config.components.get("irc_frontend"): + self.logger.info("Starting IRC frontend") + self.frontend = Frontend(self.config) + self._start_thread(name, self.frontend.loop) + + if self.config.components.get("irc_watcher"): + self.logger.info("Starting IRC watcher") + self.watcher = Watcher(self.config, self.frontend) + self._start_thread(name, self.watcher.loop) + + if self.config.components.get("wiki_scheduler"): + self.logger.info("Starting wiki scheduler") + self._start_thread(name, self._wiki_scheduler) + + def _loop(self): + while 1: + with self._lock: + if self.frontend and self.frontend.is_stopped(): + self.frontend._connect() + if self.watcher and self.watcher.is_stopped(): + self.watcher._connect() + sleep(5) + + def run(self): + self.config.load() + self.config.decrypt(config.wiki, "password") + self.config.decrypt(config.wiki, "search", "credentials", "key") + self.config.decrypt(config.wiki, "search", "credentials", "secret") + self.config.decrypt(config.irc, "frontend", "nickservPassword") + self.config.decrypt(config.irc, "watcher", "nickservPassword") + self._start_components() + self._loop() + + def reload(self): + #components = self.config.components + with self._lock: + self.config.load() + #if self.config.components.get("irc_frontend"): + + def stop(self): + if self.frontend: + self.frontend.stop() + if self.watcher: + self.watcher.stop() + self._keep_scheduling = False diff --git a/earwigbot/commands/restart.py b/earwigbot/commands/restart.py index 527fc82..2b39e5e 100644 --- a/earwigbot/commands/restart.py +++ b/earwigbot/commands/restart.py @@ -34,4 +34,4 @@ class Command(BaseCommand): return self.connection.logger.info("Restarting bot per owner request") - self.connection.is_running = False + self.connection.stop() diff --git a/earwigbot/commands/threads.py b/earwigbot/commands/threads.py index 33f686d..7cf70ae 100644 --- a/earwigbot/commands/threads.py +++ b/earwigbot/commands/threads.py @@ -78,10 +78,9 @@ class Command(BaseCommand): for thread in threads: tname = thread.name if tname == "MainThread": - tname = self.get_main_thread_name() - t = "\x0302{0}\x0301 (as main thread, id {1})" - normal_threads.append(t.format(tname, thread.ident)) - elif tname in ["irc-frontend", "irc-watcher", "wiki-scheduler"]: + t = "\x0302MainThread\x0301 (id {1})" + normal_threads.append(t.format(thread.ident)) + elif tname in config.components: t = "\x0302{0}\x0301 (id {1})" normal_threads.append(t.format(tname, thread.ident)) elif tname.startswith("reminder"): @@ -157,12 +156,3 @@ class Command(BaseCommand): task_manager.start(task_name, **data.kwargs) msg = "task \x0302{0}\x0301 started.".format(task_name) self.connection.reply(data, msg) - - def get_main_thread_name(self): - """Return the "proper" name of the MainThread.""" - if "irc_frontend" in config.components: - return "irc-frontend" - elif "wiki_schedule" in config.components: - return "wiki-scheduler" - else: - return "irc-watcher" diff --git a/earwigbot/config.py b/earwigbot/config.py index f1a977c..4cf0721 100644 --- a/earwigbot/config.py +++ b/earwigbot/config.py @@ -20,31 +20,7 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. -""" -EarwigBot's YAML Config File Parser - -This handles all tasks involving reading and writing to our config file, -including encrypting and decrypting passwords and making a new config file from -scratch at the inital bot run. - -Usually you'll just want to do "from earwigbot.config import config", which -returns a singleton _BotConfig object, with data accessible from various -attributes and functions: - -* config.components - enabled components -* config.wiki - information about wiki-editing -* config.tasks - information for bot tasks -* config.irc - information about IRC -* config.metadata - miscellaneous information -* config.schedule() - tasks scheduled to run at a given time - -Additionally, _BotConfig has some functions used in config loading: -* config.load() - loads and parses our config file, returning True if - passwords are stored encrypted or False otherwise -* config.decrypt() - given a key, decrypts passwords inside our config - variables; won't work if passwords aren't encrypted -""" - +from getpass import getpass import logging import logging.handlers from os import mkdir, path @@ -53,44 +29,36 @@ import yaml from earwigbot import blowfish -__all__ = ["config"] - -class _ConfigNode(object): - def __iter__(self): - for key in self.__dict__.iterkeys(): - yield key - - def __getitem__(self, item): - return self.__dict__.__getitem__(item) - - def _dump(self): - data = self.__dict__.copy() - for key, val in data.iteritems(): - if isinstance(val, _ConfigNode): - data[key] = val._dump() - return data - - def _load(self, data): - self.__dict__ = data.copy() - - def _decrypt(self, key, intermediates, item): - base = self.__dict__ - try: - for inter in intermediates: - base = base[inter] - except KeyError: - return - if item in base: - base[item] = blowfish.decrypt(key, base[item]) - - def get(self, *args, **kwargs): - return self.__dict__.get(*args, **kwargs) - - -class _BotConfig(object): - def __init__(self): - self._script_dir = path.dirname(path.abspath(__file__)) - self._root_dir = path.split(self._script_dir)[0] +class BotConfig(object): + """ + EarwigBot's YAML Config File Manager + + This handles all tasks involving reading and writing to our config file, + including encrypting and decrypting passwords and making a new config file + from scratch at the inital bot run. + + BotConfig has a few properties and functions, including the following: + * config.root_dir - bot's working directory; contains config.yml, logs/ + * config.path - path to the bot's config file + * config.components - enabled components + * config.wiki - information about wiki-editing + * config.tasks - information for bot tasks + * config.irc - information about IRC + * config.metadata - miscellaneous information + * config.schedule() - tasks scheduled to run at a given time + + BotConfig also has some functions used in config loading: + * config.load() - loads and parses our config file, returning True if + passwords are stored encrypted or False otherwise; + can also be used to easily reload config + * config.decrypt() - given a key, decrypts passwords inside our config + variables, and remembers to decrypt the password if + config is reloaded; won't do anything if passwords + aren't encrypted + """ + + def __init__(self, root_dir): + self._root_dir = root_dir self._config_path = path.join(self._root_dir, "config.yml") self._log_dir = path.join(self._root_dir, "logs") self._decryption_key = None @@ -104,17 +72,17 @@ class _BotConfig(object): self._nodes = [self._components, self._wiki, self._tasks, self._irc, self._metadata] + self._decryptable_nodes = [] def _load(self): - """Load data from our JSON config file (config.yml) into _config.""" + """Load data from our JSON config file (config.yml) into self._data.""" filename = self._config_path with open(filename, 'r') as fp: try: self._data = yaml.load(fp) except yaml.YAMLError as error: print "Error parsing config file {0}:".format(filename) - print error - exit(1) + raise def _setup_logging(self): """Configures the logging module so it works the way we want it to.""" @@ -135,7 +103,7 @@ class _BotConfig(object): else: msg = "log_dir ({0}) exists but is not a directory!" print msg.format(log_dir) - exit(1) + return main_handler = hand(logfile("bot.log"), "midnight", 1, 7) error_handler = hand(logfile("error.log"), "W6", 1, 4) @@ -149,27 +117,29 @@ class _BotConfig(object): h.setFormatter(formatter) logger.addHandler(h) - stream_handler = logging.StreamHandler() - stream_handler.setLevel(logging.DEBUG) - stream_handler.setFormatter(color_formatter) - logger.addHandler(stream_handler) + stream_handler = logging.StreamHandler() + stream_handler.setLevel(logging.DEBUG) + stream_handler.setFormatter(color_formatter) + logger.addHandler(stream_handler) - else: - logger.addHandler(logging.NullHandler()) + def _decrypt(self, node, nodes): + """Try to decrypt the contents of a config node. Use self.decrypt().""" + try: + node._decrypt(self._decryption_key, nodes[:-1], nodes[-1]) + except blowfish.BlowfishError as error: + print "Error decrypting passwords:" + raise def _make_new(self): """Make a new config file based on the user's input.""" - encrypt = raw_input("Would you like to encrypt passwords stored in config.yml? [y/n] ") - if encrypt.lower().startswith("y"): - is_encrypted = True - else: - is_encrypted = False - - return is_encrypted - - @property - def script_dir(self): - return self._script_dir + #m = "Would you like to encrypt passwords stored in config.yml? [y/n] " + #encrypt = raw_input(m) + #if encrypt.lower().startswith("y"): + # is_encrypted = True + #else: + # is_encrypted = False + raise NotImplementedError() + # yaml.dumps() @property def root_dir(self): @@ -182,7 +152,7 @@ class _BotConfig(object): @property def log_dir(self): return self._log_dir - + @property def data(self): """The entire config file.""" @@ -221,7 +191,7 @@ class _BotConfig(object): """Return True if passwords are encrypted, otherwise False.""" return self.metadata.get("encryptPasswords", False) - def load(self, config_path=None, log_dir=None): + def load(self): """Load, or reload, our config file. First, check if we have a valid config file, and if not, notify the @@ -232,19 +202,14 @@ class _BotConfig(object): wiki, tasks, irc, metadata) for easy access (as well as the internal _data variable). - If everything goes well, return True if stored passwords are - encrypted in the file, or False if they are not. + If config is being reloaded, encrypted items will be automatically + decrypted if they were decrypted beforehand. """ - if config_path: - self._config_path = config_path - if log_dir: - self._log_dir = log_dir - if not path.exists(self._config_path): print "You haven't configured the bot yet!" choice = raw_input("Would you like to do this now? [y/n] ") if choice.lower().startswith("y"): - return self._make_new() + self._make_new() else: exit(1) @@ -257,25 +222,28 @@ class _BotConfig(object): self.metadata._load(data.get("metadata", {})) self._setup_logging() - return self.is_encrypted() + if self.is_encrypted(): + if not self._decryption_key: + key = getpass("Enter key to decrypt bot passwords: ") + self._decryption_key = key + for node, nodes in self._decryptable_nodes: + self._decrypt(node, nodes) def decrypt(self, node, *nodes): """Use self._decryption_key to decrypt an object in our config tree. If this is called when passwords are not encrypted (check with - config.is_encrypted()), nothing will happen. + config.is_encrypted()), nothing will happen. We'll also keep track of + this node if config.load() is called again (i.e. to reload) and + automatically decrypt it. - An example usage would be: + Example usage: config.decrypt(config.irc, "frontend", "nickservPassword") + -> decrypts config.irc["frontend"]["nickservPassword"] """ - if not self.is_encrypted(): - return - try: - node._decrypt(self._decryption_key, nodes[:-1], nodes[-1]) - except blowfish.BlowfishError as error: - print "\nError decrypting passwords:" - print "{0}: {1}.".format(error.__class__.__name__, error) - exit(1) + self._decryptable_nodes.append((node, nodes)) + if self.is_encrypted(): + self._decrypt(node, nodes) def schedule(self, minute, hour, month_day, month, week_day): """Return a list of tasks scheduled to run at the specified time. @@ -311,6 +279,38 @@ class _BotConfig(object): return tasks +class _ConfigNode(object): + def __iter__(self): + for key in self.__dict__.iterkeys(): + yield key + + def __getitem__(self, item): + return self.__dict__.__getitem__(item) + + def _dump(self): + data = self.__dict__.copy() + for key, val in data.iteritems(): + if isinstance(val, _ConfigNode): + data[key] = val._dump() + return data + + def _load(self, data): + self.__dict__ = data.copy() + + def _decrypt(self, key, intermediates, item): + base = self.__dict__ + try: + for inter in intermediates: + base = base[inter] + except KeyError: + return + if item in base: + base[item] = blowfish.decrypt(key, base[item]) + + def get(self, *args, **kwargs): + return self.__dict__.get(*args, **kwargs) + + class _BotFormatter(logging.Formatter): def __init__(self, color=False): self._format = super(_BotFormatter, self).format @@ -336,6 +336,3 @@ class _BotFormatter(logging.Formatter): if record.levelno == logging.CRITICAL: record.lvl = l.join(("\x1b[1m\x1b[31m", "\x1b[0m")) # Bold red return record - - -config = _BotConfig() diff --git a/earwigbot/irc/connection.py b/earwigbot/irc/connection.py index 10b6a4d..cd99421 100644 --- a/earwigbot/irc/connection.py +++ b/earwigbot/irc/connection.py @@ -22,6 +22,7 @@ import socket import threading +from time import sleep __all__ = ["BrokenSocketException", "IRCConnection"] @@ -42,7 +43,7 @@ class IRCConnection(object): self.ident = ident self.realname = realname self.logger = logger - self.is_running = False + self._is_running = False # A lock to prevent us from sending two messages at once: self._lock = threading.Lock() @@ -53,8 +54,9 @@ class IRCConnection(object): try: self._sock.connect((self.host, self.port)) except socket.error: - self.logger.critical("Couldn't connect to IRC server", exc_info=1) - exit(1) + self.logger.exception("Couldn't connect to IRC server") + sleep(8) + self._connect() self._send("NICK {0}".format(self.nick)) self._send("USER {0} {1} * :{2}".format(self.ident, self.host, self.realname)) @@ -68,7 +70,7 @@ class IRCConnection(object): def _get(self, size=4096): """Receive (i.e. get) data from the server.""" - data = self._sock.recv(4096) + data = self._sock.recv(size) if not data: # Socket isn't giving us any data, so it is dead or broken: raise BrokenSocketException() @@ -121,21 +123,38 @@ class IRCConnection(object): msg = "PONG {0}".format(target) self._send(msg) + def quit(self, msg=None): + """Issue a quit message to the server.""" + if msg: + self._send("QUIT {0}".format(msg)) + else: + self._send("QUIT") + def loop(self): """Main loop for the IRC connection.""" - self.is_running = True + self._is_running = True read_buffer = "" while 1: try: read_buffer += self._get() except BrokenSocketException: - self.is_running = False + self._is_running = False break lines = read_buffer.split("\n") read_buffer = lines.pop() for line in lines: self._process_message(line) - if not self.is_running: + if self.is_stopped(): self._close() break + + def stop(self): + """Request the IRC connection to close at earliest convenience.""" + if self._is_running: + self.quit() + self._is_running = False + + def is_stopped(self): + """Return whether the IRC connection has been (or is to be) closed.""" + return not self._is_running diff --git a/earwigbot/irc/frontend.py b/earwigbot/irc/frontend.py index 2b7e7d1..13c6bc3 100644 --- a/earwigbot/irc/frontend.py +++ b/earwigbot/irc/frontend.py @@ -25,7 +25,6 @@ import re from earwigbot.commands import command_manager from earwigbot.irc import IRCConnection, Data, BrokenSocketException -from earwigbot.config import config __all__ = ["Frontend"] @@ -41,7 +40,8 @@ class Frontend(IRCConnection): """ sender_regex = re.compile(":(.*?)!(.*?)@(.*?)\Z") - def __init__(self): + def __init__(self, config): + self.config = config self.logger = logging.getLogger("earwigbot.frontend") cf = config.irc["frontend"] base = super(Frontend, self) @@ -66,7 +66,7 @@ class Frontend(IRCConnection): data.msg = " ".join(line[3:])[1:] data.chan = line[2] - if data.chan == config.irc["frontend"]["nick"]: + if data.chan == self.config.irc["frontend"]["nick"]: # This is a privmsg to us, so set 'chan' as the nick of the # sender, then check for private-only command hooks: data.chan = data.nick @@ -86,8 +86,8 @@ class Frontend(IRCConnection): elif line[1] == "376": # If we're supposed to auth to NickServ, do that: try: - username = config.irc["frontend"]["nickservUsername"] - password = config.irc["frontend"]["nickservPassword"] + username = self.config.irc["frontend"]["nickservUsername"] + password = self.config.irc["frontend"]["nickservPassword"] except KeyError: pass else: @@ -95,5 +95,5 @@ class Frontend(IRCConnection): self.say("NickServ", msg) # Join all of our startup channels: - for chan in config.irc["frontend"]["channels"]: + for chan in self.config.irc["frontend"]["channels"]: self.join(chan) diff --git a/earwigbot/irc/watcher.py b/earwigbot/irc/watcher.py index ad206d6..f387b13 100644 --- a/earwigbot/irc/watcher.py +++ b/earwigbot/irc/watcher.py @@ -24,7 +24,6 @@ import imp import logging from earwigbot.irc import IRCConnection, RC, BrokenSocketException -from earwigbot.config import config __all__ = ["Watcher"] @@ -39,7 +38,8 @@ class Watcher(IRCConnection): to channels on the IRC frontend. """ - def __init__(self, frontend=None): + def __init__(self, config, frontend=None): + self.config = config self.logger = logging.getLogger("earwigbot.watcher") cf = config.irc["watcher"] base = super(Watcher, self) @@ -58,7 +58,7 @@ class Watcher(IRCConnection): # Ignore messages originating from channels not in our list, to # prevent someone PMing us false data: - if chan not in config.irc["watcher"]["channels"]: + if chan not in self.config.irc["watcher"]["channels"]: return msg = " ".join(line[3:])[1:] @@ -72,7 +72,7 @@ class Watcher(IRCConnection): # When we've finished starting up, join all watcher channels: elif line[1] == "376": - for chan in config.irc["watcher"]["channels"]: + for chan in self.config.irc["watcher"]["channels"]: self.join(chan) def _prepare_process_hook(self): @@ -84,12 +84,12 @@ class Watcher(IRCConnection): # Set a default RC process hook that does nothing: self._process_hook = lambda rc: () try: - rules = config.data["rules"] + rules = self.config.data["rules"] except KeyError: return module = imp.new_module("_rc_event_processing_rules") try: - exec compile(rules, config.path, "exec") in module.__dict__ + exec compile(rules, self.config.path, "exec") in module.__dict__ except Exception: e = "Could not compile config file's RC event rules" self.logger.exception(e) diff --git a/earwigbot/main.py b/earwigbot/main.py deleted file mode 100644 index a738d5c..0000000 --- a/earwigbot/main.py +++ /dev/null @@ -1,132 +0,0 @@ -# -*- coding: utf-8 -*- -# -# Copyright (C) 2009-2012 by Ben Kurtovic -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. - -""" -EarwigBot's Main Module - -The core is essentially responsible for starting the various bot components -(irc, scheduler, etc) and making sure they are all happy. An explanation of the -different components follows: - -EarwigBot has three components that can run independently of each other: an IRC -front-end, an IRC watcher, and a wiki scheduler. -* The IRC front-end runs on a normal IRC server and expects users to interact - with it/give it commands. -* The IRC watcher runs on a wiki recent-changes server and listens for edits. - Users cannot interact with this part of the bot. -* The wiki scheduler runs wiki-editing bot tasks in separate threads at - user-defined times through a cron-like interface. - -There is a "priority" system here: -1. If the IRC frontend is enabled, it will run on the main thread, and the IRC - watcher and wiki scheduler (if enabled) will run on separate threads. -2. If the wiki scheduler is enabled, it will run on the main thread, and the - IRC watcher (if enabled) will run on a separate thread. -3. If the IRC watcher is enabled, it will run on the main (and only) thread. -Else, the bot will stop, as no components are enabled. -""" - -import logging -import threading -import time - -from earwigbot.config import config -from earwigbot.irc import Frontend, Watcher -from earwigbot.tasks import task_manager - -logger = logging.getLogger("earwigbot") - -def irc_watcher(frontend=None): - """Function to handle the IRC watcher as another thread (if frontend and/or - scheduler is enabled), otherwise run as the main thread.""" - while 1: # Restart the watcher component if it breaks (and nothing else) - watcher = Watcher(frontend) - try: - watcher.loop() - except: - logger.exception("Watcher had an error") - time.sleep(5) # Sleep a bit before restarting watcher - logger.warn("Watcher has stopped; restarting component") - -def wiki_scheduler(): - """Function to handle the wiki scheduler as another thread, or as the - primary thread if the IRC frontend is not enabled.""" - while 1: - time_start = time.time() - task_manager.schedule() - time_end = time.time() - time_diff = time_start - time_end - if time_diff < 60: # Sleep until the next minute - time.sleep(60 - time_diff) - -def irc_frontend(): - """If the IRC frontend is enabled, make it run on our primary thread, and - enable the wiki scheduler and IRC watcher on new threads if they are - enabled.""" - logger.info("Starting IRC frontend") - frontend = Frontend() - - if config.components.get("wiki_schedule"): - logger.info("Starting wiki scheduler") - task_manager.load() - t_scheduler = threading.Thread(target=wiki_scheduler) - t_scheduler.name = "wiki-scheduler" - t_scheduler.daemon = True - t_scheduler.start() - - if config.components.get("irc_watcher"): - logger.info("Starting IRC watcher") - t_watcher = threading.Thread(target=irc_watcher, args=(frontend,)) - t_watcher.name = "irc-watcher" - t_watcher.daemon = True - t_watcher.start() - - frontend.loop() - -def main(): - if config.components.get("irc_frontend"): - # Make the frontend run on our primary thread if enabled, and enable - # additional components through that function: - irc_frontend() - - elif config.components.get("wiki_schedule"): - # Run the scheduler on the main thread, but also run the IRC watcher on - # another thread iff it is enabled: - logger.info("Starting wiki scheduler") - task_manager.load() - if "irc_watcher" in enabled: - logger.info("Starting IRC watcher") - t_watcher = threading.Thread(target=irc_watcher) - t_watcher.name = "irc-watcher" - t_watcher.daemon = True - t_watcher.start() - wiki_scheduler() - - elif config.components.get("irc_watcher"): - # The IRC watcher is our only enabled component, so run its function - # only and don't worry about anything else: - logger.info("Starting IRC watcher") - irc_watcher() - - else: # Nothing is enabled! - logger.critical("No bot parts are enabled; stopping") - exit(1) diff --git a/earwigbot/runner.py b/earwigbot/runner.py deleted file mode 100644 index 2e03dfc..0000000 --- a/earwigbot/runner.py +++ /dev/null @@ -1,65 +0,0 @@ -#! /usr/bin/env python -# -*- coding: utf-8 -*- -# -# Copyright (C) 2009-2012 by Ben Kurtovic -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. - -""" -EarwigBot Runner - -This is a very simple script that can be run from anywhere. It will add the -'earwigbot' package to sys.path if it's not already in there (i.e., it hasn't -been "installed"), accept a root_dir (the directory in which bot.py is located) -and a decryption key from raw_input (if passwords are encrypted), then call -config.load() and decrypt any passwords, and finally call the main() function -of earwigbot.main. -""" - -from os import path -import sys - -def run(): - pkg_dir = path.split(path.dirname(path.abspath(__file__)))[0] - if pkg_dir not in sys.path: - sys.path.insert(0, pkg_dir) - - from earwigbot.config import config - from earwigbot import main - - root_dir = raw_input() - config_path = path.join(root_dir, "config.yml") - log_dir = path.join(root_dir, "logs") - is_encrypted = config.load(config_path, log_dir) - if is_encrypted: - config._decryption_key = raw_input() - config.decrypt(config.wiki, "password") - config.decrypt(config.wiki, "search", "credentials", "key") - config.decrypt(config.wiki, "search", "credentials", "secret") - config.decrypt(config.irc, "frontend", "nickservPassword") - config.decrypt(config.irc, "watcher", "nickservPassword") - - try: - main.main() - except KeyboardInterrupt: - main.logger.critical("KeyboardInterrupt: stopping main bot loop") - exit(1) - -if __name__ == "__main__": - run() diff --git a/earwigbot/tasks/__init__.py b/earwigbot/tasks/__init__.py index 4af79af..7e57b30 100644 --- a/earwigbot/tasks/__init__.py +++ b/earwigbot/tasks/__init__.py @@ -213,10 +213,6 @@ class _TaskManager(object): task_thread = threading.Thread(target=func) start_time = time.strftime("%b %d %H:%M:%S") task_thread.name = "{0} ({1})".format(task_name, start_time) - - # Stop bot task threads automagically if the main bot stops: - task_thread.daemon = True - task_thread.start() def get(self, task_name): diff --git a/earwigbot/util.py b/earwigbot/util.py new file mode 100755 index 0000000..6f87740 --- /dev/null +++ b/earwigbot/util.py @@ -0,0 +1,50 @@ +#! /usr/bin/env python +# -*- coding: utf-8 -*- +# +# Copyright (C) 2009-2012 by Ben Kurtovic +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +import argparse +from os import path + +from earwigbot import __version__ +from earwigbot.bot import Bot + +class BotUtility(object): + """ + DOCSTRING NEEDED + """ + + def version(self): + return __version__ + + def run(self): + print "EarwigBot v{0}\n".format(self.version()) + + def main(self): + root_dir = path.abspath(path.curdir()) + bot = Bot(root_dir) + bot.run() + + +main = BotUtility().main + +if __name__ == "__main__": + main() diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..41dc7d9 --- /dev/null +++ b/setup.py @@ -0,0 +1,40 @@ +#! /usr/bin/env python +# -*- coding: utf-8 -*- +# +# Copyright (C) 2009-2012 by Ben Kurtovic +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +""" +DOCSTRING NEEDED +""" + +from setuptools import setup + +setup( + name = "earwigbot", + entry_points = {"console_scripts": ["earwigbot = earwigbot.util:main"]}, + install_requires = ["PyYAML>=3.10", "oursql>=0.9.3", "oauth2>=1.5.211", + "numpy>=1.6.1", "matplotlib>=1.1.0"], + version = "0.1.dev", + author = "Ben Kurtovic", + author_email = "ben.kurtovic@verizon.net", + license = "MIT License", + url = "https://github.com/earwig/earwigbot", +) diff --git a/earwigbot/tests/__init__.py b/tests/__init__.py similarity index 98% rename from earwigbot/tests/__init__.py rename to tests/__init__.py index c3e6ac3..dfbc32c 100644 --- a/earwigbot/tests/__init__.py +++ b/tests/__init__.py @@ -23,7 +23,7 @@ """ EarwigBot's Unit Tests -This module __init__ file provides some support code for unit tests. +This package __init__ file provides some support code for unit tests. CommandTestCase is a subclass of unittest.TestCase that provides setUp() for creating a fake connection and some other helpful methods. It uses @@ -92,6 +92,7 @@ class CommandTestCase(TestCase): line = ":Foo!bar@example.com JOIN :#channel".strip().split() return self.maker(line, line[2][1:]) + class FakeConnection(IRCConnection): def __init__(self): pass diff --git a/earwigbot/tests/test_blowfish.py b/tests/test_blowfish.py similarity index 100% rename from earwigbot/tests/test_blowfish.py rename to tests/test_blowfish.py diff --git a/earwigbot/tests/test_calc.py b/tests/test_calc.py similarity index 100% rename from earwigbot/tests/test_calc.py rename to tests/test_calc.py diff --git a/earwigbot/tests/test_test.py b/tests/test_test.py similarity index 100% rename from earwigbot/tests/test_test.py rename to tests/test_test.py From abe58a07f6d4b4208608bc356ffc4fbeba17ac45 Mon Sep 17 00:00:00 2001 From: Ben Kurtovic Date: Fri, 6 Apr 2012 02:56:48 -0400 Subject: [PATCH 02/26] CommandManager as attr of Bot, plus cleanup --- earwigbot/bot.py | 22 ++++++++----- earwigbot/commands/__init__.py | 75 +++++++++++++++++++----------------------- earwigbot/irc/frontend.py | 8 ++--- earwigbot/irc/watcher.py | 8 +++-- earwigbot/util.py | 18 +++++++--- setup.py | 6 ++-- 6 files changed, 74 insertions(+), 63 deletions(-) diff --git a/earwigbot/bot.py b/earwigbot/bot.py index 65c5442..15c5fef 100644 --- a/earwigbot/bot.py +++ b/earwigbot/bot.py @@ -23,10 +23,13 @@ import threading from time import sleep, time +from earwigbot.commands import CommandManager from earwigbot.config import BotConfig from earwigbot.irc import Frontend, Watcher from earwigbot.tasks import task_manager +__all__ = ["Bot"] + class Bot(object): """ The Bot class is the core of EarwigBot, essentially responsible for @@ -46,16 +49,14 @@ class Bot(object): def __init__(self, root_dir): self.config = BotConfig(root_dir) self.logger = logging.getLogger("earwigbot") + self.commands = CommandManager(self) + self.tasks = None self.frontend = None self.watcher = None self._keep_scheduling = True self._lock = threading.Lock() - def _start_thread(self, name, target): - thread = threading.Thread(name=name, target=target) - thread.start() - def _wiki_scheduler(self): while self._keep_scheduling: time_start = time() @@ -68,17 +69,18 @@ class Bot(object): def _start_components(self): if self.config.components.get("irc_frontend"): self.logger.info("Starting IRC frontend") - self.frontend = Frontend(self.config) - self._start_thread(name, self.frontend.loop) + self.frontend = Frontend(self) + self.commands.load() + threading.Thread(name=name, target=self.frontend.loop).start() if self.config.components.get("irc_watcher"): self.logger.info("Starting IRC watcher") - self.watcher = Watcher(self.config, self.frontend) - self._start_thread(name, self.watcher.loop) + self.watcher = Watcher(self) + threading.Thread(name=name, target=self.watcher.loop).start() if self.config.components.get("wiki_scheduler"): self.logger.info("Starting wiki scheduler") - self._start_thread(name, self._wiki_scheduler) + threading.Thread(name=name, target=self._wiki_scheduler).start() def _loop(self): while 1: @@ -104,6 +106,7 @@ class Bot(object): with self._lock: self.config.load() #if self.config.components.get("irc_frontend"): + # self.commands.load() def stop(self): if self.frontend: @@ -111,3 +114,4 @@ class Bot(object): if self.watcher: self.watcher.stop() self._keep_scheduling = False + sleep(3) # Give a few seconds to finish closing IRC connections diff --git a/earwigbot/commands/__init__.py b/earwigbot/commands/__init__.py index d5543bb..824fe2b 100644 --- a/earwigbot/commands/__init__.py +++ b/earwigbot/commands/__init__.py @@ -25,17 +25,16 @@ EarwigBot's IRC Command Manager This package provides the IRC "commands" used by the bot's front-end component. This module contains the BaseCommand class (import with -`from earwigbot.commands import BaseCommand`) and an internal _CommandManager -class. This can be accessed through the `command_manager` singleton. +`from earwigbot.commands import BaseCommand`) and an internal CommandManager +class. This can be accessed through `bot.commands`. """ +import imp import logging -import os -import sys +from os import listdir, path +from re import sub -from earwigbot.config import config - -__all__ = ["BaseCommand", "command_manager"] +__all__ = ["BaseCommand", "CommandManager"] class BaseCommand(object): """A base class for commands on IRC. @@ -88,32 +87,33 @@ class BaseCommand(object): pass -class _CommandManager(object): - def __init__(self): +class CommandManager(object): + def __init__(self, bot): + self.bot = bot self.logger = logging.getLogger("earwigbot.tasks") - self._base_dir = os.path.dirname(os.path.abspath(__file__)) - self._connection = None + self._dirs = [path.dirname(__file__), bot.config.root_dir] self._commands = {} - def _load_command(self, filename): - """Load a specific command from a module, identified by filename. + def _load_command(self, name, path): + """Load a specific command from a module, identified by name and path. - Given a Connection object and a filename, we'll first try to import - it, and if that works, make an instance of the 'Command' class inside - (assuming it is an instance of BaseCommand), add it to self._commands, - and log the addition. Any problems along the way will either be - ignored or logged. + We'll first try to import it using imp magic, and if that works, make + an instance of the 'Command' class inside (assuming it is an instance + of BaseCommand), add it to self._commands, and log the addition. Any + problems along the way will either be ignored or logged. """ - # Strip .py from the filename's end and join with our package name: - name = ".".join(("commands", filename[:-3])) + f, path, desc = imp.find_module(name, [path]) try: - __import__(name) - except: - self.logger.exception("Couldn't load file {0}".format(filename)) + module = imp.load_module(name, f, path, desc) + except Exception: + e = "Couldn't load module {0} from {1}" + self.logger.exception(e.format(name, path)) return + finally: + f.close() try: - command = sys.modules[name].Command(self._connection) + command = module.Command(self.bot.frontend) except AttributeError: return # No command in this module if not isinstance(command, BaseCommand): @@ -122,20 +122,16 @@ class _CommandManager(object): self._commands[command.name] = command self.logger.debug("Added command {0}".format(command.name)) - def load(self, connection): - """Load all valid commands into self._commands. - - `connection` is a Connection object that is given to each command's - constructor. - """ - self._connection = connection - - files = os.listdir(self._base_dir) - files.sort() - for filename in files: - if filename.startswith("_") or not filename.endswith(".py"): - continue - self._load_command(filename) + def load(self): + """Load (or reload) all valid commands into self._commands.""" + dirs = [path.join(path.dirname(__file__), "commands"), + path.join(bot.config.root_dir, "commands")] + for dir in dirs: + files = listdir(dir) + files = [sub("\.pyc?$", "", f) for f in files if f[0] != "_"] + files = list(set(files)) # Remove duplicates + for filename in sorted(files): + self._load_command(filename, dir) msg = "Found {0} commands: {1}" commands = ", ".join(self._commands.keys()) @@ -158,6 +154,3 @@ class _CommandManager(object): e = "Error executing command '{0}'" self.logger.exception(e.format(data.command)) break - - -command_manager = _CommandManager() diff --git a/earwigbot/irc/frontend.py b/earwigbot/irc/frontend.py index 13c6bc3..83a0780 100644 --- a/earwigbot/irc/frontend.py +++ b/earwigbot/irc/frontend.py @@ -23,7 +23,6 @@ import logging import re -from earwigbot.commands import command_manager from earwigbot.irc import IRCConnection, Data, BrokenSocketException __all__ = ["Frontend"] @@ -40,14 +39,15 @@ class Frontend(IRCConnection): """ sender_regex = re.compile(":(.*?)!(.*?)@(.*?)\Z") - def __init__(self, config): - self.config = config + def __init__(self, bot): + self.bot = bot + self.config = bot.config self.logger = logging.getLogger("earwigbot.frontend") + cf = config.irc["frontend"] base = super(Frontend, self) base.__init__(cf["host"], cf["port"], cf["nick"], cf["ident"], cf["realname"], self.logger) - command_manager.load(self) self._connect() def _process_message(self, line): diff --git a/earwigbot/irc/watcher.py b/earwigbot/irc/watcher.py index f387b13..b29f4f8 100644 --- a/earwigbot/irc/watcher.py +++ b/earwigbot/irc/watcher.py @@ -38,14 +38,16 @@ class Watcher(IRCConnection): to channels on the IRC frontend. """ - def __init__(self, config, frontend=None): - self.config = config + def __init__(self, bot): + self.bot = bot + self.config = bot.config + self.frontend = bot.frontend self.logger = logging.getLogger("earwigbot.watcher") + cf = config.irc["watcher"] base = super(Watcher, self) base.__init__(cf["host"], cf["port"], cf["nick"], cf["ident"], cf["realname"], self.logger) - self.frontend = frontend self._prepare_process_hook() self._connect() diff --git a/earwigbot/util.py b/earwigbot/util.py index 6f87740..915747a 100755 --- a/earwigbot/util.py +++ b/earwigbot/util.py @@ -36,12 +36,22 @@ class BotUtility(object): return __version__ def run(self): - print "EarwigBot v{0}\n".format(self.version()) - - def main(self): root_dir = path.abspath(path.curdir()) bot = Bot(root_dir) - bot.run() + try: + bot.run() + finally: + bot.stop() + + def main(self): + print "EarwigBot v{0}\n".format(self.version()) + parser = argparse.ArgumentParser(description=BotUtility.__doc__) + + parser.add_argument("-V", "--version", action="version", + version=self.version()) + + args = parser.parse_args() +# args.func(args) main = BotUtility().main diff --git a/setup.py b/setup.py index 41dc7d9..348b593 100644 --- a/setup.py +++ b/setup.py @@ -30,8 +30,10 @@ from setuptools import setup setup( name = "earwigbot", entry_points = {"console_scripts": ["earwigbot = earwigbot.util:main"]}, - install_requires = ["PyYAML>=3.10", "oursql>=0.9.3", "oauth2>=1.5.211", - "numpy>=1.6.1", "matplotlib>=1.1.0"], + install_requires = ["PyYAML>=3.10", + "oursql>=0.9.3", + "oauth2>=1.5.211", + "matplotlib>=1.1.0"], version = "0.1.dev", author = "Ben Kurtovic", author_email = "ben.kurtovic@verizon.net", From d901a252bb7ca0f20ffe3134e4395b2c9b47f979 Mon Sep 17 00:00:00 2001 From: Ben Kurtovic Date: Fri, 6 Apr 2012 14:41:27 -0400 Subject: [PATCH 03/26] More cleanup for IRC stuff --- earwigbot/bot.py | 69 ++++++++++++++++++++----------------- earwigbot/commands/__init__.py | 77 ++++++++++++++++++++++-------------------- earwigbot/commands/restart.py | 13 +++++-- earwigbot/irc/connection.py | 10 +++--- earwigbot/irc/frontend.py | 35 +++++++++---------- earwigbot/irc/watcher.py | 32 +++++++++--------- 6 files changed, 125 insertions(+), 111 deletions(-) diff --git a/earwigbot/bot.py b/earwigbot/bot.py index 15c5fef..c6da54b 100644 --- a/earwigbot/bot.py +++ b/earwigbot/bot.py @@ -54,23 +54,13 @@ class Bot(object): self.frontend = None self.watcher = None - self._keep_scheduling = True - self._lock = threading.Lock() - - def _wiki_scheduler(self): - while self._keep_scheduling: - time_start = time() - task_manager.schedule() - time_end = time() - time_diff = time_start - time_end - if time_diff < 60: # Sleep until the next minute - sleep(60 - time_diff) - - def _start_components(self): + self.component_lock = threading.Lock() + self._keep_looping = True + + def _start_irc_components(self): if self.config.components.get("irc_frontend"): self.logger.info("Starting IRC frontend") self.frontend = Frontend(self) - self.commands.load() threading.Thread(name=name, target=self.frontend.loop).start() if self.config.components.get("irc_watcher"): @@ -78,17 +68,35 @@ class Bot(object): self.watcher = Watcher(self) threading.Thread(name=name, target=self.watcher.loop).start() + def _start_wiki_scheduler(self): + def wiki_scheduler(): + while self._keep_looping: + time_start = time() + task_manager.schedule() + time_end = time() + time_diff = time_start - time_end + if time_diff < 60: # Sleep until the next minute + sleep(60 - time_diff) + if self.config.components.get("wiki_scheduler"): self.logger.info("Starting wiki scheduler") - threading.Thread(name=name, target=self._wiki_scheduler).start() + threading.Thread(name=name, target=wiki_scheduler).start() + + def _stop_irc_components(self): + if self.frontend: + self.frontend.stop() + if self.watcher: + self.watcher.stop() def _loop(self): - while 1: - with self._lock: + while self._keep_looping: + with self.component_lock: if self.frontend and self.frontend.is_stopped(): - self.frontend._connect() + self.frontend = Frontend(self) + threading.Thread(name=name, target=self.frontend.loop).start() if self.watcher and self.watcher.is_stopped(): - self.watcher._connect() + self.watcher = Watcher(self) + threading.Thread(name=name, target=self.watcher.loop).start() sleep(5) def run(self): @@ -97,21 +105,20 @@ class Bot(object): self.config.decrypt(config.wiki, "search", "credentials", "key") self.config.decrypt(config.wiki, "search", "credentials", "secret") self.config.decrypt(config.irc, "frontend", "nickservPassword") - self.config.decrypt(config.irc, "watcher", "nickservPassword") - self._start_components() + self.config.decrypt(config.irc, "watcher", "nickservPassword") + self.commands.load() + self._start_irc_components() + self._start_wiki_scheduler() self._loop() - def reload(self): - #components = self.config.components - with self._lock: + def restart(self): + with self.component_lock: + self._stop_irc_components() self.config.load() - #if self.config.components.get("irc_frontend"): - # self.commands.load() + self.commands.load() + self._start_irc_components() def stop(self): - if self.frontend: - self.frontend.stop() - if self.watcher: - self.watcher.stop() - self._keep_scheduling = False + self._stop_irc_components() + self._keep_looping = False sleep(3) # Give a few seconds to finish closing IRC connections diff --git a/earwigbot/commands/__init__.py b/earwigbot/commands/__init__.py index 824fe2b..da152cb 100644 --- a/earwigbot/commands/__init__.py +++ b/earwigbot/commands/__init__.py @@ -30,9 +30,9 @@ class. This can be accessed through `bot.commands`. """ import imp -import logging from os import listdir, path from re import sub +from threading import Lock __all__ = ["BaseCommand", "CommandManager"] @@ -49,21 +49,24 @@ class BaseCommand(object): # command subclass: hooks = ["msg"] - def __init__(self, connection): + def __init__(self, bot): """Constructor for new commands. This is called once when the command is loaded (from - commands._load_command()). `connection` is a Connection object, - allowing us to do self.connection.say(), self.connection.send(), etc, - from within a method. + commands._load_command()). `bot` is out base Bot object. Generally you + shouldn't need to override this; if you do, call + super(Command, self).__init__() first. """ - self.connection = connection - logger_name = ".".join(("earwigbot", "commands", self.name)) - self.logger = logging.getLogger(logger_name) - self.logger.setLevel(logging.DEBUG) + self.bot = bot + self.logger = bot.commands.getLogger(self.name) + + def _execute(self, data): + """Make a quick connection alias and then process() the message.""" + self.connection = self.bot.frontend + self.process(data) def check(self, data): - """Returns whether this command should be called in response to 'data'. + """Return whether this command should be called in response to 'data'. Given a Data() instance, return True if we should respond to this activity, or False if we should ignore it or it doesn't apply to us. @@ -72,17 +75,16 @@ class BaseCommand(object): return False. This is the default behavior of check(); you need only override it if you wish to change that. """ - if data.is_command and data.command == self.name: - return True - return False + return data.is_command and data.command == self.name def process(self, data): """Main entry point for doing a command. Handle an activity (usually a message) on IRC. At this point, thanks to self.check() which is called automatically by the command handler, - we know this is something we should respond to, so (usually) something - like 'if data.command != "command_name": return' is unnecessary. + we know this is something we should respond to, so something like + `if data.command != "command_name": return` is usually unnecessary. + Note that """ pass @@ -90,9 +92,9 @@ class BaseCommand(object): class CommandManager(object): def __init__(self, bot): self.bot = bot - self.logger = logging.getLogger("earwigbot.tasks") - self._dirs = [path.dirname(__file__), bot.config.root_dir] + self.logger = bot.logger.getLogger("commands") self._commands = {} + self._command_access_lock = Lock() def _load_command(self, name, path): """Load a specific command from a module, identified by name and path. @@ -113,7 +115,7 @@ class CommandManager(object): f.close() try: - command = module.Command(self.bot.frontend) + command = module.Command(self.bot) except AttributeError: return # No command in this module if not isinstance(command, BaseCommand): @@ -124,14 +126,16 @@ class CommandManager(object): def load(self): """Load (or reload) all valid commands into self._commands.""" - dirs = [path.join(path.dirname(__file__), "commands"), - path.join(bot.config.root_dir, "commands")] - for dir in dirs: - files = listdir(dir) - files = [sub("\.pyc?$", "", f) for f in files if f[0] != "_"] - files = list(set(files)) # Remove duplicates - for filename in sorted(files): - self._load_command(filename, dir) + self._commands = {} + with self._command_access_lock: + dirs = [path.join(path.dirname(__file__), "commands"), + path.join(bot.config.root_dir, "commands")] + for dir in dirs: + files = listdir(dir) + files = [sub("\.pyc?$", "", f) for f in files if f[0] != "_"] + files = list(set(files)) # Remove duplicates + for filename in sorted(files): + self._load_command(filename, dir) msg = "Found {0} commands: {1}" commands = ", ".join(self._commands.keys()) @@ -143,14 +147,13 @@ class CommandManager(object): def check(self, hook, data): """Given an IRC event, check if there's anything we can respond to.""" - # Parse command arguments into data.command and data.args: - data.parse_args() - for command in self._commands.values(): - if hook in command.hooks: - if command.check(data): - try: - command.process(data) - except Exception: - e = "Error executing command '{0}'" - self.logger.exception(e.format(data.command)) - break + with self._command_access_lock: + for command in self._commands.values(): + if hook in command.hooks: + if command.check(data): + try: + command._execute(data) + except Exception: + e = "Error executing command '{0}':" + self.logger.exception(e.format(data.command)) + break diff --git a/earwigbot/commands/restart.py b/earwigbot/commands/restart.py index 2b39e5e..4902551 100644 --- a/earwigbot/commands/restart.py +++ b/earwigbot/commands/restart.py @@ -27,11 +27,20 @@ class Command(BaseCommand): """Restart the bot. Only the owner can do this.""" name = "restart" + def check(self, data): + commands = ["restart", "reload"] + return data.is_command and data.command in commands + def process(self, data): if data.host not in config.irc["permissions"]["owners"]: msg = "you must be a bot owner to use this command." self.connection.reply(data, msg) return - self.connection.logger.info("Restarting bot per owner request") - self.connection.stop() + if data.command == "restart": + self.connection.logger.info("Restarting bot per owner request") + self.connection.bot.restart() + + elif data.command == "reload": + self.connection.bot.commands.load() + self.connection.logger.info("IRC commands reloaded") diff --git a/earwigbot/irc/connection.py b/earwigbot/irc/connection.py index cd99421..7e2bb19 100644 --- a/earwigbot/irc/connection.py +++ b/earwigbot/irc/connection.py @@ -21,7 +21,7 @@ # SOFTWARE. import socket -import threading +from threading import Lock from time import sleep __all__ = ["BrokenSocketException", "IRCConnection"] @@ -36,17 +36,16 @@ class BrokenSocketException(Exception): class IRCConnection(object): """A class to interface with IRC.""" - def __init__(self, host, port, nick, ident, realname, logger): + def __init__(self, host, port, nick, ident, realname): self.host = host self.port = port self.nick = nick self.ident = ident self.realname = realname - self.logger = logger self._is_running = False # A lock to prevent us from sending two messages at once: - self._lock = threading.Lock() + self._send_lock = Lock() def _connect(self): """Connect to our IRC server.""" @@ -78,8 +77,7 @@ class IRCConnection(object): def _send(self, msg): """Send data to the server.""" - # Ensure that we only send one message at a time with a blocking lock: - with self._lock: + with self._send_lock: self._sock.sendall(msg + "\r\n") self.logger.debug(msg) diff --git a/earwigbot/irc/frontend.py b/earwigbot/irc/frontend.py index 83a0780..88fddbd 100644 --- a/earwigbot/irc/frontend.py +++ b/earwigbot/irc/frontend.py @@ -20,10 +20,9 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. -import logging import re -from earwigbot.irc import IRCConnection, Data, BrokenSocketException +from earwigbot.irc import IRCConnection, Data __all__ = ["Frontend"] @@ -41,13 +40,12 @@ class Frontend(IRCConnection): def __init__(self, bot): self.bot = bot - self.config = bot.config - self.logger = logging.getLogger("earwigbot.frontend") + self.logger = bot.logger.getLogger("frontend") - cf = config.irc["frontend"] + cf = bot.config.irc["frontend"] base = super(Frontend, self) base.__init__(cf["host"], cf["port"], cf["nick"], cf["ident"], - cf["realname"], self.logger) + cf["realname"]) self._connect() def _process_message(self, line): @@ -58,36 +56,35 @@ class Frontend(IRCConnection): if line[1] == "JOIN": data.nick, data.ident, data.host = self.sender_regex.findall(line[0])[0] data.chan = line[2] - # Check for 'join' hooks in our commands: - command_manager.check("join", data) + data.parse_args() + self.bot.commands.check("join", data) elif line[1] == "PRIVMSG": data.nick, data.ident, data.host = self.sender_regex.findall(line[0])[0] data.msg = " ".join(line[3:])[1:] data.chan = line[2] + data.parse_args() - if data.chan == self.config.irc["frontend"]["nick"]: + if data.chan == self.bot.config.irc["frontend"]["nick"]: # This is a privmsg to us, so set 'chan' as the nick of the # sender, then check for private-only command hooks: data.chan = data.nick - command_manager.check("msg_private", data) + self.bot.commands.check("msg_private", data) else: # Check for public-only command hooks: - command_manager.check("msg_public", data) + self.bot.commands.check("msg_public", data) # Check for command hooks that apply to all messages: - command_manager.check("msg", data) + self.bot.commands.check("msg", data) - # If we are pinged, pong back: - elif line[0] == "PING": + elif line[0] == "PING": # If we are pinged, pong back self.pong(line[1]) - # On successful connection to the server: - elif line[1] == "376": + elif line[1] == "376": # On successful connection to the server # If we're supposed to auth to NickServ, do that: try: - username = self.config.irc["frontend"]["nickservUsername"] - password = self.config.irc["frontend"]["nickservPassword"] + username = self.bot.config.irc["frontend"]["nickservUsername"] + password = self.bot.config.irc["frontend"]["nickservPassword"] except KeyError: pass else: @@ -95,5 +92,5 @@ class Frontend(IRCConnection): self.say("NickServ", msg) # Join all of our startup channels: - for chan in self.config.irc["frontend"]["channels"]: + for chan in self.bot.config.irc["frontend"]["channels"]: self.join(chan) diff --git a/earwigbot/irc/watcher.py b/earwigbot/irc/watcher.py index b29f4f8..572a3aa 100644 --- a/earwigbot/irc/watcher.py +++ b/earwigbot/irc/watcher.py @@ -21,9 +21,8 @@ # SOFTWARE. import imp -import logging -from earwigbot.irc import IRCConnection, RC, BrokenSocketException +from earwigbot.irc import IRCConnection, RC __all__ = ["Watcher"] @@ -40,14 +39,12 @@ class Watcher(IRCConnection): def __init__(self, bot): self.bot = bot - self.config = bot.config - self.frontend = bot.frontend - self.logger = logging.getLogger("earwigbot.watcher") + self.logger = bot.logger.getLogger("watcher") - cf = config.irc["watcher"] + cf = bot.config.irc["watcher"] base = super(Watcher, self) base.__init__(cf["host"], cf["port"], cf["nick"], cf["ident"], - cf["realname"], self.logger) + cf["realname"]) self._prepare_process_hook() self._connect() @@ -60,7 +57,7 @@ class Watcher(IRCConnection): # Ignore messages originating from channels not in our list, to # prevent someone PMing us false data: - if chan not in self.config.irc["watcher"]["channels"]: + if chan not in self.bot.config.irc["watcher"]["channels"]: return msg = " ".join(line[3:])[1:] @@ -74,7 +71,7 @@ class Watcher(IRCConnection): # When we've finished starting up, join all watcher channels: elif line[1] == "376": - for chan in self.config.irc["watcher"]["channels"]: + for chan in self.bot.config.irc["watcher"]["channels"]: self.join(chan) def _prepare_process_hook(self): @@ -86,14 +83,15 @@ class Watcher(IRCConnection): # Set a default RC process hook that does nothing: self._process_hook = lambda rc: () try: - rules = self.config.data["rules"] + rules = self.bot.config.data["rules"] except KeyError: return module = imp.new_module("_rc_event_processing_rules") + path = self.bot.config.path try: - exec compile(rules, self.config.path, "exec") in module.__dict__ + exec compile(rules, path, "exec") in module.__dict__ except Exception: - e = "Could not compile config file's RC event rules" + e = "Could not compile config file's RC event rules:" self.logger.exception(e) return self._process_hook_module = module @@ -113,7 +111,9 @@ class Watcher(IRCConnection): our config. """ chans = self._process_hook(rc) - if chans and self.frontend: - pretty = rc.prettify() - for chan in chans: - self.frontend.say(chan, pretty) + with self.bot.component_lock: + frontend = self.bot.frontend + if chans and frontend and not frontend.is_stopped(): + pretty = rc.prettify() + for chan in chans: + frontend.say(chan, pretty) From 079b424eca6be7bf7c4eb2eb286f1907ce783b2c Mon Sep 17 00:00:00 2001 From: Ben Kurtovic Date: Fri, 6 Apr 2012 14:46:11 -0400 Subject: [PATCH 04/26] Fixing up !restart and !reload a bit --- earwigbot/commands/__init__.py | 2 +- earwigbot/commands/restart.py | 9 +++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/earwigbot/commands/__init__.py b/earwigbot/commands/__init__.py index da152cb..e97d2b9 100644 --- a/earwigbot/commands/__init__.py +++ b/earwigbot/commands/__init__.py @@ -126,8 +126,8 @@ class CommandManager(object): def load(self): """Load (or reload) all valid commands into self._commands.""" - self._commands = {} with self._command_access_lock: + self._commands.clear() dirs = [path.join(path.dirname(__file__), "commands"), path.join(bot.config.root_dir, "commands")] for dir in dirs: diff --git a/earwigbot/commands/restart.py b/earwigbot/commands/restart.py index 4902551..7456e4b 100644 --- a/earwigbot/commands/restart.py +++ b/earwigbot/commands/restart.py @@ -38,9 +38,10 @@ class Command(BaseCommand): return if data.command == "restart": - self.connection.logger.info("Restarting bot per owner request") - self.connection.bot.restart() + self.logger.info("Restarting bot per owner request") + self.bot.restart() elif data.command == "reload": - self.connection.bot.commands.load() - self.connection.logger.info("IRC commands reloaded") + self.logger.info("Reloading IRC commands") + self.bot.commands.load() + self.connection.reply("IRC commands reloaded.") From 2211acc81d2dd039f576fd0765ec48c79474c6e9 Mon Sep 17 00:00:00 2001 From: Ben Kurtovic Date: Fri, 6 Apr 2012 15:18:15 -0400 Subject: [PATCH 05/26] Update TaskManager implementation --- earwigbot/bot.py | 24 +++--- earwigbot/commands/__init__.py | 10 ++- earwigbot/tasks/__init__.py | 167 +++++++++++++++++++++----------------- earwigbot/tasks/afc_catdelink.py | 2 +- earwigbot/tasks/afc_copyvios.py | 2 +- earwigbot/tasks/afc_dailycats.py | 2 +- earwigbot/tasks/afc_history.py | 2 +- earwigbot/tasks/afc_statistics.py | 2 +- earwigbot/tasks/afc_undated.py | 2 +- earwigbot/tasks/blptag.py | 2 +- earwigbot/tasks/feed_dailycats.py | 2 +- earwigbot/tasks/wrongmime.py | 2 +- 12 files changed, 121 insertions(+), 98 deletions(-) diff --git a/earwigbot/bot.py b/earwigbot/bot.py index c6da54b..42f3b89 100644 --- a/earwigbot/bot.py +++ b/earwigbot/bot.py @@ -20,13 +20,13 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. -import threading +from threading import Lock, Thread from time import sleep, time from earwigbot.commands import CommandManager from earwigbot.config import BotConfig from earwigbot.irc import Frontend, Watcher -from earwigbot.tasks import task_manager +from earwigbot.tasks import TaskManager __all__ = ["Bot"] @@ -50,29 +50,29 @@ class Bot(object): self.config = BotConfig(root_dir) self.logger = logging.getLogger("earwigbot") self.commands = CommandManager(self) - self.tasks = None + self.tasks = TaskManager(self) self.frontend = None self.watcher = None - self.component_lock = threading.Lock() + self.component_lock = Lock() self._keep_looping = True def _start_irc_components(self): if self.config.components.get("irc_frontend"): self.logger.info("Starting IRC frontend") self.frontend = Frontend(self) - threading.Thread(name=name, target=self.frontend.loop).start() + Thread(name=name, target=self.frontend.loop).start() if self.config.components.get("irc_watcher"): self.logger.info("Starting IRC watcher") self.watcher = Watcher(self) - threading.Thread(name=name, target=self.watcher.loop).start() + Thread(name=name, target=self.watcher.loop).start() def _start_wiki_scheduler(self): def wiki_scheduler(): while self._keep_looping: time_start = time() - task_manager.schedule() + self.tasks.schedule() time_end = time() time_diff = time_start - time_end if time_diff < 60: # Sleep until the next minute @@ -80,7 +80,7 @@ class Bot(object): if self.config.components.get("wiki_scheduler"): self.logger.info("Starting wiki scheduler") - threading.Thread(name=name, target=wiki_scheduler).start() + Thread(name=name, target=wiki_scheduler).start() def _stop_irc_components(self): if self.frontend: @@ -93,10 +93,10 @@ class Bot(object): with self.component_lock: if self.frontend and self.frontend.is_stopped(): self.frontend = Frontend(self) - threading.Thread(name=name, target=self.frontend.loop).start() + Thread(name=name, target=self.frontend.loop).start() if self.watcher and self.watcher.is_stopped(): self.watcher = Watcher(self) - threading.Thread(name=name, target=self.watcher.loop).start() + Thread(name=name, target=self.watcher.loop).start() sleep(5) def run(self): @@ -106,7 +106,8 @@ class Bot(object): self.config.decrypt(config.wiki, "search", "credentials", "secret") self.config.decrypt(config.irc, "frontend", "nickservPassword") self.config.decrypt(config.irc, "watcher", "nickservPassword") - self.commands.load() + self.commands.load() + self.tasks.load() self._start_irc_components() self._start_wiki_scheduler() self._loop() @@ -116,6 +117,7 @@ class Bot(object): self._stop_irc_components() self.config.load() self.commands.load() + self.tasks.load() self._start_irc_components() def stop(self): diff --git a/earwigbot/commands/__init__.py b/earwigbot/commands/__init__.py index e97d2b9..a22fcf6 100644 --- a/earwigbot/commands/__init__.py +++ b/earwigbot/commands/__init__.py @@ -115,9 +115,15 @@ class CommandManager(object): f.close() try: - command = module.Command(self.bot) + command_class = module.Command except AttributeError: return # No command in this module + try: + command = command_class(self.bot) + except Exception: + e = "Error initializing Command() class in {0} (from {1})" + self.logger.exception(e.format(name, path)) + return if not isinstance(command, BaseCommand): return @@ -129,7 +135,7 @@ class CommandManager(object): with self._command_access_lock: self._commands.clear() dirs = [path.join(path.dirname(__file__), "commands"), - path.join(bot.config.root_dir, "commands")] + path.join(self.bot.config.root_dir, "commands")] for dir in dirs: files = listdir(dir) files = [sub("\.pyc?$", "", f) for f in files if f[0] != "_"] diff --git a/earwigbot/tasks/__init__.py b/earwigbot/tasks/__init__.py index 7e57b30..dcd963c 100644 --- a/earwigbot/tasks/__init__.py +++ b/earwigbot/tasks/__init__.py @@ -25,39 +25,40 @@ EarwigBot's Wiki Task Manager This package provides the wiki bot "tasks" EarwigBot runs. This module contains the BaseTask class (import with `from earwigbot.tasks import BaseTask`) and an -internal _TaskManager class. This can be accessed through the `task_manager` -singleton. +internal TaskManager class. This can be accessed through `bot.tasks`. """ -import logging -import os -import sys -import threading -import time +import imp +from os import listdir, path +from threading import Lock, Thread +from time import gmtime, strftime from earwigbot import wiki -from earwigbot.config import config -__all__ = ["BaseTask", "task_manager"] +__all__ = ["BaseTask", "TaskManager"] class BaseTask(object): """A base class for bot tasks that edit Wikipedia.""" name = None number = 0 - def __init__(self): + def __init__(self, bot): """Constructor for new tasks. This is called once immediately after the task class is loaded by - the task manager (in tasks._load_task()). + the task manager (in tasks._load_task()). Don't override this directly + (or if you do, remember super(Task, self).__init()) - use setup(). """ - pass + self.bot = bot + self.logger = bot.tasks.logger.getLogger(self.name) + self.setup() + + def setup(self): + """Hook called immediately after the task is loaded. - def _setup_logger(self): - """Set up a basic module-level logger.""" - logger_name = ".".join(("earwigbot", "tasks", self.name)) - self.logger = logging.getLogger(logger_name) - self.logger.setLevel(logging.DEBUG) + Does nothing by default; feel free to override. + """ + pass def run(self, **kwargs): """Main entry point to run a given task. @@ -83,7 +84,7 @@ class BaseTask(object): If the config value is not found, we just return the arg as-is. """ try: - summary = config.wiki["summary"] + summary = self.bot.config.wiki["summary"] except KeyError: return comment return summary.replace("$1", str(self.number)).replace("$2", comment) @@ -111,7 +112,7 @@ class BaseTask(object): site = wiki.get_site() try: - cfg = config.wiki["shutoff"] + cfg = self.bot.config.wiki["shutoff"] except KeyError: return False title = cfg.get("page", "User:$1/Shutoff/Task $2") @@ -130,91 +131,107 @@ class BaseTask(object): return True -class _TaskManager(object): - def __init__(self): - self.logger = logging.getLogger("earwigbot.commands") - self._base_dir = os.path.dirname(os.path.abspath(__file__)) +class TaskManager(object): + def __init__(self, bot): + self.bot = bot + self.logger = bot.logger.getLogger("tasks") self._tasks = {} + self._task_access_lock = Lock() + + def _wrapper(self, task, **kwargs): + """Wrapper for task classes: run the task and catch any errors.""" + try: + task.run(**kwargs) + except Exception: + msg = "Task '{0}' raised an exception and had to stop:" + self.logger.exception(msg.format(task.name)) + else: + msg = "Task '{0}' finished without error" + self.logger.info(msg.format(task.name)) + + def _load_task(self, name, path): + """Load a specific task from a module, identified by name and path. - def _load_task(self, filename): - """Load a specific task from a module, identified by file name.""" - # Strip .py from the filename's end and join with our package name: - name = ".".join(("tasks", filename[:-3])) + We'll first try to import it using imp magic, and if that works, make + an instance of the 'Task' class inside (assuming it is an instance of + BaseTask), add it to self._tasks, and log the addition. Any problems + along the way will either be ignored or logged. + """ + f, path, desc = imp.find_module(name, [path]) try: - __import__(name) - except: - self.logger.exception("Couldn't load file {0}:".format(filename)) + module = imp.load_module(name, f, path, desc) + except Exception: + e = "Couldn't load module {0} from {1}" + self.logger.exception(e.format(name, path)) return + finally: + f.close() try: - task = sys.modules[name].Task() + task_class = module.Task except AttributeError: return # No task in this module + try: + task = task_class(self.bot) + except Exception: + e = "Error initializing Task() class in {0} (from {1})" + self.logger.exception(e.format(name, path)) + return if not isinstance(task, BaseTask): return - task._setup_logger() self._tasks[task.name] = task self.logger.debug("Added task {0}".format(task.name)) - def _wrapper(self, task, **kwargs): - """Wrapper for task classes: run the task and catch any errors.""" - try: - task.run(**kwargs) - except: - msg = "Task '{0}' raised an exception and had to stop" - self.logger.exception(msg.format(task.name)) - else: - msg = "Task '{0}' finished without error" - self.logger.info(msg.format(task.name)) - def load(self): - """Load all valid tasks from tasks/ into self._tasks.""" - files = os.listdir(self._base_dir) - files.sort() - - for filename in files: - if filename.startswith("_") or not filename.endswith(".py"): - continue - self._load_task(filename) + """Load (or reload) all valid tasks into self._tasks.""" + with self._task_access_lock: + self._tasks.clear() + dirs = [path.join(path.dirname(__file__), "tasks"), + path.join(self.bot.config.root_dir, "tasks")] + for dir in dirs: + files = listdir(dir) + files = [sub("\.pyc?$", "", f) for f in files if f[0] != "_"] + files = list(set(files)) # Remove duplicates + for filename in sorted(files): + self._load_task(filename) msg = "Found {0} tasks: {1}" tasks = ', '.join(self._tasks.keys()) self.logger.info(msg.format(len(self._tasks), tasks)) + def start(self, task_name, **kwargs): + """Start a given task in a new thread. kwargs are passed to task.run""" + msg = "Starting task '{0}' in a new thread" + self.logger.info(msg.format(task_name)) + + with self._task_access_lock: + try: + task = self._tasks[task_name] + except KeyError: + e = "Couldn't find task '{0}':" + self.logger.error(e.format(task_name)) + return + + task_thread = Thread(target=self._wrapper, args=(task,), kwargs=kwargs) + start_time = strftime("%b %d %H:%M:%S") + task_thread.name = "{0} ({1})".format(task_name, start_time) + task_thread.start() + def schedule(self, now=None): """Start all tasks that are supposed to be run at a given time.""" if not now: - now = time.gmtime() + now = gmtime() # Get list of tasks to run this turn: - tasks = config.schedule(now.tm_min, now.tm_hour, now.tm_mday, - now.tm_mon, now.tm_wday) + tasks = self.bot.config.schedule(now.tm_min, now.tm_hour, now.tm_mday, + now.tm_mon, now.tm_wday) for task in tasks: if isinstance(task, list): # They've specified kwargs, - self.start(task[0], **task[1]) # so pass those to start_task + self.start(task[0], **task[1]) # so pass those to start else: # Otherwise, just pass task_name self.start(task) - def start(self, task_name, **kwargs): - """Start a given task in a new thread. Pass args to the task's run() - function.""" - msg = "Starting task '{0}' in a new thread" - self.logger.info(msg.format(task_name)) - - try: - task = self._tasks[task_name] - except KeyError: - e = "Couldn't find task '{0}': bot/tasks/{0}.py does not exist" - self.logger.error(e.format(task_name)) - return - - func = lambda: self._wrapper(task, **kwargs) - task_thread = threading.Thread(target=func) - start_time = time.strftime("%b %d %H:%M:%S") - task_thread.name = "{0} ({1})".format(task_name, start_time) - task_thread.start() - def get(self, task_name): """Return the class instance associated with a certain task name. @@ -225,5 +242,3 @@ class _TaskManager(object): def get_all(self): """Return our dict of all loaded tasks.""" return self._tasks - -task_manager = _TaskManager() diff --git a/earwigbot/tasks/afc_catdelink.py b/earwigbot/tasks/afc_catdelink.py index c5d3c0f..7888e70 100644 --- a/earwigbot/tasks/afc_catdelink.py +++ b/earwigbot/tasks/afc_catdelink.py @@ -27,7 +27,7 @@ class Task(BaseTask): submissions.""" name = "afc_catdelink" - def __init__(self): + def setup(self): pass def run(self, **kwargs): diff --git a/earwigbot/tasks/afc_copyvios.py b/earwigbot/tasks/afc_copyvios.py index 4db5a63..0956b43 100644 --- a/earwigbot/tasks/afc_copyvios.py +++ b/earwigbot/tasks/afc_copyvios.py @@ -36,7 +36,7 @@ class Task(BaseTask): name = "afc_copyvios" number = 1 - def __init__(self): + def setup(self): cfg = config.tasks.get(self.name, {}) self.template = cfg.get("template", "AfC suspected copyvio") self.ignore_list = cfg.get("ignoreList", []) diff --git a/earwigbot/tasks/afc_dailycats.py b/earwigbot/tasks/afc_dailycats.py index efddd20..b286924 100644 --- a/earwigbot/tasks/afc_dailycats.py +++ b/earwigbot/tasks/afc_dailycats.py @@ -27,7 +27,7 @@ class Task(BaseTask): name = "afc_dailycats" number = 3 - def __init__(self): + def setup(self): pass def run(self, **kwargs): diff --git a/earwigbot/tasks/afc_history.py b/earwigbot/tasks/afc_history.py index 03117ad..9e146f9 100644 --- a/earwigbot/tasks/afc_history.py +++ b/earwigbot/tasks/afc_history.py @@ -57,7 +57,7 @@ class Task(BaseTask): """ name = "afc_history" - def __init__(self): + def setup(self): cfg = config.tasks.get(self.name, {}) self.num_days = cfg.get("days", 90) self.categories = cfg.get("categories", {}) diff --git a/earwigbot/tasks/afc_statistics.py b/earwigbot/tasks/afc_statistics.py index 3de023d..79070ae 100644 --- a/earwigbot/tasks/afc_statistics.py +++ b/earwigbot/tasks/afc_statistics.py @@ -53,7 +53,7 @@ class Task(BaseTask): name = "afc_statistics" number = 2 - def __init__(self): + def setup(self): self.cfg = cfg = config.tasks.get(self.name, {}) # Set some wiki-related attributes: diff --git a/earwigbot/tasks/afc_undated.py b/earwigbot/tasks/afc_undated.py index 512f09d..e596d1d 100644 --- a/earwigbot/tasks/afc_undated.py +++ b/earwigbot/tasks/afc_undated.py @@ -26,7 +26,7 @@ class Task(BaseTask): """A task to clear [[Category:Undated AfC submissions]].""" name = "afc_undated" - def __init__(self): + def setup(self): pass def run(self, **kwargs): diff --git a/earwigbot/tasks/blptag.py b/earwigbot/tasks/blptag.py index 5bb4052..505b7d9 100644 --- a/earwigbot/tasks/blptag.py +++ b/earwigbot/tasks/blptag.py @@ -27,7 +27,7 @@ class Task(BaseTask): {{WP Biography}}.""" name = "blptag" - def __init__(self): + def setup(self): pass def run(self, **kwargs): diff --git a/earwigbot/tasks/feed_dailycats.py b/earwigbot/tasks/feed_dailycats.py index 3d6afd7..a068af6 100644 --- a/earwigbot/tasks/feed_dailycats.py +++ b/earwigbot/tasks/feed_dailycats.py @@ -26,7 +26,7 @@ class Task(BaseTask): """A task to create daily categories for [[WP:FEED]].""" name = "feed_dailycats" - def __init__(self): + def setup(self): pass def run(self, **kwargs): diff --git a/earwigbot/tasks/wrongmime.py b/earwigbot/tasks/wrongmime.py index 1b51589..ed531d5 100644 --- a/earwigbot/tasks/wrongmime.py +++ b/earwigbot/tasks/wrongmime.py @@ -27,7 +27,7 @@ class Task(BaseTask): type.""" name = "wrongmime" - def __init__(self): + def setup(self): pass def run(self, **kwargs): From 8a7eb798a3a50a8a9aefbce163b60a1f902747b4 Mon Sep 17 00:00:00 2001 From: Ben Kurtovic Date: Fri, 6 Apr 2012 15:47:06 -0400 Subject: [PATCH 06/26] Fix all references to earwigbot.config.config; update SitesDBManager --- earwigbot/bot.py | 18 ++++++----- earwigbot/commands/__init__.py | 5 ++-- earwigbot/commands/afc_status.py | 3 +- earwigbot/commands/chanops.py | 3 +- earwigbot/commands/ctcp.py | 3 +- earwigbot/commands/git.py | 3 +- earwigbot/commands/restart.py | 3 +- earwigbot/commands/threads.py | 5 ++-- earwigbot/tasks/__init__.py | 1 + earwigbot/tasks/afc_copyvios.py | 3 +- earwigbot/tasks/afc_history.py | 3 +- earwigbot/tasks/afc_statistics.py | 3 +- earwigbot/wiki/__init__.py | 8 +++-- earwigbot/wiki/sitesdb.py | 63 +++++++++++---------------------------- 14 files changed, 48 insertions(+), 76 deletions(-) diff --git a/earwigbot/bot.py b/earwigbot/bot.py index 42f3b89..a005ac6 100644 --- a/earwigbot/bot.py +++ b/earwigbot/bot.py @@ -27,6 +27,7 @@ from earwigbot.commands import CommandManager from earwigbot.config import BotConfig from earwigbot.irc import Frontend, Watcher from earwigbot.tasks import TaskManager +from earwigbot.wiki import SitesDBManager __all__ = ["Bot"] @@ -51,6 +52,7 @@ class Bot(object): self.logger = logging.getLogger("earwigbot") self.commands = CommandManager(self) self.tasks = TaskManager(self) + self.wiki = SitesDBManager(self.config) self.frontend = None self.watcher = None @@ -100,12 +102,13 @@ class Bot(object): sleep(5) def run(self): - self.config.load() - self.config.decrypt(config.wiki, "password") - self.config.decrypt(config.wiki, "search", "credentials", "key") - self.config.decrypt(config.wiki, "search", "credentials", "secret") - self.config.decrypt(config.irc, "frontend", "nickservPassword") - self.config.decrypt(config.irc, "watcher", "nickservPassword") + config = self.config + config.load() + config.decrypt(config.wiki, "password") + config.decrypt(config.wiki, "search", "credentials", "key") + config.decrypt(config.wiki, "search", "credentials", "secret") + config.decrypt(config.irc, "frontend", "nickservPassword") + config.decrypt(config.irc, "watcher", "nickservPassword") self.commands.load() self.tasks.load() self._start_irc_components() @@ -121,6 +124,7 @@ class Bot(object): self._start_irc_components() def stop(self): - self._stop_irc_components() + with self.component_lock: + self._stop_irc_components() self._keep_looping = False sleep(3) # Give a few seconds to finish closing IRC connections diff --git a/earwigbot/commands/__init__.py b/earwigbot/commands/__init__.py index a22fcf6..ba09eac 100644 --- a/earwigbot/commands/__init__.py +++ b/earwigbot/commands/__init__.py @@ -58,9 +58,10 @@ class BaseCommand(object): super(Command, self).__init__() first. """ self.bot = bot + self.config = bot.config self.logger = bot.commands.getLogger(self.name) - def _execute(self, data): + def _wrap_process(self, data): """Make a quick connection alias and then process() the message.""" self.connection = self.bot.frontend self.process(data) @@ -158,7 +159,7 @@ class CommandManager(object): if hook in command.hooks: if command.check(data): try: - command._execute(data) + command._wrap_process(data) except Exception: e = "Error executing command '{0}':" self.logger.exception(e.format(data.command)) diff --git a/earwigbot/commands/afc_status.py b/earwigbot/commands/afc_status.py index a475caf..f03bf98 100644 --- a/earwigbot/commands/afc_status.py +++ b/earwigbot/commands/afc_status.py @@ -24,7 +24,6 @@ import re from earwigbot import wiki from earwigbot.commands import BaseCommand -from earwigbot.config import config class Command(BaseCommand): """Get the number of pending AfC submissions, open redirect requests, and @@ -39,7 +38,7 @@ class Command(BaseCommand): try: if data.line[1] == "JOIN" and data.chan == "#wikipedia-en-afc": - if data.nick != config.irc["frontend"]["nick"]: + if data.nick != self.config.irc["frontend"]["nick"]: return True except IndexError: pass diff --git a/earwigbot/commands/chanops.py b/earwigbot/commands/chanops.py index dd59353..1bec2d6 100644 --- a/earwigbot/commands/chanops.py +++ b/earwigbot/commands/chanops.py @@ -21,7 +21,6 @@ # SOFTWARE. from earwigbot.commands import BaseCommand -from earwigbot.config import config class Command(BaseCommand): """Voice, devoice, op, or deop users in the channel.""" @@ -39,7 +38,7 @@ class Command(BaseCommand): self.connection.reply(data, msg) return - if data.host not in config.irc["permissions"]["admins"]: + if data.host not in self.config.irc["permissions"]["admins"]: msg = "you must be a bot admin to use this command." self.connection.reply(data, msg) return diff --git a/earwigbot/commands/ctcp.py b/earwigbot/commands/ctcp.py index 80b56e2..6641d7e 100644 --- a/earwigbot/commands/ctcp.py +++ b/earwigbot/commands/ctcp.py @@ -25,7 +25,6 @@ import time from earwigbot import __version__ from earwigbot.commands import BaseCommand -from earwigbot.config import config class Command(BaseCommand): """Not an actual command, this module is used to respond to the CTCP @@ -63,7 +62,7 @@ class Command(BaseCommand): elif command == "VERSION": default = "EarwigBot - $1 - Python/$2 https://github.com/earwig/earwigbot" - vers = config.irc.get("version", default) + vers = self.config.irc.get("version", default) vers = vers.replace("$1", __version__) vers = vers.replace("$2", platform.python_version()) self.connection.notice(target, "\x01VERSION {0}\x01".format(vers)) diff --git a/earwigbot/commands/git.py b/earwigbot/commands/git.py index dfd9aba..c86ff70 100644 --- a/earwigbot/commands/git.py +++ b/earwigbot/commands/git.py @@ -25,7 +25,6 @@ import subprocess import re from earwigbot.commands import BaseCommand -from earwigbot.config import config class Command(BaseCommand): """Commands to interface with the bot's git repository; use '!git' for a @@ -34,7 +33,7 @@ class Command(BaseCommand): def process(self, data): self.data = data - if data.host not in config.irc["permissions"]["owners"]: + if data.host not in self.config.irc["permissions"]["owners"]: msg = "you must be a bot owner to use this command." self.connection.reply(data, msg) return diff --git a/earwigbot/commands/restart.py b/earwigbot/commands/restart.py index 7456e4b..8756fe1 100644 --- a/earwigbot/commands/restart.py +++ b/earwigbot/commands/restart.py @@ -21,7 +21,6 @@ # SOFTWARE. from earwigbot.commands import BaseCommand -from earwigbot.config import config class Command(BaseCommand): """Restart the bot. Only the owner can do this.""" @@ -32,7 +31,7 @@ class Command(BaseCommand): return data.is_command and data.command in commands def process(self, data): - if data.host not in config.irc["permissions"]["owners"]: + if data.host not in self.config.irc["permissions"]["owners"]: msg = "you must be a bot owner to use this command." self.connection.reply(data, msg) return diff --git a/earwigbot/commands/threads.py b/earwigbot/commands/threads.py index 7cf70ae..976eb71 100644 --- a/earwigbot/commands/threads.py +++ b/earwigbot/commands/threads.py @@ -24,7 +24,6 @@ import threading import re from earwigbot.commands import BaseCommand -from earwigbot.config import config from earwigbot.irc import KwargParseException from earwigbot.tasks import task_manager @@ -40,7 +39,7 @@ class Command(BaseCommand): def process(self, data): self.data = data - if data.host not in config.irc["permissions"]["owners"]: + if data.host not in self.config.irc["permissions"]["owners"]: msg = "you must be a bot owner to use this command." self.connection.reply(data, msg) return @@ -80,7 +79,7 @@ class Command(BaseCommand): if tname == "MainThread": t = "\x0302MainThread\x0301 (id {1})" normal_threads.append(t.format(thread.ident)) - elif tname in config.components: + elif tname in self.config.components: t = "\x0302{0}\x0301 (id {1})" normal_threads.append(t.format(tname, thread.ident)) elif tname.startswith("reminder"): diff --git a/earwigbot/tasks/__init__.py b/earwigbot/tasks/__init__.py index dcd963c..7c268f9 100644 --- a/earwigbot/tasks/__init__.py +++ b/earwigbot/tasks/__init__.py @@ -50,6 +50,7 @@ class BaseTask(object): (or if you do, remember super(Task, self).__init()) - use setup(). """ self.bot = bot + self.config = bot.config self.logger = bot.tasks.logger.getLogger(self.name) self.setup() diff --git a/earwigbot/tasks/afc_copyvios.py b/earwigbot/tasks/afc_copyvios.py index 0956b43..552654c 100644 --- a/earwigbot/tasks/afc_copyvios.py +++ b/earwigbot/tasks/afc_copyvios.py @@ -27,7 +27,6 @@ from threading import Lock import oursql from earwigbot import wiki -from earwigbot.config import config from earwigbot.tasks import BaseTask class Task(BaseTask): @@ -37,7 +36,7 @@ class Task(BaseTask): number = 1 def setup(self): - cfg = config.tasks.get(self.name, {}) + cfg = self.config.tasks.get(self.name, {}) self.template = cfg.get("template", "AfC suspected copyvio") self.ignore_list = cfg.get("ignoreList", []) self.min_confidence = cfg.get("minConfidence", 0.5) diff --git a/earwigbot/tasks/afc_history.py b/earwigbot/tasks/afc_history.py index 9e146f9..e59a5c0 100644 --- a/earwigbot/tasks/afc_history.py +++ b/earwigbot/tasks/afc_history.py @@ -32,7 +32,6 @@ from numpy import arange import oursql from earwigbot import wiki -from earwigbot.config import config from earwigbot.tasks import BaseTask # Valid submission statuses: @@ -58,7 +57,7 @@ class Task(BaseTask): name = "afc_history" def setup(self): - cfg = config.tasks.get(self.name, {}) + cfg = self.config.tasks.get(self.name, {}) self.num_days = cfg.get("days", 90) self.categories = cfg.get("categories", {}) diff --git a/earwigbot/tasks/afc_statistics.py b/earwigbot/tasks/afc_statistics.py index 79070ae..0fcbec4 100644 --- a/earwigbot/tasks/afc_statistics.py +++ b/earwigbot/tasks/afc_statistics.py @@ -30,7 +30,6 @@ from time import sleep import oursql from earwigbot import wiki -from earwigbot.config import config from earwigbot.tasks import BaseTask # Chart status number constants: @@ -54,7 +53,7 @@ class Task(BaseTask): number = 2 def setup(self): - self.cfg = cfg = config.tasks.get(self.name, {}) + self.cfg = cfg = self.config.tasks.get(self.name, {}) # Set some wiki-related attributes: self.pagename = cfg.get("page", "Template:AFC statistics") diff --git a/earwigbot/wiki/__init__.py b/earwigbot/wiki/__init__.py index e48be82..00b5dd4 100644 --- a/earwigbot/wiki/__init__.py +++ b/earwigbot/wiki/__init__.py @@ -27,7 +27,11 @@ This is a collection of classes and functions to read from and write to Wikipedia and other wiki sites. No connection whatsoever to python-wikitools written by Mr.Z-man, other than a similar purpose. We share no code. -Import the toolset with `from earwigbot import wiki`. +Import the toolset directly with `from earwigbot import wiki`. If using the +built-in integration with the rest of the bot, that's usually not necessary: +Bot() objects contain a `wiki` attribute containing a SitesDBManager object +tied to the sites.db file located in the same directory as config.yml. That +object has the principal methods get_site, add_site, and remove_site. """ import logging as _log @@ -40,5 +44,5 @@ from earwigbot.wiki.exceptions import * from earwigbot.wiki.category import Category from earwigbot.wiki.page import Page from earwigbot.wiki.site import Site -from earwigbot.wiki.sitesdb import get_site, add_site, remove_site +from earwigbot.wiki.sitesdb import SitesDBManager from earwigbot.wiki.user import User diff --git a/earwigbot/wiki/sitesdb.py b/earwigbot/wiki/sitesdb.py index 0bd5c76..a5c9fe7 100644 --- a/earwigbot/wiki/sitesdb.py +++ b/earwigbot/wiki/sitesdb.py @@ -29,11 +29,10 @@ import stat import sqlite3 as sqlite from earwigbot import __version__ -from earwigbot.config import config from earwigbot.wiki.exceptions import SiteNotFoundError from earwigbot.wiki.site import Site -__all__ = ["SitesDBManager", "get_site", "add_site", "remove_site"] +__all__ = ["SitesDBManager"] class SitesDBManager(object): """ @@ -47,31 +46,19 @@ class SitesDBManager(object): remove_site -- removes a site from the database, given its name There's usually no need to use this class directly. All public methods - here are available as earwigbot.wiki.get_site(), earwigbot.wiki.add_site(), - and earwigbot.wiki.remove_site(), which use a sites.db file located in the - same directory as our config.yml file. Lower-level access can be achieved + here are available as bot.wiki.get_site(), bot.wiki.add_site(), and + bot.wiki.remove_site(), which use a sites.db file located in the same + directory as our config.yml file. Lower-level access can be achieved by importing the manager class - (`from earwigbot.wiki.sitesdb import SitesDBManager`). + (`from earwigbot.wiki import SitesDBManager`). """ - def __init__(self, db_file): - """Set up the manager with an attribute for the sitesdb filename.""" + def __init__(self, config): + """Set up the manager with an attribute for the BotConfig object.""" + self.config = config + self._sitesdb = path.join(config.root_dir, "sitesdb") + self._cookie_file = path.join(config.root_dir, ".cookies") self._cookiejar = None - self._sitesdb = db_file - - def _load_config(self): - """Load the bot's config. - - Called by a config-requiring function, such as get_site(), when config - has not been loaded. This will usually happen only if we're running - code directly from Python's interpreter and not the bot itself, because - bot.py and earwigbot.runner will already call these functions. - """ - is_encrypted = config.load() - if is_encrypted: # Passwords in the config file are encrypted - key = getpass("Enter key to unencrypt bot passwords: ") - config._decryption_key = key - config.decrypt(config.wiki, "password") def _get_cookiejar(self): """Return a LWPCookieJar object loaded from our .cookies file. @@ -89,8 +76,7 @@ class SitesDBManager(object): if self._cookiejar: return self._cookiejar - cookie_file = path.join(config.root_dir, ".cookies") - self._cookiejar = LWPCookieJar(cookie_file) + self._cookiejar = LWPCookieJar(self._cookie_file) try: self._cookiejar.load() @@ -163,10 +149,12 @@ class SitesDBManager(object): This calls _load_site_from_sitesdb(), so SiteNotFoundError will be raised if the site is not in our sitesdb. """ + cookiejar = self._get_cookiejar() (name, project, lang, base_url, article_path, script_path, sql, namespaces) = self._load_site_from_sitesdb(name) + + config = self.config login = (config.wiki.get("username"), config.wiki.get("password")) - cookiejar = self._get_cookiejar() user_agent = config.wiki.get("userAgent") use_https = config.wiki.get("useHTTPS", False) assert_edit = config.wiki.get("assert") @@ -265,9 +253,6 @@ class SitesDBManager(object): cannot be found in the sitesdb, SiteNotFoundError will be raised. An empty sitesdb will be created if none is found. """ - if not config.is_loaded(): - self._load_config() - # Someone specified a project without a lang, or vice versa: if (project and not lang) or (not project and lang): e = "Keyword arguments 'lang' and 'project' must be specified together." @@ -276,7 +261,7 @@ class SitesDBManager(object): # No args given, so return our default site: if not name and not project and not lang: try: - default = config.wiki["defaultSite"] + default = self.config.wiki["defaultSite"] except KeyError: e = "Default site is not specified in config." raise SiteNotFoundError(e) @@ -322,17 +307,15 @@ class SitesDBManager(object): site info). Raises SiteNotFoundError if not enough information has been provided to identify the site (e.g. a project but not a lang). """ - if not config.is_loaded(): - self._load_config() - if not base_url: if not project or not lang: e = "Without a base_url, both a project and a lang must be given." raise SiteNotFoundError(e) base_url = "//{0}.{1}.org".format(lang, project) + cookiejar = self._get_cookiejar() + config = self.config login = (config.wiki.get("username"), config.wiki.get("password")) - cookiejar = self._get_cookiejar() user_agent = config.wiki.get("userAgent") use_https = config.wiki.get("useHTTPS", False) assert_edit = config.wiki.get("assert") @@ -358,9 +341,6 @@ class SitesDBManager(object): was given but not a language, or vice versa. Will create an empty sitesdb if none was found. """ - if not config.is_loaded(): - self._load_config() - # Someone specified a project without a lang, or vice versa: if (project and not lang) or (not project and lang): e = "Keyword arguments 'lang' and 'project' must be specified together." @@ -381,12 +361,3 @@ class SitesDBManager(object): return self._remove_site_from_sitesdb(name) return False - -_root = path.split(path.split(path.dirname(path.abspath(__file__)))[0])[0] -_dbfile = path.join(_root, "sites.db") -_manager = SitesDBManager(_dbfile) -del _root, _dbfile - -get_site = _manager.get_site -add_site = _manager.add_site -remove_site = _manager.remove_site From 535e4efab61c02f5e03464563968eed351f76c0d Mon Sep 17 00:00:00 2001 From: Ben Kurtovic Date: Fri, 6 Apr 2012 15:55:39 -0400 Subject: [PATCH 07/26] Update references to CommandManager and TaskManager --- earwigbot/commands/afc_report.py | 5 ++--- earwigbot/commands/help.py | 4 ++-- earwigbot/commands/threads.py | 7 +++---- 3 files changed, 7 insertions(+), 9 deletions(-) diff --git a/earwigbot/commands/afc_report.py b/earwigbot/commands/afc_report.py index c6a5840..75c2f64 100644 --- a/earwigbot/commands/afc_report.py +++ b/earwigbot/commands/afc_report.py @@ -24,7 +24,6 @@ import re from earwigbot import wiki from earwigbot.commands import BaseCommand -from earwigbot.tasks import task_manager class Command(BaseCommand): """Get information about an AFC submission by name.""" @@ -36,9 +35,9 @@ class Command(BaseCommand): self.data = data try: - self.statistics = task_manager.get("afc_statistics") + self.statistics = self.bot.tasks.get("afc_statistics") except KeyError: - e = "Cannot run command: requires afc_statistics task." + e = "Cannot run command: requires afc_statistics task (from earwigbot_plugins)." self.logger.error(e) return diff --git a/earwigbot/commands/help.py b/earwigbot/commands/help.py index 5a6f9dd..bca6ee2 100644 --- a/earwigbot/commands/help.py +++ b/earwigbot/commands/help.py @@ -22,7 +22,7 @@ import re -from earwigbot.commands import BaseCommand, command_manager +from earwigbot.commands import BaseCommand from earwigbot.irc import Data class Command(BaseCommand): @@ -30,7 +30,7 @@ class Command(BaseCommand): name = "help" def process(self, data): - self.cmnds = command_manager.get_all() + self.cmnds = self.bot.commands.get_all() if not data.args: self.do_main_help(data) else: diff --git a/earwigbot/commands/threads.py b/earwigbot/commands/threads.py index 976eb71..13d0a70 100644 --- a/earwigbot/commands/threads.py +++ b/earwigbot/commands/threads.py @@ -25,7 +25,6 @@ import re from earwigbot.commands import BaseCommand from earwigbot.irc import KwargParseException -from earwigbot.tasks import task_manager class Command(BaseCommand): """Manage wiki tasks from IRC, and check on thread status.""" @@ -104,7 +103,7 @@ class Command(BaseCommand): def do_listall(self): """With !tasks listall or !tasks all, list all loaded tasks, and report whether they are currently running or idle.""" - all_tasks = task_manager.get_all().keys() + all_tasks = self.bot.tasks.get_all().keys() threads = threading.enumerate() tasklist = [] @@ -145,13 +144,13 @@ class Command(BaseCommand): self.connection.reply(data, msg) return - if task_name not in task_manager.get_all().keys(): + if task_name not in self.bot.tasks.get_all().keys(): # This task does not exist or hasn't been loaded: msg = "task could not be found; either tasks/{0}.py doesn't exist, or it wasn't loaded correctly." self.connection.reply(data, msg.format(task_name)) return data.kwargs["fromIRC"] = True - task_manager.start(task_name, **data.kwargs) + self.bot.tasks.start(task_name, **data.kwargs) msg = "task \x0302{0}\x0301 started.".format(task_name) self.connection.reply(data, msg) From 60b52f6aeb623fcff077e93181c983d47ada74e1 Mon Sep 17 00:00:00 2001 From: Ben Kurtovic Date: Fri, 6 Apr 2012 16:12:28 -0400 Subject: [PATCH 08/26] __iter__ for Command+TaskManager instead of get_all() --- earwigbot/commands/__init__.py | 17 +++++++++++++---- earwigbot/commands/help.py | 8 ++++---- earwigbot/commands/threads.py | 10 +++------- earwigbot/tasks/__init__.py | 8 ++++---- 4 files changed, 24 insertions(+), 19 deletions(-) diff --git a/earwigbot/commands/__init__.py b/earwigbot/commands/__init__.py index ba09eac..4531abc 100644 --- a/earwigbot/commands/__init__.py +++ b/earwigbot/commands/__init__.py @@ -71,6 +71,8 @@ class BaseCommand(object): Given a Data() instance, return True if we should respond to this activity, or False if we should ignore it or it doesn't apply to us. + Be aware that since this is called for each message sent on IRC, it + should not be cheap to execute and unlikely to throw exceptions. Most commands return True if data.command == self.name, otherwise they return False. This is the default behavior of check(); you need only @@ -97,6 +99,10 @@ class CommandManager(object): self._commands = {} self._command_access_lock = Lock() + def __iter__(self): + for name in self._commands: + yield name + def _load_command(self, name, path): """Load a specific command from a module, identified by name and path. @@ -148,10 +154,6 @@ class CommandManager(object): commands = ", ".join(self._commands.keys()) self.logger.info(msg.format(len(self._commands), commands)) - def get_all(self): - """Return our dict of all loaded commands.""" - return self._commands - def check(self, hook, data): """Given an IRC event, check if there's anything we can respond to.""" with self._command_access_lock: @@ -164,3 +166,10 @@ class CommandManager(object): e = "Error executing command '{0}':" self.logger.exception(e.format(data.command)) break + + def get(self, command_name): + """Return the class instance associated with a certain command name. + + Will raise KeyError if the command is not found. + """ + return self._command[command_name] diff --git a/earwigbot/commands/help.py b/earwigbot/commands/help.py index bca6ee2..95d9e39 100644 --- a/earwigbot/commands/help.py +++ b/earwigbot/commands/help.py @@ -30,7 +30,6 @@ class Command(BaseCommand): name = "help" def process(self, data): - self.cmnds = self.bot.commands.get_all() if not data.args: self.do_main_help(data) else: @@ -39,7 +38,7 @@ class Command(BaseCommand): def do_main_help(self, data): """Give the user a general help message with a list of all commands.""" msg = "Hi, I'm a bot! I have {0} commands loaded: {1}. You can get help for any command with '!help '." - cmnds = sorted(self.cmnds.keys()) + cmnds = sorted(self.bot.commands) msg = msg.format(len(cmnds), ', '.join(cmnds)) self.connection.reply(data, msg) @@ -53,13 +52,14 @@ class Command(BaseCommand): dummy.command = command.lower() dummy.is_command = True - for cmnd in self.cmnds.values(): + for cmnd_name in self.bot.commands: + cmnd = self.bot.commands.get(cmnd_name) if not cmnd.check(dummy): continue if cmnd.__doc__: doc = cmnd.__doc__.replace("\n", "") doc = re.sub("\s\s+", " ", doc) - msg = "info for command \x0303{0}\x0301: \"{1}\"" + msg = "help for command \x0303{0}\x0301: \"{1}\"" self.connection.reply(data, msg.format(command, doc)) return break diff --git a/earwigbot/commands/threads.py b/earwigbot/commands/threads.py index 13d0a70..bfe894b 100644 --- a/earwigbot/commands/threads.py +++ b/earwigbot/commands/threads.py @@ -103,13 +103,9 @@ class Command(BaseCommand): def do_listall(self): """With !tasks listall or !tasks all, list all loaded tasks, and report whether they are currently running or idle.""" - all_tasks = self.bot.tasks.get_all().keys() threads = threading.enumerate() tasklist = [] - - all_tasks.sort() - - for task in all_tasks: + for task in sorted(self.bot.tasks): threadlist = [t for t in threads if t.name.startswith(task)] ids = [str(t.ident) for t in threadlist] if not ids: @@ -144,9 +140,9 @@ class Command(BaseCommand): self.connection.reply(data, msg) return - if task_name not in self.bot.tasks.get_all().keys(): + if task_name not in self.bot.tasks: # This task does not exist or hasn't been loaded: - msg = "task could not be found; either tasks/{0}.py doesn't exist, or it wasn't loaded correctly." + msg = "task could not be found; either it doesn't exist, or it wasn't loaded correctly." self.connection.reply(data, msg.format(task_name)) return diff --git a/earwigbot/tasks/__init__.py b/earwigbot/tasks/__init__.py index 7c268f9..a1f75f3 100644 --- a/earwigbot/tasks/__init__.py +++ b/earwigbot/tasks/__init__.py @@ -139,6 +139,10 @@ class TaskManager(object): self._tasks = {} self._task_access_lock = Lock() + def __iter__(self): + for name in self._tasks: + yield name + def _wrapper(self, task, **kwargs): """Wrapper for task classes: run the task and catch any errors.""" try: @@ -239,7 +243,3 @@ class TaskManager(object): Will raise KeyError if the task is not found. """ return self._tasks[task_name] - - def get_all(self): - """Return our dict of all loaded tasks.""" - return self._tasks From 0234d8ce63e18f1faa9d77f09ce2165d1c36b206 Mon Sep 17 00:00:00 2001 From: Ben Kurtovic Date: Fri, 6 Apr 2012 16:22:47 -0400 Subject: [PATCH 09/26] Docstring updates; watcher's process() now takes Bot object --- earwigbot/bot.py | 9 +++++++-- earwigbot/irc/watcher.py | 13 +++++++------ 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/earwigbot/bot.py b/earwigbot/bot.py index a005ac6..ac2c566 100644 --- a/earwigbot/bot.py +++ b/earwigbot/bot.py @@ -34,8 +34,7 @@ __all__ = ["Bot"] class Bot(object): """ The Bot class is the core of EarwigBot, essentially responsible for - starting the various bot components and making sure they are all happy. An - explanation of the different components follows: + starting the various bot components and making sure they are all happy. EarwigBot has three components that can run independently of each other: an IRC front-end, an IRC watcher, and a wiki scheduler. @@ -45,6 +44,12 @@ class Bot(object): edits. Users cannot interact with this part of the bot. * The wiki scheduler runs wiki-editing bot tasks in separate threads at user-defined times through a cron-like interface. + + The Bot() object is accessable from within commands and tasks as self.bot. + This is the primary way to access data from other components of the bot. + For example, our BotConfig object is accessable from bot.config, tasks + can be started with bot.tasks.start(), and sites can be loaded from the + wiki toolset with bot.wiki.get_site(). """ def __init__(self, root_dir): diff --git a/earwigbot/irc/watcher.py b/earwigbot/irc/watcher.py index 572a3aa..f306802 100644 --- a/earwigbot/irc/watcher.py +++ b/earwigbot/irc/watcher.py @@ -33,8 +33,8 @@ class Watcher(IRCConnection): The IRC watcher runs on a wiki recent-changes server and listens for edits. Users cannot interact with this part of the bot. When an event occurs, we run it through some rules stored in our config, which can result - in wiki bot tasks being started (located in tasks/) or messages being sent - to channels on the IRC frontend. + in wiki bot tasks being started or messages being sent to channels on the + IRC frontend. """ def __init__(self, bot): @@ -77,8 +77,9 @@ class Watcher(IRCConnection): def _prepare_process_hook(self): """Create our RC event process hook from information in config. - This will get put in the function self._process_hook, which takes an RC - object and returns a list of frontend channels to report this event to. + This will get put in the function self._process_hook, which takes the + Bot object and an RC object and returns a list of frontend channels to + report this event to. """ # Set a default RC process hook that does nothing: self._process_hook = lambda rc: () @@ -98,7 +99,7 @@ class Watcher(IRCConnection): try: self._process_hook = module.process except AttributeError: - e = "RC event rules compiled correctly, but no process(rc) function was found" + e = "RC event rules compiled correctly, but no process(bot, rc) function was found" self.logger.error(e) return @@ -110,7 +111,7 @@ class Watcher(IRCConnection): self._prepare_process_hook() from information in the "rules" section of our config. """ - chans = self._process_hook(rc) + chans = self._process_hook(self.bot, rc) with self.bot.component_lock: frontend = self.bot.frontend if chans and frontend and not frontend.is_stopped(): From 45a472c2efd49facceefbca2536a02c4f46e981d Mon Sep 17 00:00:00 2001 From: Ben Kurtovic Date: Fri, 6 Apr 2012 21:28:28 -0400 Subject: [PATCH 10/26] setup.py should be good now --- earwigbot/__init__.py | 4 +++- setup.py | 37 +++++++++++++++++++++++++++---------- 2 files changed, 30 insertions(+), 11 deletions(-) diff --git a/earwigbot/__init__.py b/earwigbot/__init__.py index a78194c..fb656f3 100644 --- a/earwigbot/__init__.py +++ b/earwigbot/__init__.py @@ -21,7 +21,9 @@ # SOFTWARE. """ -EarwigBot - http://earwig.github.com/earwig/earwigbot +EarwigBot is a Python robot that edits Wikipedia and interacts with people over +IRC. - http://earwig.github.com/earwig/earwigbot + See README.md for a basic overview, or the docs/ directory for details. """ diff --git a/setup.py b/setup.py index 348b593..a9eb2d1 100644 --- a/setup.py +++ b/setup.py @@ -21,22 +21,39 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. -""" -DOCSTRING NEEDED -""" - from setuptools import setup +from earwigbot import __version__ + +with open("README.rst") as fp: + long_docs = fp.read() + setup( name = "earwigbot", + packages = ["earwigbot"], entry_points = {"console_scripts": ["earwigbot = earwigbot.util:main"]}, - install_requires = ["PyYAML>=3.10", - "oursql>=0.9.3", - "oauth2>=1.5.211", - "matplotlib>=1.1.0"], - version = "0.1.dev", + install_requires = ["PyYAML >= 3.10", # Config parsing + "oursql >= 0.9.3", # Talking with MediaWiki databases + "oauth2 >= 1.5.211"], # Talking with Yahoo BOSS Search + test_suite = "tests", + version = __version__, author = "Ben Kurtovic", author_email = "ben.kurtovic@verizon.net", - license = "MIT License", url = "https://github.com/earwig/earwigbot", + description = "EarwigBot is a Python robot that edits Wikipedia and interacts with people over IRC.", + long_description = long_docs, + download_url = "https://github.com/earwig/earwigbot/tarball/{0}".format(__version__), + keywords = "earwig earwigbot irc wikipedia wiki mediawiki", + license = "MIT License", + classifiers = [ + "Development Status :: 3 - Alpha", + "Environment :: Console", + "Intended Audience :: Developers", + "License :: OSI Approved :: MIT License", + "Natural Language :: English", + "Operating System :: OS Independent", + "Programming Language :: Python :: 2.7", + "Topic :: Communications :: Chat :: Internet Relay Chat", + "Topic :: Internet :: WWW/HTTP" + ], ) From e47365350722db230cbeac0f9cf15cfc5fe5cd4a Mon Sep 17 00:00:00 2001 From: Ben Kurtovic Date: Sat, 7 Apr 2012 03:45:59 -0400 Subject: [PATCH 11/26] Updates to setup.py, unit tests, and logging --- .gitignore | 4 +-- earwigbot/commands/__init__.py | 4 +-- earwigbot/irc/frontend.py | 2 +- earwigbot/irc/watcher.py | 2 +- earwigbot/tasks/__init__.py | 4 +-- setup.py | 4 +-- tests/__init__.py | 61 +++++++++++++++++++++++++++++++++--------- tests/test_calc.py | 2 +- tests/test_test.py | 6 ++--- 9 files changed, 62 insertions(+), 27 deletions(-) diff --git a/.gitignore b/.gitignore index 91e9551..93e373c 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,3 @@ -# Ignore python bytecode: *.pyc - -# Ignore OS X's stuff: +*.egg-info .DS_Store diff --git a/earwigbot/commands/__init__.py b/earwigbot/commands/__init__.py index 4531abc..dfe2312 100644 --- a/earwigbot/commands/__init__.py +++ b/earwigbot/commands/__init__.py @@ -59,7 +59,7 @@ class BaseCommand(object): """ self.bot = bot self.config = bot.config - self.logger = bot.commands.getLogger(self.name) + self.logger = bot.commands.logger.getChild(self.name) def _wrap_process(self, data): """Make a quick connection alias and then process() the message.""" @@ -95,7 +95,7 @@ class BaseCommand(object): class CommandManager(object): def __init__(self, bot): self.bot = bot - self.logger = bot.logger.getLogger("commands") + self.logger = bot.logger.getChild("commands") self._commands = {} self._command_access_lock = Lock() diff --git a/earwigbot/irc/frontend.py b/earwigbot/irc/frontend.py index 88fddbd..2065ff1 100644 --- a/earwigbot/irc/frontend.py +++ b/earwigbot/irc/frontend.py @@ -40,7 +40,7 @@ class Frontend(IRCConnection): def __init__(self, bot): self.bot = bot - self.logger = bot.logger.getLogger("frontend") + self.logger = bot.logger.getChild("frontend") cf = bot.config.irc["frontend"] base = super(Frontend, self) diff --git a/earwigbot/irc/watcher.py b/earwigbot/irc/watcher.py index f306802..8a45dff 100644 --- a/earwigbot/irc/watcher.py +++ b/earwigbot/irc/watcher.py @@ -39,7 +39,7 @@ class Watcher(IRCConnection): def __init__(self, bot): self.bot = bot - self.logger = bot.logger.getLogger("watcher") + self.logger = bot.logger.getChild("watcher") cf = bot.config.irc["watcher"] base = super(Watcher, self) diff --git a/earwigbot/tasks/__init__.py b/earwigbot/tasks/__init__.py index a1f75f3..9a667be 100644 --- a/earwigbot/tasks/__init__.py +++ b/earwigbot/tasks/__init__.py @@ -51,7 +51,7 @@ class BaseTask(object): """ self.bot = bot self.config = bot.config - self.logger = bot.tasks.logger.getLogger(self.name) + self.logger = bot.tasks.logger.getChild(self.name) self.setup() def setup(self): @@ -135,7 +135,7 @@ class BaseTask(object): class TaskManager(object): def __init__(self, bot): self.bot = bot - self.logger = bot.logger.getLogger("tasks") + self.logger = bot.logger.getChild("tasks") self._tasks = {} self._task_access_lock = Lock() diff --git a/setup.py b/setup.py index a9eb2d1..315498b 100644 --- a/setup.py +++ b/setup.py @@ -21,7 +21,7 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. -from setuptools import setup +from setuptools import setup, find_packages from earwigbot import __version__ @@ -30,7 +30,7 @@ with open("README.rst") as fp: setup( name = "earwigbot", - packages = ["earwigbot"], + packages = find_packages(exclude=("tests",)), entry_points = {"console_scripts": ["earwigbot = earwigbot.util:main"]}, install_requires = ["PyYAML >= 3.10", # Config parsing "oursql >= 0.9.3", # Talking with MediaWiki databases diff --git a/tests/__init__.py b/tests/__init__.py index dfbc32c..2cfeb0d 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -23,26 +23,41 @@ """ EarwigBot's Unit Tests -This package __init__ file provides some support code for unit tests. +This __init__ file provides some support code for unit tests. + +Test cases: + -- CommandTestCase provides setUp() for creating a fake connection, plus + some other helpful methods for testing IRC commands. + +Fake objects: + -- FakeBot implements Bot, using the Fake* equivalents of all objects + whenever possible. + -- FakeBotConfig implements BotConfig with silent logging. + -- FakeIRCConnection implements IRCConnection, using an internal string + buffer for data instead of sending it over a socket. -CommandTestCase is a subclass of unittest.TestCase that provides setUp() for -creating a fake connection and some other helpful methods. It uses -FakeConnection, a subclass of classes.Connection, but with an internal string -instead of a socket for data. """ +import logging +from os import path import re +from threading import Lock from unittest import TestCase +from earwigbot.bot import Bot +from earwigbot.commands import CommandManager +from earwigbot.config import BotConfig from earwigbot.irc import IRCConnection, Data +from earwigbot.tasks import TaskManager +from earwigbot.wiki import SitesDBManager class CommandTestCase(TestCase): re_sender = re.compile(":(.*?)!(.*?)@(.*?)\Z") def setUp(self, command): - self.connection = FakeConnection() - self.connection._connect() - self.command = command(self.connection) + self.bot = FakeBot(path.dirname(__file__)) + self.command = command(self.bot) + self.command.connection = self.connection = self.bot.frontend def get_single(self): data = self.connection._get().split("\n") @@ -93,15 +108,37 @@ class CommandTestCase(TestCase): return self.maker(line, line[2][1:]) -class FakeConnection(IRCConnection): - def __init__(self): - pass +class FakeBot(Bot): + def __init__(self, root_dir): + self.config = FakeBotConfig(root_dir) + self.logger = logging.getLogger("earwigbot") + self.commands = CommandManager(self) + self.tasks = TaskManager(self) + self.wiki = SitesDBManager(self.config) + self.frontend = FakeIRCConnection(self) + self.watcher = FakeIRCConnection(self) + + self.component_lock = Lock() + self._keep_looping = True + + +class FakeBotConfig(BotConfig): + def _setup_logging(self): + logger = logging.getLogger("earwigbot") + logger.addHandler(logging.NullHandler()) + + +class FakeIRCConnection(IRCConnection): + def __init__(self, bot): + self.bot = bot + self._is_running = False + self._connect() def _connect(self): self._buffer = "" def _close(self): - pass + self._buffer = "" def _get(self, size=4096): data, self._buffer = self._buffer, "" diff --git a/tests/test_calc.py b/tests/test_calc.py index ae613f2..233a79a 100644 --- a/tests/test_calc.py +++ b/tests/test_calc.py @@ -23,7 +23,7 @@ import unittest from earwigbot.commands.calc import Command -from earwigbot.tests import CommandTestCase +from tests import CommandTestCase class TestCalc(CommandTestCase): diff --git a/tests/test_test.py b/tests/test_test.py index 7215613..f06fb2b 100644 --- a/tests/test_test.py +++ b/tests/test_test.py @@ -23,7 +23,7 @@ import unittest from earwigbot.commands.test import Command -from earwigbot.tests import CommandTestCase +from tests import CommandTestCase class TestTest(CommandTestCase): @@ -38,12 +38,12 @@ class TestTest(CommandTestCase): self.assertTrue(self.command.check(self.make_msg("TEST", "foo"))) def test_process(self): - def _test(): + def test(): self.command.process(self.make_msg("test")) self.assertSaidIn(["Hey \x02Foo\x0F!", "'sup \x02Foo\x0F?"]) for i in xrange(64): - _test() + test() if __name__ == "__main__": unittest.main(verbosity=2) From 54739d55531d4da6e376167a514aeb5c11fa65a9 Mon Sep 17 00:00:00 2001 From: Ben Kurtovic Date: Sat, 7 Apr 2012 12:17:49 -0400 Subject: [PATCH 12/26] Update references to Wiki Toolset, plus other fixes --- earwigbot/__init__.py | 2 +- earwigbot/commands/afc_report.py | 2 +- earwigbot/commands/afc_status.py | 3 +- earwigbot/commands/editcount.py | 2 +- earwigbot/commands/registration.py | 2 +- earwigbot/commands/rights.py | 2 +- earwigbot/config.py | 2 ++ earwigbot/tasks/__init__.py | 4 +-- earwigbot/tasks/afc_catdelink.py | 2 ++ earwigbot/tasks/afc_copyvios.py | 5 +-- earwigbot/tasks/afc_dailycats.py | 2 ++ earwigbot/tasks/afc_history.py | 38 +++++++++++---------- earwigbot/tasks/afc_statistics.py | 62 ++++++++++++++++++----------------- earwigbot/tasks/afc_undated.py | 2 ++ earwigbot/tasks/blptag.py | 2 ++ earwigbot/tasks/feed_dailycats.py | 2 ++ earwigbot/tasks/wikiproject_tagger.py | 35 ++++++++++++++++++++ earwigbot/tasks/wrongmime.py | 2 ++ earwigbot/util.py | 2 ++ earwigbot/wiki/__init__.py | 20 +++++------ earwigbot/wiki/category.py | 2 ++ earwigbot/wiki/constants.py | 5 ++- earwigbot/wiki/page.py | 2 ++ earwigbot/wiki/site.py | 2 ++ earwigbot/wiki/user.py | 2 ++ setup.py | 2 +- 26 files changed, 137 insertions(+), 71 deletions(-) create mode 100644 earwigbot/tasks/wikiproject_tagger.py diff --git a/earwigbot/__init__.py b/earwigbot/__init__.py index fb656f3..282f66b 100644 --- a/earwigbot/__init__.py +++ b/earwigbot/__init__.py @@ -28,7 +28,7 @@ See README.md for a basic overview, or the docs/ directory for details. """ __author__ = "Ben Kurtovic" -__copyright__ = "Copyright (C) 2009, 2010, 2011 by Ben Kurtovic" +__copyright__ = "Copyright (C) 2009, 2010, 2011, 2012 by Ben Kurtovic" __license__ = "MIT License" __version__ = "0.1.dev" __email__ = "ben.kurtovic@verizon.net" diff --git a/earwigbot/commands/afc_report.py b/earwigbot/commands/afc_report.py index 75c2f64..9803a33 100644 --- a/earwigbot/commands/afc_report.py +++ b/earwigbot/commands/afc_report.py @@ -30,7 +30,7 @@ class Command(BaseCommand): name = "report" def process(self, data): - self.site = wiki.get_site() + self.site = self.bot.wiki.get_site() self.site._maxlag = None self.data = data diff --git a/earwigbot/commands/afc_status.py b/earwigbot/commands/afc_status.py index f03bf98..08333a9 100644 --- a/earwigbot/commands/afc_status.py +++ b/earwigbot/commands/afc_status.py @@ -22,7 +22,6 @@ import re -from earwigbot import wiki from earwigbot.commands import BaseCommand class Command(BaseCommand): @@ -45,7 +44,7 @@ class Command(BaseCommand): return False def process(self, data): - self.site = wiki.get_site() + self.site = self.bot.wiki.get_site() self.site._maxlag = None if data.line[1] == "JOIN": diff --git a/earwigbot/commands/editcount.py b/earwigbot/commands/editcount.py index 9c58726..92341b0 100644 --- a/earwigbot/commands/editcount.py +++ b/earwigbot/commands/editcount.py @@ -41,7 +41,7 @@ class Command(BaseCommand): else: name = ' '.join(data.args) - site = wiki.get_site() + site = self.bot.wiki.get_site() site._maxlag = None user = site.get_user(name) diff --git a/earwigbot/commands/registration.py b/earwigbot/commands/registration.py index 55b762f..6db8775 100644 --- a/earwigbot/commands/registration.py +++ b/earwigbot/commands/registration.py @@ -41,7 +41,7 @@ class Command(BaseCommand): else: name = ' '.join(data.args) - site = wiki.get_site() + site = self.bot.wiki.get_site() site._maxlag = None user = site.get_user(name) diff --git a/earwigbot/commands/rights.py b/earwigbot/commands/rights.py index 1a9dd99..65d3975 100644 --- a/earwigbot/commands/rights.py +++ b/earwigbot/commands/rights.py @@ -39,7 +39,7 @@ class Command(BaseCommand): else: name = ' '.join(data.args) - site = wiki.get_site() + site = self.bot.wiki.get_site() site._maxlag = None user = site.get_user(name) diff --git a/earwigbot/config.py b/earwigbot/config.py index 4cf0721..f2830e0 100644 --- a/earwigbot/config.py +++ b/earwigbot/config.py @@ -29,6 +29,8 @@ import yaml from earwigbot import blowfish +__all__ = ["BotConfig"] + class BotConfig(object): """ EarwigBot's YAML Config File Manager diff --git a/earwigbot/tasks/__init__.py b/earwigbot/tasks/__init__.py index 9a667be..aa6e35e 100644 --- a/earwigbot/tasks/__init__.py +++ b/earwigbot/tasks/__init__.py @@ -110,10 +110,10 @@ class BaseTask(object): try: site = self.site except AttributeError: - site = wiki.get_site() + site = self.bot.wiki.get_site() try: - cfg = self.bot.config.wiki["shutoff"] + cfg = self.config.wiki["shutoff"] except KeyError: return False title = cfg.get("page", "User:$1/Shutoff/Task $2") diff --git a/earwigbot/tasks/afc_catdelink.py b/earwigbot/tasks/afc_catdelink.py index 7888e70..ba57b5e 100644 --- a/earwigbot/tasks/afc_catdelink.py +++ b/earwigbot/tasks/afc_catdelink.py @@ -22,6 +22,8 @@ from earwigbot.tasks import BaseTask +__all__ = ["Task"] + class Task(BaseTask): """A task to delink mainspace categories in declined [[WP:AFC]] submissions.""" diff --git a/earwigbot/tasks/afc_copyvios.py b/earwigbot/tasks/afc_copyvios.py index 552654c..2e651aa 100644 --- a/earwigbot/tasks/afc_copyvios.py +++ b/earwigbot/tasks/afc_copyvios.py @@ -26,9 +26,10 @@ from threading import Lock import oursql -from earwigbot import wiki from earwigbot.tasks import BaseTask +__all__ = ["Task"] + class Task(BaseTask): """A task to check newly-edited [[WP:AFC]] submissions for copyright violations.""" @@ -62,7 +63,7 @@ class Task(BaseTask): if self.shutoff_enabled(): return title = kwargs["page"] - page = wiki.get_site().get_page(title) + page = self.bot.wiki.get_site().get_page(title) with self.db_access_lock: self.conn = oursql.connect(**self.conn_data) self.process(page) diff --git a/earwigbot/tasks/afc_dailycats.py b/earwigbot/tasks/afc_dailycats.py index b286924..5ce4b49 100644 --- a/earwigbot/tasks/afc_dailycats.py +++ b/earwigbot/tasks/afc_dailycats.py @@ -22,6 +22,8 @@ from earwigbot.tasks import BaseTask +__all__ = ["Task"] + class Task(BaseTask): """ A task to create daily categories for [[WP:AFC]].""" name = "afc_dailycats" diff --git a/earwigbot/tasks/afc_history.py b/earwigbot/tasks/afc_history.py index e59a5c0..eb5f0b5 100644 --- a/earwigbot/tasks/afc_history.py +++ b/earwigbot/tasks/afc_history.py @@ -34,11 +34,7 @@ import oursql from earwigbot import wiki from earwigbot.tasks import BaseTask -# Valid submission statuses: -STATUS_NONE = 0 -STATUS_PEND = 1 -STATUS_DECLINE = 2 -STATUS_ACCEPT = 3 +__all__ = ["Task"] class Task(BaseTask): """A task to generate charts about AfC submissions over time. @@ -56,6 +52,12 @@ class Task(BaseTask): """ name = "afc_history" + # Valid submission statuses: + STATUS_NONE = 0 + STATUS_PEND = 1 + STATUS_DECLINE = 2 + STATUS_ACCEPT = 3 + def setup(self): cfg = self.config.tasks.get(self.name, {}) self.num_days = cfg.get("days", 90) @@ -72,7 +74,7 @@ class Task(BaseTask): self.db_access_lock = Lock() def run(self, **kwargs): - self.site = wiki.get_site() + self.site = self.bot.wiki.get_site() with self.db_access_lock: self.conn = oursql.connect(**self.conn_data) @@ -136,7 +138,7 @@ class Task(BaseTask): stored = cursor.fetchall() status = self.get_status(title, pageid) - if status == STATUS_NONE: + if status == self.STATUS_NONE: if stored: cursor.execute(q_delete, (pageid,)) continue @@ -154,14 +156,14 @@ class Task(BaseTask): ns = page.namespace() if ns == wiki.NS_FILE_TALK: # Ignore accepted FFU requests - return STATUS_NONE + return self.STATUS_NONE if ns == wiki.NS_TALK: new_page = page.toggle_talk() sleep(2) if new_page.is_redirect(): - return STATUS_NONE # Ignore accepted AFC/R requests - return STATUS_ACCEPT + return self.STATUS_NONE # Ignore accepted AFC/R requests + return self.STATUS_ACCEPT cats = self.categories sq = self.site.sql_query @@ -169,16 +171,16 @@ class Task(BaseTask): match = lambda cat: list(sq(query, (cat.replace(" ", "_"), pageid))) if match(cats["pending"]): - return STATUS_PEND + return self.STATUS_PEND elif match(cats["unsubmitted"]): - return STATUS_NONE + return self.STATUS_NONE elif match(cats["declined"]): - return STATUS_DECLINE - return STATUS_NONE + return self.STATUS_DECLINE + return self.STATUS_NONE def get_date_counts(self, date): query = "SELECT COUNT(*) FROM page WHERE page_date = ? AND page_status = ?" - statuses = [STATUS_PEND, STATUS_DECLINE, STATUS_ACCEPT] + statuses = [self.STATUS_PEND, self.STATUS_DECLINE, self.STATUS_ACCEPT] counts = {} with self.conn.cursor() as cursor: for status in statuses: @@ -192,9 +194,9 @@ class Task(BaseTask): plt.xlabel(self.graph.get("xaxis", "Date")) plt.ylabel(self.graph.get("yaxis", "Submissions")) - pends = [d[STATUS_PEND] for d in data.itervalues()] - declines = [d[STATUS_DECLINE] for d in data.itervalues()] - accepts = [d[STATUS_ACCEPT] for d in data.itervalues()] + pends = [d[self.STATUS_PEND] for d in data.itervalues()] + declines = [d[self.STATUS_DECLINE] for d in data.itervalues()] + accepts = [d[self.STATUS_ACCEPT] for d in data.itervalues()] pends_declines = [p + d for p, d in zip(pends, declines)] ind = arange(len(data)) xsize = self.graph.get("xsize", 1200) diff --git a/earwigbot/tasks/afc_statistics.py b/earwigbot/tasks/afc_statistics.py index 0fcbec4..6601243 100644 --- a/earwigbot/tasks/afc_statistics.py +++ b/earwigbot/tasks/afc_statistics.py @@ -32,14 +32,7 @@ import oursql from earwigbot import wiki from earwigbot.tasks import BaseTask -# Chart status number constants: -CHART_NONE = 0 -CHART_PEND = 1 -CHART_DRAFT = 2 -CHART_REVIEW = 3 -CHART_ACCEPT = 4 -CHART_DECLINE = 5 -CHART_MISPLACE = 6 +__all__ = ["Task"] class Task(BaseTask): """A task to generate statistics for WikiProject Articles for Creation. @@ -52,6 +45,15 @@ class Task(BaseTask): name = "afc_statistics" number = 2 + # Chart status number constants: + CHART_NONE = 0 + CHART_PEND = 1 + CHART_DRAFT = 2 + CHART_REVIEW = 3 + CHART_ACCEPT = 4 + CHART_DECLINE = 5 + CHART_MISPLACE = 6 + def setup(self): self.cfg = cfg = self.config.tasks.get(self.name, {}) @@ -82,7 +84,7 @@ class Task(BaseTask): (self.save()). We will additionally create an SQL connection with our local database. """ - self.site = wiki.get_site() + self.site = self.bot.wiki.get_site() with self.db_access_lock: self.conn = oursql.connect(**self.conn_data) @@ -285,7 +287,7 @@ class Task(BaseTask): query = """DELETE FROM page, row USING page JOIN row ON page_id = row_id WHERE row_chart IN (?, ?) AND ADDTIME(page_special_time, '36:00:00') < NOW()""" - cursor.execute(query, (CHART_ACCEPT, CHART_DECLINE)) + cursor.execute(query, (self.CHART_ACCEPT, self.CHART_DECLINE)) def update(self, **kwargs): """Update a page by name, regardless of whether anything has changed. @@ -332,7 +334,7 @@ class Task(BaseTask): namespace = self.site.get_page(title).namespace() status, chart = self.get_status_and_chart(content, namespace) - if chart == CHART_NONE: + if chart == self.CHART_NONE: msg = "Could not find a status for [[{0}]]".format(title) self.logger.warn(msg) return @@ -366,7 +368,7 @@ class Task(BaseTask): namespace = self.site.get_page(title).namespace() status, chart = self.get_status_and_chart(content, namespace) - if chart == CHART_NONE: + if chart == self.CHART_NONE: self.untrack_page(cursor, pageid) return @@ -498,23 +500,23 @@ class Task(BaseTask): statuses = self.get_statuses(content) if "R" in statuses: - status, chart = "r", CHART_REVIEW + status, chart = "r", self.CHART_REVIEW elif "H" in statuses: - status, chart = "p", CHART_DRAFT + status, chart = "p", self.CHART_DRAFT elif "P" in statuses: - status, chart = "p", CHART_PEND + status, chart = "p", self.CHART_PEND elif "T" in statuses: - status, chart = None, CHART_NONE + status, chart = None, self.CHART_NONE elif "D" in statuses: - status, chart = "d", CHART_DECLINE + status, chart = "d", self.CHART_DECLINE else: - status, chart = None, CHART_NONE + status, chart = None, self.CHART_NONE if namespace == wiki.NS_MAIN: if not statuses: - status, chart = "a", CHART_ACCEPT + status, chart = "a", self.CHART_ACCEPT else: - status, chart = None, CHART_MISPLACE + status, chart = None, self.CHART_MISPLACE return status, chart @@ -613,23 +615,23 @@ class Task(BaseTask): returned if we cannot determine when the page was "special"-ed, or if it was "special"-ed more than 250 edits ago. """ - if chart ==CHART_NONE: + if chart ==self.CHART_NONE: return None, None, None - elif chart == CHART_MISPLACE: + elif chart == self.CHART_MISPLACE: return self.get_create(pageid) - elif chart == CHART_ACCEPT: + elif chart == self.CHART_ACCEPT: search_for = None search_not = ["R", "H", "P", "T", "D"] - elif chart == CHART_DRAFT: + elif chart == self.CHART_DRAFT: search_for = "H" search_not = [] - elif chart == CHART_PEND: + elif chart == self.CHART_PEND: search_for = "P" search_not = [] - elif chart == CHART_REVIEW: + elif chart == self.CHART_REVIEW: search_for = "R" search_not = [] - elif chart == CHART_DECLINE: + elif chart == self.CHART_DECLINE: search_for = "D" search_not = ["R", "H", "P", "T"] @@ -683,12 +685,12 @@ class Task(BaseTask): """ notes = "" - ignored_charts = [CHART_NONE, CHART_ACCEPT, CHART_DECLINE] + ignored_charts = [self.CHART_NONE, self.CHART_ACCEPT, self.CHART_DECLINE] if chart in ignored_charts: return notes statuses = self.get_statuses(content) - if "D" in statuses and chart != CHART_MISPLACE: + if "D" in statuses and chart != self.CHART_MISPLACE: notes += "|nr=1" # Submission was resubmitted if len(content) < 500: @@ -705,7 +707,7 @@ class Task(BaseTask): if time_since_modify > max_time: notes += "|no=1" # Submission hasn't been touched in over 4 days - if chart in [CHART_PEND, CHART_DRAFT]: + if chart in [self.CHART_PEND, self.CHART_DRAFT]: submitter = self.site.get_user(s_user) try: if submitter.blockinfo(): diff --git a/earwigbot/tasks/afc_undated.py b/earwigbot/tasks/afc_undated.py index e596d1d..93144e8 100644 --- a/earwigbot/tasks/afc_undated.py +++ b/earwigbot/tasks/afc_undated.py @@ -22,6 +22,8 @@ from earwigbot.tasks import BaseTask +__all__ = ["Task"] + class Task(BaseTask): """A task to clear [[Category:Undated AfC submissions]].""" name = "afc_undated" diff --git a/earwigbot/tasks/blptag.py b/earwigbot/tasks/blptag.py index 505b7d9..76f80be 100644 --- a/earwigbot/tasks/blptag.py +++ b/earwigbot/tasks/blptag.py @@ -22,6 +22,8 @@ from earwigbot.tasks import BaseTask +__all__ = ["Task"] + class Task(BaseTask): """A task to add |blp=yes to {{WPB}} or {{WPBS}} when it is used along with {{WP Biography}}.""" diff --git a/earwigbot/tasks/feed_dailycats.py b/earwigbot/tasks/feed_dailycats.py index a068af6..361b16d 100644 --- a/earwigbot/tasks/feed_dailycats.py +++ b/earwigbot/tasks/feed_dailycats.py @@ -22,6 +22,8 @@ from earwigbot.tasks import BaseTask +__all__ = ["Task"] + class Task(BaseTask): """A task to create daily categories for [[WP:FEED]].""" name = "feed_dailycats" diff --git a/earwigbot/tasks/wikiproject_tagger.py b/earwigbot/tasks/wikiproject_tagger.py new file mode 100644 index 0000000..6b63d6d --- /dev/null +++ b/earwigbot/tasks/wikiproject_tagger.py @@ -0,0 +1,35 @@ +# -*- coding: utf-8 -*- +# +# Copyright (C) 2009-2012 by Ben Kurtovic +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +from earwigbot.tasks import BaseTask + +__all__ = ["Task"] + +class Task(BaseTask): + """A task to tag talk pages with WikiProject Banners.""" + name = "wikiproject_tagger" + + def setup(self): + pass + + def run(self, **kwargs): + pass diff --git a/earwigbot/tasks/wrongmime.py b/earwigbot/tasks/wrongmime.py index ed531d5..484fb7b 100644 --- a/earwigbot/tasks/wrongmime.py +++ b/earwigbot/tasks/wrongmime.py @@ -22,6 +22,8 @@ from earwigbot.tasks import BaseTask +__all__ = ["Task"] + class Task(BaseTask): """A task to tag files whose extensions do not agree with their MIME type.""" diff --git a/earwigbot/util.py b/earwigbot/util.py index 915747a..8b3ad5a 100755 --- a/earwigbot/util.py +++ b/earwigbot/util.py @@ -27,6 +27,8 @@ from os import path from earwigbot import __version__ from earwigbot.bot import Bot +__all__ = ["BotUtility", "main"] + class BotUtility(object): """ DOCSTRING NEEDED diff --git a/earwigbot/wiki/__init__.py b/earwigbot/wiki/__init__.py index 00b5dd4..f2f0e89 100644 --- a/earwigbot/wiki/__init__.py +++ b/earwigbot/wiki/__init__.py @@ -28,21 +28,21 @@ Wikipedia and other wiki sites. No connection whatsoever to python-wikitools written by Mr.Z-man, other than a similar purpose. We share no code. Import the toolset directly with `from earwigbot import wiki`. If using the -built-in integration with the rest of the bot, that's usually not necessary: -Bot() objects contain a `wiki` attribute containing a SitesDBManager object -tied to the sites.db file located in the same directory as config.yml. That -object has the principal methods get_site, add_site, and remove_site. +built-in integration with the rest of the bot, Bot() objects contain a `wiki` +attribute, which is a SitesDBManager object tied to the sites.db file located +in the same directory as config.yml. That object has the principal methods +get_site, add_site, and remove_site that should handle all of your Site (and +thus, Page, Category, and User) needs. """ import logging as _log logger = _log.getLogger("earwigbot.wiki") logger.addHandler(_log.NullHandler()) +from earwigbot.wiki.category import * from earwigbot.wiki.constants import * from earwigbot.wiki.exceptions import * - -from earwigbot.wiki.category import Category -from earwigbot.wiki.page import Page -from earwigbot.wiki.site import Site -from earwigbot.wiki.sitesdb import SitesDBManager -from earwigbot.wiki.user import User +from earwigbot.wiki.page import * +from earwigbot.wiki.site import * +from earwigbot.wiki.sitesdb import * +from earwigbot.wiki.user import * diff --git a/earwigbot/wiki/category.py b/earwigbot/wiki/category.py index c220674..67426f4 100644 --- a/earwigbot/wiki/category.py +++ b/earwigbot/wiki/category.py @@ -22,6 +22,8 @@ from earwigbot.wiki.page import Page +__all__ = ["Category"] + class Category(Page): """ EarwigBot's Wiki Toolset: Category Class diff --git a/earwigbot/wiki/constants.py b/earwigbot/wiki/constants.py index 22aef9c..8e818a6 100644 --- a/earwigbot/wiki/constants.py +++ b/earwigbot/wiki/constants.py @@ -27,13 +27,16 @@ This module defines some useful constants: * USER_AGENT - our default User Agent when making API queries * NS_* - default namespace IDs for easy lookup -Import with `from earwigbot.wiki import constants` or `from earwigbot.wiki.constants import *`. +Import directly with `from earwigbot.wiki import constants` or +`from earwigbot.wiki.constants import *`. These are also available from +earwigbot.wiki (e.g. `earwigbot.wiki.USER_AGENT`). """ # Default User Agent when making API queries: from earwigbot import __version__ as _v from platform import python_version as _p USER_AGENT = "EarwigBot/{0} (Python/{1}; https://github.com/earwig/earwigbot)".format(_v, _p()) +del _v, _p # Default namespace IDs: NS_MAIN = 0 diff --git a/earwigbot/wiki/page.py b/earwigbot/wiki/page.py index dfd5268..957878e 100644 --- a/earwigbot/wiki/page.py +++ b/earwigbot/wiki/page.py @@ -28,6 +28,8 @@ from urllib import quote from earwigbot.wiki.copyright import CopyrightMixin from earwigbot.wiki.exceptions import * +__all__ = ["Page"] + class Page(CopyrightMixin): """ EarwigBot's Wiki Toolset: Page Class diff --git a/earwigbot/wiki/site.py b/earwigbot/wiki/site.py index 5c0b1c7..f5a3aca 100644 --- a/earwigbot/wiki/site.py +++ b/earwigbot/wiki/site.py @@ -43,6 +43,8 @@ from earwigbot.wiki.exceptions import * from earwigbot.wiki.page import Page from earwigbot.wiki.user import User +__all__ = ["Site"] + class Site(object): """ EarwigBot's Wiki Toolset: Site Class diff --git a/earwigbot/wiki/user.py b/earwigbot/wiki/user.py index 880b804..2747e2d 100644 --- a/earwigbot/wiki/user.py +++ b/earwigbot/wiki/user.py @@ -26,6 +26,8 @@ from earwigbot.wiki.constants import * from earwigbot.wiki.exceptions import UserNotFoundError from earwigbot.wiki.page import Page +__all__ = ["User"] + class User(object): """ EarwigBot's Wiki Toolset: User Class diff --git a/setup.py b/setup.py index 315498b..67234d9 100644 --- a/setup.py +++ b/setup.py @@ -42,7 +42,7 @@ setup( url = "https://github.com/earwig/earwigbot", description = "EarwigBot is a Python robot that edits Wikipedia and interacts with people over IRC.", long_description = long_docs, - download_url = "https://github.com/earwig/earwigbot/tarball/{0}".format(__version__), + download_url = "https://github.com/earwig/earwigbot/tarball/v{0}".format(__version__), keywords = "earwig earwigbot irc wikipedia wiki mediawiki", license = "MIT License", classifiers = [ From 362db3d1c89e75294669253dc26ac527ad9d8b4d Mon Sep 17 00:00:00 2001 From: Ben Kurtovic Date: Sat, 7 Apr 2012 14:21:43 -0400 Subject: [PATCH 13/26] git version info, logging updates, utility --- earwigbot/__init__.py | 15 +++++++++++++++ earwigbot/bot.py | 5 +++++ earwigbot/config.py | 4 ++-- earwigbot/util.py | 28 ++++++++++++++++------------ setup.py | 4 +++- 5 files changed, 41 insertions(+), 15 deletions(-) diff --git a/earwigbot/__init__.py b/earwigbot/__init__.py index 282f66b..1ebde74 100644 --- a/earwigbot/__init__.py +++ b/earwigbot/__init__.py @@ -32,5 +32,20 @@ __copyright__ = "Copyright (C) 2009, 2010, 2011, 2012 by Ben Kurtovic" __license__ = "MIT License" __version__ = "0.1.dev" __email__ = "ben.kurtovic@verizon.net" +__release__ = False + +if not __release__: + def _add_git_commit_id_to_version(version): + from git import Repo + from os.path import split, dirname + path = split(dirname(__file__))[0] + commit_id = Repo(path).head.object.hexsha + return version + ".git+" + commit_id[:8] + try: + __version__ = _add_git_commit_id_to_version(__version__) + except Exception: + pass + finally: + del _add_git_commit_id_to_version from earwigbot import blowfish, bot, commands, config, irc, tasks, util, wiki diff --git a/earwigbot/bot.py b/earwigbot/bot.py index ac2c566..ea40dcd 100644 --- a/earwigbot/bot.py +++ b/earwigbot/bot.py @@ -20,6 +20,7 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. +import logging from threading import Lock, Thread from time import sleep, time @@ -99,9 +100,11 @@ class Bot(object): while self._keep_looping: with self.component_lock: if self.frontend and self.frontend.is_stopped(): + self.logger.warn("IRC frontend has stopped; restarting") self.frontend = Frontend(self) Thread(name=name, target=self.frontend.loop).start() if self.watcher and self.watcher.is_stopped(): + self.logger.warn("IRC watcher has stopped; restarting") self.watcher = Watcher(self) Thread(name=name, target=self.watcher.loop).start() sleep(5) @@ -121,6 +124,7 @@ class Bot(object): self._loop() def restart(self): + self.logger.info("Restarting bot per request from owner") with self.component_lock: self._stop_irc_components() self.config.load() @@ -129,6 +133,7 @@ class Bot(object): self._start_irc_components() def stop(self): + self.logger.info("Shutting down bot") with self.component_lock: self._stop_irc_components() self._keep_looping = False diff --git a/earwigbot/config.py b/earwigbot/config.py index f2830e0..fc73a08 100644 --- a/earwigbot/config.py +++ b/earwigbot/config.py @@ -208,8 +208,8 @@ class BotConfig(object): decrypted if they were decrypted beforehand. """ if not path.exists(self._config_path): - print "You haven't configured the bot yet!" - choice = raw_input("Would you like to do this now? [y/n] ") + print "Config file not found:", self._config_path + choice = raw_input("Would you like to create a config file now? [y/n] ") if choice.lower().startswith("y"): self._make_new() else: diff --git a/earwigbot/util.py b/earwigbot/util.py index 8b3ad5a..5f139da 100755 --- a/earwigbot/util.py +++ b/earwigbot/util.py @@ -31,29 +31,33 @@ __all__ = ["BotUtility", "main"] class BotUtility(object): """ - DOCSTRING NEEDED + This is a command-line utility for EarwigBot that enables you to easily + start the bot without writing generally unnecessary three-line bootstrap + scripts. It supports starting the bot from any directory, as well as + starting individual tasks instead of the entire bot. """ def version(self): - return __version__ + return "EarwigBot v{0}".format(__version__) - def run(self): - root_dir = path.abspath(path.curdir()) + def run(self, root_dir): bot = Bot(root_dir) - try: - bot.run() - finally: - bot.stop() + print self.version() + #try: + # bot.run() + #finally: + # bot.stop() def main(self): - print "EarwigBot v{0}\n".format(self.version()) parser = argparse.ArgumentParser(description=BotUtility.__doc__) - parser.add_argument("-V", "--version", action="version", + parser.add_argument("-v", "--version", action="version", version=self.version()) - + parser.add_argument("root_dir", metavar="path", nargs="?", default=path.curdir) args = parser.parse_args() -# args.func(args) + + root_dir = path.abspath(args.root_dir) + self.run(root_dir) main = BotUtility().main diff --git a/setup.py b/setup.py index 67234d9..490faa5 100644 --- a/setup.py +++ b/setup.py @@ -34,7 +34,9 @@ setup( entry_points = {"console_scripts": ["earwigbot = earwigbot.util:main"]}, install_requires = ["PyYAML >= 3.10", # Config parsing "oursql >= 0.9.3", # Talking with MediaWiki databases - "oauth2 >= 1.5.211"], # Talking with Yahoo BOSS Search + "oauth2 >= 1.5.211", # Talking with Yahoo BOSS Search + "GitPython >= 0.3.2.RC1", # Interfacing with git + ], test_suite = "tests", version = __version__, author = "Ben Kurtovic", From 03062e808b67e5e16c4c0c33e197468ff5678d3b Mon Sep 17 00:00:00 2001 From: Ben Kurtovic Date: Sat, 7 Apr 2012 17:41:33 -0400 Subject: [PATCH 14/26] Wrote the command-line utility, added logging levels, improved Bot organization - Fixed loading bugs in CommandLoader and TaskLoader --- earwigbot/bot.py | 21 +++++-------- earwigbot/commands/__init__.py | 31 ++++++++++++------- earwigbot/config.py | 31 ++++++++++++++----- earwigbot/tasks/__init__.py | 32 ++++++++++++------- earwigbot/util.py | 70 +++++++++++++++++++++++------------------- 5 files changed, 112 insertions(+), 73 deletions(-) diff --git a/earwigbot/bot.py b/earwigbot/bot.py index ea40dcd..8bf92bb 100644 --- a/earwigbot/bot.py +++ b/earwigbot/bot.py @@ -53,8 +53,8 @@ class Bot(object): wiki toolset with bot.wiki.get_site(). """ - def __init__(self, root_dir): - self.config = BotConfig(root_dir) + def __init__(self, root_dir, level=logging.INFO): + self.config = BotConfig(root_dir, level) self.logger = logging.getLogger("earwigbot") self.commands = CommandManager(self) self.tasks = TaskManager(self) @@ -65,6 +65,10 @@ class Bot(object): self.component_lock = Lock() self._keep_looping = True + self.config.load() + self.commands.load() + self.tasks.load() + def _start_irc_components(self): if self.config.components.get("irc_frontend"): self.logger.info("Starting IRC frontend") @@ -107,18 +111,10 @@ class Bot(object): self.logger.warn("IRC watcher has stopped; restarting") self.watcher = Watcher(self) Thread(name=name, target=self.watcher.loop).start() - sleep(5) + sleep(3) def run(self): - config = self.config - config.load() - config.decrypt(config.wiki, "password") - config.decrypt(config.wiki, "search", "credentials", "key") - config.decrypt(config.wiki, "search", "credentials", "secret") - config.decrypt(config.irc, "frontend", "nickservPassword") - config.decrypt(config.irc, "watcher", "nickservPassword") - self.commands.load() - self.tasks.load() + self.logger.info("Starting bot") self._start_irc_components() self._start_wiki_scheduler() self._loop() @@ -137,4 +133,3 @@ class Bot(object): with self.component_lock: self._stop_irc_components() self._keep_looping = False - sleep(3) # Give a few seconds to finish closing IRC connections diff --git a/earwigbot/commands/__init__.py b/earwigbot/commands/__init__.py index dfe2312..98cebec 100644 --- a/earwigbot/commands/__init__.py +++ b/earwigbot/commands/__init__.py @@ -135,22 +135,31 @@ class CommandManager(object): return self._commands[command.name] = command - self.logger.debug("Added command {0}".format(command.name)) + self.logger.debug("Loaded command {0}".format(command.name)) + + def _load_directory(self, dir): + """Load all valid commands in a given directory.""" + processed = [] + for name in listdir(dir): + if not name.endswith(".py") and not name.endswith(".pyc"): + continue + if name.startswith("_") or name.startswith("."): + continue + modname = sub("\.pyc?$", "", name) # Remove extension + if modname not in processed: + self._load_command(modname, dir) + processed.append(modname) def load(self): """Load (or reload) all valid commands into self._commands.""" with self._command_access_lock: self._commands.clear() - dirs = [path.join(path.dirname(__file__), "commands"), - path.join(self.bot.config.root_dir, "commands")] - for dir in dirs: - files = listdir(dir) - files = [sub("\.pyc?$", "", f) for f in files if f[0] != "_"] - files = list(set(files)) # Remove duplicates - for filename in sorted(files): - self._load_command(filename, dir) - - msg = "Found {0} commands: {1}" + builtin_dir = path.dirname(__file__) + plugins_dir = path.join(self.bot.config.root_dir, "commands") + self._load_directory(builtin_dir) # Built-in commands + self._load_directory(plugins_dir) # Custom commands, aka plugins + + msg = "Loaded {0} commands: {1}" commands = ", ".join(self._commands.keys()) self.logger.info(msg.format(len(self._commands), commands)) diff --git a/earwigbot/config.py b/earwigbot/config.py index fc73a08..962f8d3 100644 --- a/earwigbot/config.py +++ b/earwigbot/config.py @@ -59,8 +59,9 @@ class BotConfig(object): aren't encrypted """ - def __init__(self, root_dir): + def __init__(self, root_dir, level): self._root_dir = root_dir + self._logging_level = level self._config_path = path.join(self._root_dir, "config.yml") self._log_dir = path.join(self._root_dir, "logs") self._decryption_key = None @@ -74,7 +75,14 @@ class BotConfig(object): self._nodes = [self._components, self._wiki, self._tasks, self._irc, self._metadata] - self._decryptable_nodes = [] + + self._decryptable_nodes = [ # Default nodes to decrypt + (self._wiki, ("password")), + (self._wiki, ("search", "credentials", "key")), + (self._wiki, ("search", "credentials", "secret")), + (self._irc, ("frontend", "nickservPassword")), + (self._irc, ("watcher", "nickservPassword")), + ] def _load(self): """Load data from our JSON config file (config.yml) into self._data.""" @@ -119,10 +127,10 @@ class BotConfig(object): h.setFormatter(formatter) logger.addHandler(h) - stream_handler = logging.StreamHandler() - stream_handler.setLevel(logging.DEBUG) - stream_handler.setFormatter(color_formatter) - logger.addHandler(stream_handler) + self._stream_handler = stream = logging.StreamHandler() + stream.setLevel(self._logging_level) + stream.setFormatter(color_formatter) + logger.addHandler(stream) def _decrypt(self, node, nodes): """Try to decrypt the contents of a config node. Use self.decrypt().""" @@ -148,6 +156,15 @@ class BotConfig(object): return self._root_dir @property + def logging_level(self): + return self._logging_level + + @logging_level.setter + def logging_level(self, level): + self._logging_level = level + self._stream_handler.setLevel(level) + + @property def path(self): return self._config_path @@ -213,7 +230,7 @@ class BotConfig(object): if choice.lower().startswith("y"): self._make_new() else: - exit(1) + exit(1) # TODO: raise an exception instead self._load() data = self._data diff --git a/earwigbot/tasks/__init__.py b/earwigbot/tasks/__init__.py index aa6e35e..d70bcde 100644 --- a/earwigbot/tasks/__init__.py +++ b/earwigbot/tasks/__init__.py @@ -30,6 +30,7 @@ internal TaskManager class. This can be accessed through `bot.tasks`. import imp from os import listdir, path +from re import sub from threading import Lock, Thread from time import gmtime, strftime @@ -186,22 +187,31 @@ class TaskManager(object): return self._tasks[task.name] = task - self.logger.debug("Added task {0}".format(task.name)) + self.logger.debug("Loaded task {0}".format(task.name)) + + def _load_directory(self, dir): + """Load all valid tasks in a given directory.""" + processed = [] + for name in listdir(dir): + if not name.endswith(".py") and not name.endswith(".pyc"): + continue + if name.startswith("_") or name.startswith("."): + continue + modname = sub("\.pyc?$", "", name) # Remove extension + if modname not in processed: + self._load_task(modname, dir) + processed.append(modname) def load(self): """Load (or reload) all valid tasks into self._tasks.""" with self._task_access_lock: self._tasks.clear() - dirs = [path.join(path.dirname(__file__), "tasks"), - path.join(self.bot.config.root_dir, "tasks")] - for dir in dirs: - files = listdir(dir) - files = [sub("\.pyc?$", "", f) for f in files if f[0] != "_"] - files = list(set(files)) # Remove duplicates - for filename in sorted(files): - self._load_task(filename) - - msg = "Found {0} tasks: {1}" + builtin_dir = path.dirname(__file__) + plugins_dir = path.join(self.bot.config.root_dir, "tasks") + self._load_directory(builtin_dir) # Built-in tasks + self._load_directory(plugins_dir) # Custom tasks, aka plugins + + msg = "Loaded {0} tasks: {1}" tasks = ', '.join(self._tasks.keys()) self.logger.info(msg.format(len(self._tasks), tasks)) diff --git a/earwigbot/util.py b/earwigbot/util.py index 5f139da..442bc6e 100755 --- a/earwigbot/util.py +++ b/earwigbot/util.py @@ -21,46 +21,54 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. +""" +This is EarwigBot's command-line utility, enabling you to easily start the +bot or run specific tasks. +""" + import argparse +import logging from os import path from earwigbot import __version__ from earwigbot.bot import Bot -__all__ = ["BotUtility", "main"] - -class BotUtility(object): - """ - This is a command-line utility for EarwigBot that enables you to easily - start the bot without writing generally unnecessary three-line bootstrap - scripts. It supports starting the bot from any directory, as well as - starting individual tasks instead of the entire bot. - """ - - def version(self): - return "EarwigBot v{0}".format(__version__) - - def run(self, root_dir): - bot = Bot(root_dir) - print self.version() - #try: - # bot.run() - #finally: - # bot.stop() - - def main(self): - parser = argparse.ArgumentParser(description=BotUtility.__doc__) - - parser.add_argument("-v", "--version", action="version", - version=self.version()) - parser.add_argument("root_dir", metavar="path", nargs="?", default=path.curdir) - args = parser.parse_args() +__all__ = ["main"] - root_dir = path.abspath(args.root_dir) - self.run(root_dir) +def main(): + version = "EarwigBot v{0}".format(__version__) + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("path", nargs="?", metavar="PATH", default=path.curdir, + help="path to the bot's working directory, which will be created if it doesn't exist; current directory assumed if not specified") + parser.add_argument("-v", "--version", action="version", version=version) + parser.add_argument("-d", "--debug", action="store_true", + help="print all logs, including DEBUG-level messages") + parser.add_argument("-q", "--quiet", action="store_true", + help="don't print any logs except warnings and errors") + parser.add_argument("-t", "--task", metavar="NAME", + help="given the name of a task, the bot will run it instead of the main bot and then exit") + args = parser.parse_args() + if args.debug and args.quiet: + parser.print_usage() + print "earwigbot: error: cannot show debug messages and be quiet at the same time" + return + level = logging.INFO + if args.debug: + level = logging.DEBUG + elif args.quiet: + level = logging.WARNING -main = BotUtility().main + print version + print + bot = Bot(path.abspath(args.path), level=level) + try: + if args.task: + bot.tasks.start(args.task) + else: + bot.run() + finally: + bot.stop() if __name__ == "__main__": main() From 0c6f627e430fc3f41a091e8ebe067b1ff2710caf Mon Sep 17 00:00:00 2001 From: Ben Kurtovic Date: Sat, 7 Apr 2012 17:58:38 -0400 Subject: [PATCH 15/26] Move Command+TaskManagers to a common earwigbot.managers module --- earwigbot/__init__.py | 3 +- earwigbot/bot.py | 13 +-- earwigbot/commands/__init__.py | 106 +---------------- earwigbot/managers.py | 250 +++++++++++++++++++++++++++++++++++++++++ earwigbot/tasks/__init__.py | 140 ++--------------------- earwigbot/wiki/__init__.py | 8 +- earwigbot/wiki/sitesdb.py | 7 +- 7 files changed, 278 insertions(+), 249 deletions(-) create mode 100644 earwigbot/managers.py diff --git a/earwigbot/__init__.py b/earwigbot/__init__.py index 1ebde74..39f0938 100644 --- a/earwigbot/__init__.py +++ b/earwigbot/__init__.py @@ -48,4 +48,5 @@ if not __release__: finally: del _add_git_commit_id_to_version -from earwigbot import blowfish, bot, commands, config, irc, tasks, util, wiki +from earwigbot import (blowfish, bot, commands, config, irc, managers, tasks, + util, wiki) diff --git a/earwigbot/bot.py b/earwigbot/bot.py index 8bf92bb..071cca2 100644 --- a/earwigbot/bot.py +++ b/earwigbot/bot.py @@ -24,11 +24,10 @@ import logging from threading import Lock, Thread from time import sleep, time -from earwigbot.commands import CommandManager from earwigbot.config import BotConfig from earwigbot.irc import Frontend, Watcher -from earwigbot.tasks import TaskManager -from earwigbot.wiki import SitesDBManager +from earwigbot.managers import CommandManager, TaskManager +from earwigbot.wiki import SitesDB __all__ = ["Bot"] @@ -58,7 +57,7 @@ class Bot(object): self.logger = logging.getLogger("earwigbot") self.commands = CommandManager(self) self.tasks = TaskManager(self) - self.wiki = SitesDBManager(self.config) + self.wiki = SitesDB(self.config) self.frontend = None self.watcher = None @@ -73,12 +72,12 @@ class Bot(object): if self.config.components.get("irc_frontend"): self.logger.info("Starting IRC frontend") self.frontend = Frontend(self) - Thread(name=name, target=self.frontend.loop).start() + Thread(name="irc_frontend", target=self.frontend.loop).start() if self.config.components.get("irc_watcher"): self.logger.info("Starting IRC watcher") self.watcher = Watcher(self) - Thread(name=name, target=self.watcher.loop).start() + Thread(name="irc_watcher", target=self.watcher.loop).start() def _start_wiki_scheduler(self): def wiki_scheduler(): @@ -92,7 +91,7 @@ class Bot(object): if self.config.components.get("wiki_scheduler"): self.logger.info("Starting wiki scheduler") - Thread(name=name, target=wiki_scheduler).start() + Thread(name="wiki_scheduler", target=wiki_scheduler).start() def _stop_irc_components(self): if self.frontend: diff --git a/earwigbot/commands/__init__.py b/earwigbot/commands/__init__.py index 98cebec..33604f1 100644 --- a/earwigbot/commands/__init__.py +++ b/earwigbot/commands/__init__.py @@ -21,20 +21,16 @@ # SOFTWARE. """ -EarwigBot's IRC Command Manager +EarwigBot's IRC Commands This package provides the IRC "commands" used by the bot's front-end component. This module contains the BaseCommand class (import with -`from earwigbot.commands import BaseCommand`) and an internal CommandManager -class. This can be accessed through `bot.commands`. +`from earwigbot.commands import BaseCommand`), whereas the package contains +various built-in commands. Additional commands can be installed as plugins in +the bot's working directory. """ -import imp -from os import listdir, path -from re import sub -from threading import Lock - -__all__ = ["BaseCommand", "CommandManager"] +__all__ = ["BaseCommand"] class BaseCommand(object): """A base class for commands on IRC. @@ -90,95 +86,3 @@ class BaseCommand(object): Note that """ pass - - -class CommandManager(object): - def __init__(self, bot): - self.bot = bot - self.logger = bot.logger.getChild("commands") - self._commands = {} - self._command_access_lock = Lock() - - def __iter__(self): - for name in self._commands: - yield name - - def _load_command(self, name, path): - """Load a specific command from a module, identified by name and path. - - We'll first try to import it using imp magic, and if that works, make - an instance of the 'Command' class inside (assuming it is an instance - of BaseCommand), add it to self._commands, and log the addition. Any - problems along the way will either be ignored or logged. - """ - f, path, desc = imp.find_module(name, [path]) - try: - module = imp.load_module(name, f, path, desc) - except Exception: - e = "Couldn't load module {0} from {1}" - self.logger.exception(e.format(name, path)) - return - finally: - f.close() - - try: - command_class = module.Command - except AttributeError: - return # No command in this module - try: - command = command_class(self.bot) - except Exception: - e = "Error initializing Command() class in {0} (from {1})" - self.logger.exception(e.format(name, path)) - return - if not isinstance(command, BaseCommand): - return - - self._commands[command.name] = command - self.logger.debug("Loaded command {0}".format(command.name)) - - def _load_directory(self, dir): - """Load all valid commands in a given directory.""" - processed = [] - for name in listdir(dir): - if not name.endswith(".py") and not name.endswith(".pyc"): - continue - if name.startswith("_") or name.startswith("."): - continue - modname = sub("\.pyc?$", "", name) # Remove extension - if modname not in processed: - self._load_command(modname, dir) - processed.append(modname) - - def load(self): - """Load (or reload) all valid commands into self._commands.""" - with self._command_access_lock: - self._commands.clear() - builtin_dir = path.dirname(__file__) - plugins_dir = path.join(self.bot.config.root_dir, "commands") - self._load_directory(builtin_dir) # Built-in commands - self._load_directory(plugins_dir) # Custom commands, aka plugins - - msg = "Loaded {0} commands: {1}" - commands = ", ".join(self._commands.keys()) - self.logger.info(msg.format(len(self._commands), commands)) - - def check(self, hook, data): - """Given an IRC event, check if there's anything we can respond to.""" - with self._command_access_lock: - for command in self._commands.values(): - if hook in command.hooks: - if command.check(data): - try: - command._wrap_process(data) - except Exception: - e = "Error executing command '{0}':" - self.logger.exception(e.format(data.command)) - break - - def get(self, command_name): - """Return the class instance associated with a certain command name. - - Will raise KeyError if the command is not found. - """ - return self._command[command_name] diff --git a/earwigbot/managers.py b/earwigbot/managers.py new file mode 100644 index 0000000..5df4e73 --- /dev/null +++ b/earwigbot/managers.py @@ -0,0 +1,250 @@ +#! /usr/bin/env python +# -*- coding: utf-8 -*- +# +# Copyright (C) 2009-2012 by Ben Kurtovic +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +import imp +from os import listdir, path +from re import sub +from threading import Lock, Thread +from time import gmtime, strftime + +from earwigbot.commands import BaseCommand +from earwigbot.tasks import BaseTask + +__all__ = ["CommandManager", "TaskManager"] + +class _BaseManager(object): + pass + + +class CommandManager(_BaseManager): + def __init__(self, bot): + self.bot = bot + self.logger = bot.logger.getChild("commands") + self._commands = {} + self._command_access_lock = Lock() + + def __iter__(self): + for name in self._commands: + yield name + + def _load_command(self, name, path): + """Load a specific command from a module, identified by name and path. + + We'll first try to import it using imp magic, and if that works, make + an instance of the 'Command' class inside (assuming it is an instance + of BaseCommand), add it to self._commands, and log the addition. Any + problems along the way will either be ignored or logged. + """ + f, path, desc = imp.find_module(name, [path]) + try: + module = imp.load_module(name, f, path, desc) + except Exception: + e = "Couldn't load module {0} from {1}" + self.logger.exception(e.format(name, path)) + return + finally: + f.close() + + try: + command_class = module.Command + except AttributeError: + return # No command in this module + try: + command = command_class(self.bot) + except Exception: + e = "Error initializing Command() class in {0} (from {1})" + self.logger.exception(e.format(name, path)) + return + if not isinstance(command, BaseCommand): + return + + self._commands[command.name] = command + self.logger.debug("Loaded command {0}".format(command.name)) + + def _load_directory(self, dir): + """Load all valid commands in a given directory.""" + processed = [] + for name in listdir(dir): + if not name.endswith(".py") and not name.endswith(".pyc"): + continue + if name.startswith("_") or name.startswith("."): + continue + modname = sub("\.pyc?$", "", name) # Remove extension + if modname not in processed: + self._load_command(modname, dir) + processed.append(modname) + + def load(self): + """Load (or reload) all valid commands into self._commands.""" + with self._command_access_lock: + self._commands.clear() + builtin_dir = path.join(path.dirname(__file__), "commands") + plugins_dir = path.join(self.bot.config.root_dir, "commands") + self._load_directory(builtin_dir) # Built-in commands + self._load_directory(plugins_dir) # Custom commands, aka plugins + + msg = "Loaded {0} commands: {1}" + commands = ", ".join(self._commands.keys()) + self.logger.info(msg.format(len(self._commands), commands)) + + def check(self, hook, data): + """Given an IRC event, check if there's anything we can respond to.""" + with self._command_access_lock: + for command in self._commands.values(): + if hook in command.hooks: + if command.check(data): + try: + command._wrap_process(data) + except Exception: + e = "Error executing command '{0}':" + self.logger.exception(e.format(data.command)) + break + + def get(self, command_name): + """Return the class instance associated with a certain command name. + + Will raise KeyError if the command is not found. + """ + return self._command[command_name] + + +class TaskManager(_BaseManager): + def __init__(self, bot): + self.bot = bot + self.logger = bot.logger.getChild("tasks") + self._tasks = {} + self._task_access_lock = Lock() + + def __iter__(self): + for name in self._tasks: + yield name + + def _wrapper(self, task, **kwargs): + """Wrapper for task classes: run the task and catch any errors.""" + try: + task.run(**kwargs) + except Exception: + msg = "Task '{0}' raised an exception and had to stop:" + self.logger.exception(msg.format(task.name)) + else: + msg = "Task '{0}' finished without error" + self.logger.info(msg.format(task.name)) + + def _load_task(self, name, path): + """Load a specific task from a module, identified by name and path. + + We'll first try to import it using imp magic, and if that works, make + an instance of the 'Task' class inside (assuming it is an instance of + BaseTask), add it to self._tasks, and log the addition. Any problems + along the way will either be ignored or logged. + """ + f, path, desc = imp.find_module(name, [path]) + try: + module = imp.load_module(name, f, path, desc) + except Exception: + e = "Couldn't load module {0} from {1}" + self.logger.exception(e.format(name, path)) + return + finally: + f.close() + + try: + task_class = module.Task + except AttributeError: + return # No task in this module + try: + task = task_class(self.bot) + except Exception: + e = "Error initializing Task() class in {0} (from {1})" + self.logger.exception(e.format(name, path)) + return + if not isinstance(task, BaseTask): + return + + self._tasks[task.name] = task + self.logger.debug("Loaded task {0}".format(task.name)) + + def _load_directory(self, dir): + """Load all valid tasks in a given directory.""" + processed = [] + for name in listdir(dir): + if not name.endswith(".py") and not name.endswith(".pyc"): + continue + if name.startswith("_") or name.startswith("."): + continue + modname = sub("\.pyc?$", "", name) # Remove extension + if modname not in processed: + self._load_task(modname, dir) + processed.append(modname) + + def load(self): + """Load (or reload) all valid tasks into self._tasks.""" + with self._task_access_lock: + self._tasks.clear() + builtin_dir = path.join(path.dirname(__file__), "tasks") + plugins_dir = path.join(self.bot.config.root_dir, "tasks") + self._load_directory(builtin_dir) # Built-in tasks + self._load_directory(plugins_dir) # Custom tasks, aka plugins + + msg = "Loaded {0} tasks: {1}" + tasks = ', '.join(self._tasks.keys()) + self.logger.info(msg.format(len(self._tasks), tasks)) + + def start(self, task_name, **kwargs): + """Start a given task in a new thread. kwargs are passed to task.run""" + msg = "Starting task '{0}' in a new thread" + self.logger.info(msg.format(task_name)) + + with self._task_access_lock: + try: + task = self._tasks[task_name] + except KeyError: + e = "Couldn't find task '{0}':" + self.logger.error(e.format(task_name)) + return + + task_thread = Thread(target=self._wrapper, args=(task,), kwargs=kwargs) + start_time = strftime("%b %d %H:%M:%S") + task_thread.name = "{0} ({1})".format(task_name, start_time) + task_thread.start() + + def schedule(self, now=None): + """Start all tasks that are supposed to be run at a given time.""" + if not now: + now = gmtime() + # Get list of tasks to run this turn: + tasks = self.bot.config.schedule(now.tm_min, now.tm_hour, now.tm_mday, + now.tm_mon, now.tm_wday) + + for task in tasks: + if isinstance(task, list): # They've specified kwargs, + self.start(task[0], **task[1]) # so pass those to start + else: # Otherwise, just pass task_name + self.start(task) + + def get(self, task_name): + """Return the class instance associated with a certain task name. + + Will raise KeyError if the task is not found. + """ + return self._tasks[task_name] diff --git a/earwigbot/tasks/__init__.py b/earwigbot/tasks/__init__.py index d70bcde..bfa7ef9 100644 --- a/earwigbot/tasks/__init__.py +++ b/earwigbot/tasks/__init__.py @@ -21,22 +21,20 @@ # SOFTWARE. """ -EarwigBot's Wiki Task Manager +EarwigBot's Bot Tasks This package provides the wiki bot "tasks" EarwigBot runs. This module contains -the BaseTask class (import with `from earwigbot.tasks import BaseTask`) and an -internal TaskManager class. This can be accessed through `bot.tasks`. -""" +the BaseTask class (import with `from earwigbot.tasks import BaseTask`), +whereas the package contains various built-in tasks. Additional tasks can be +installed as plugins in the bot's working directory. -import imp -from os import listdir, path -from re import sub -from threading import Lock, Thread -from time import gmtime, strftime +To run a task, use bot.tasks.start(name, **kwargs). **kwargs get passed to the +Task's run() function. +""" from earwigbot import wiki -__all__ = ["BaseTask", "TaskManager"] +__all__ = ["BaseTask"] class BaseTask(object): """A base class for bot tasks that edit Wikipedia.""" @@ -131,125 +129,3 @@ class BaseTask(object): self.logger.warn("Emergency task shutoff has been enabled!") return True - - -class TaskManager(object): - def __init__(self, bot): - self.bot = bot - self.logger = bot.logger.getChild("tasks") - self._tasks = {} - self._task_access_lock = Lock() - - def __iter__(self): - for name in self._tasks: - yield name - - def _wrapper(self, task, **kwargs): - """Wrapper for task classes: run the task and catch any errors.""" - try: - task.run(**kwargs) - except Exception: - msg = "Task '{0}' raised an exception and had to stop:" - self.logger.exception(msg.format(task.name)) - else: - msg = "Task '{0}' finished without error" - self.logger.info(msg.format(task.name)) - - def _load_task(self, name, path): - """Load a specific task from a module, identified by name and path. - - We'll first try to import it using imp magic, and if that works, make - an instance of the 'Task' class inside (assuming it is an instance of - BaseTask), add it to self._tasks, and log the addition. Any problems - along the way will either be ignored or logged. - """ - f, path, desc = imp.find_module(name, [path]) - try: - module = imp.load_module(name, f, path, desc) - except Exception: - e = "Couldn't load module {0} from {1}" - self.logger.exception(e.format(name, path)) - return - finally: - f.close() - - try: - task_class = module.Task - except AttributeError: - return # No task in this module - try: - task = task_class(self.bot) - except Exception: - e = "Error initializing Task() class in {0} (from {1})" - self.logger.exception(e.format(name, path)) - return - if not isinstance(task, BaseTask): - return - - self._tasks[task.name] = task - self.logger.debug("Loaded task {0}".format(task.name)) - - def _load_directory(self, dir): - """Load all valid tasks in a given directory.""" - processed = [] - for name in listdir(dir): - if not name.endswith(".py") and not name.endswith(".pyc"): - continue - if name.startswith("_") or name.startswith("."): - continue - modname = sub("\.pyc?$", "", name) # Remove extension - if modname not in processed: - self._load_task(modname, dir) - processed.append(modname) - - def load(self): - """Load (or reload) all valid tasks into self._tasks.""" - with self._task_access_lock: - self._tasks.clear() - builtin_dir = path.dirname(__file__) - plugins_dir = path.join(self.bot.config.root_dir, "tasks") - self._load_directory(builtin_dir) # Built-in tasks - self._load_directory(plugins_dir) # Custom tasks, aka plugins - - msg = "Loaded {0} tasks: {1}" - tasks = ', '.join(self._tasks.keys()) - self.logger.info(msg.format(len(self._tasks), tasks)) - - def start(self, task_name, **kwargs): - """Start a given task in a new thread. kwargs are passed to task.run""" - msg = "Starting task '{0}' in a new thread" - self.logger.info(msg.format(task_name)) - - with self._task_access_lock: - try: - task = self._tasks[task_name] - except KeyError: - e = "Couldn't find task '{0}':" - self.logger.error(e.format(task_name)) - return - - task_thread = Thread(target=self._wrapper, args=(task,), kwargs=kwargs) - start_time = strftime("%b %d %H:%M:%S") - task_thread.name = "{0} ({1})".format(task_name, start_time) - task_thread.start() - - def schedule(self, now=None): - """Start all tasks that are supposed to be run at a given time.""" - if not now: - now = gmtime() - # Get list of tasks to run this turn: - tasks = self.bot.config.schedule(now.tm_min, now.tm_hour, now.tm_mday, - now.tm_mon, now.tm_wday) - - for task in tasks: - if isinstance(task, list): # They've specified kwargs, - self.start(task[0], **task[1]) # so pass those to start - else: # Otherwise, just pass task_name - self.start(task) - - def get(self, task_name): - """Return the class instance associated with a certain task name. - - Will raise KeyError if the task is not found. - """ - return self._tasks[task_name] diff --git a/earwigbot/wiki/__init__.py b/earwigbot/wiki/__init__.py index f2f0e89..558754b 100644 --- a/earwigbot/wiki/__init__.py +++ b/earwigbot/wiki/__init__.py @@ -29,10 +29,10 @@ written by Mr.Z-man, other than a similar purpose. We share no code. Import the toolset directly with `from earwigbot import wiki`. If using the built-in integration with the rest of the bot, Bot() objects contain a `wiki` -attribute, which is a SitesDBManager object tied to the sites.db file located -in the same directory as config.yml. That object has the principal methods -get_site, add_site, and remove_site that should handle all of your Site (and -thus, Page, Category, and User) needs. +attribute, which is a SitesDB object tied to the sites.db file located in the +same directory as config.yml. That object has the principal methods get_site, +add_site, and remove_site that should handle all of your Site (and thus, Page, +Category, and User) needs. """ import logging as _log diff --git a/earwigbot/wiki/sitesdb.py b/earwigbot/wiki/sitesdb.py index a5c9fe7..7ad8bf8 100644 --- a/earwigbot/wiki/sitesdb.py +++ b/earwigbot/wiki/sitesdb.py @@ -32,9 +32,9 @@ from earwigbot import __version__ from earwigbot.wiki.exceptions import SiteNotFoundError from earwigbot.wiki.site import Site -__all__ = ["SitesDBManager"] +__all__ = ["SitesDB"] -class SitesDBManager(object): +class SitesDB(object): """ EarwigBot's Wiki Toolset: Sites Database Manager @@ -49,8 +49,7 @@ class SitesDBManager(object): here are available as bot.wiki.get_site(), bot.wiki.add_site(), and bot.wiki.remove_site(), which use a sites.db file located in the same directory as our config.yml file. Lower-level access can be achieved - by importing the manager class - (`from earwigbot.wiki import SitesDBManager`). + by importing the manager class (`from earwigbot.wiki import SitesDB`). """ def __init__(self, config): From 03fe3305d9592b54813bd2c3d46b5cffe9f9d808 Mon Sep 17 00:00:00 2001 From: Ben Kurtovic Date: Sat, 7 Apr 2012 18:21:56 -0400 Subject: [PATCH 16/26] Avoid duplicating code thanks to _BaseManager --- earwigbot/managers.py | 175 ++++++++++++++-------------------------------- earwigbot/wiki/sitesdb.py | 2 +- 2 files changed, 55 insertions(+), 122 deletions(-) diff --git a/earwigbot/managers.py b/earwigbot/managers.py index 5df4e73..d3fbfca 100644 --- a/earwigbot/managers.py +++ b/earwigbot/managers.py @@ -33,22 +33,22 @@ from earwigbot.tasks import BaseTask __all__ = ["CommandManager", "TaskManager"] class _BaseManager(object): - pass - - -class CommandManager(_BaseManager): - def __init__(self, bot): + def __init__(self, bot, name, attribute, base): self.bot = bot - self.logger = bot.logger.getChild("commands") - self._commands = {} - self._command_access_lock = Lock() + self.logger = bot.logger.getChild(name) + + self._resources = {} + self._resource_name = name # e.g. "commands" or "tasks" + self._resource_attribute = attribute # e.g. "Command" or "Task" + self._resource_base = base # e.g. BaseCommand or BaseTask + self._resource_access_lock = Lock() def __iter__(self): - for name in self._commands: + for name in self._resources: yield name - def _load_command(self, name, path): - """Load a specific command from a module, identified by name and path. + def _load_resource(self, name, path): + """Load a specific resource from a module, identified by name and path. We'll first try to import it using imp magic, and if that works, make an instance of the 'Command' class inside (assuming it is an instance @@ -59,30 +59,30 @@ class CommandManager(_BaseManager): try: module = imp.load_module(name, f, path, desc) except Exception: - e = "Couldn't load module {0} from {1}" + e = "Couldn't load module {0} (from {1})" self.logger.exception(e.format(name, path)) return finally: f.close() + attr = self._resource_attribute + if not hasattr(module, attr): + return # No resources in this module + resource_class = getattr(module, attr) try: - command_class = module.Command - except AttributeError: - return # No command in this module - try: - command = command_class(self.bot) + resource = resource_class(self.bot) # Create instance of resource except Exception: - e = "Error initializing Command() class in {0} (from {1})" - self.logger.exception(e.format(name, path)) + e = "Error instantiating {0} class in {1} (from {2})" + self.logger.exception(e.format(attr, name, path)) return - if not isinstance(command, BaseCommand): + if not isinstance(resource, self._resource_base): return - self._commands[command.name] = command - self.logger.debug("Loaded command {0}".format(command.name)) + self._resources[resource.name] = resource + self.logger.debug("Loaded {0} {1}".format(attr.lower(), resource.name)) def _load_directory(self, dir): - """Load all valid commands in a given directory.""" + """Load all valid resources in a given directory.""" processed = [] for name in listdir(dir): if not name.endswith(".py") and not name.endswith(".pyc"): @@ -91,26 +91,40 @@ class CommandManager(_BaseManager): continue modname = sub("\.pyc?$", "", name) # Remove extension if modname not in processed: - self._load_command(modname, dir) + self._load_resource(modname, dir) processed.append(modname) def load(self): - """Load (or reload) all valid commands into self._commands.""" - with self._command_access_lock: - self._commands.clear() - builtin_dir = path.join(path.dirname(__file__), "commands") - plugins_dir = path.join(self.bot.config.root_dir, "commands") - self._load_directory(builtin_dir) # Built-in commands - self._load_directory(plugins_dir) # Custom commands, aka plugins - - msg = "Loaded {0} commands: {1}" - commands = ", ".join(self._commands.keys()) - self.logger.info(msg.format(len(self._commands), commands)) + """Load (or reload) all valid resources into self._resources.""" + name = self._resource_name # e.g. "commands" or "tasks" + with self._resource_access_lock: + self._resources.clear() + builtin_dir = path.join(path.dirname(__file__), name) + plugins_dir = path.join(self.bot.config.root_dir, name) + self._load_directory(builtin_dir) # Built-in resources + self._load_directory(plugins_dir) # Custom resources, aka plugins + + msg = "Loaded {0} {1}: {2}" + resources = ", ".join(self._resources.keys()) + self.logger.info(msg.format(len(self._resources), name, resources)) + + def get(self, key): + """Return the class instance associated with a certain resource. + + Will raise KeyError if the resource (command or task) is not found. + """ + return self._resources[key] + + +class CommandManager(_BaseManager): + def __init__(self, bot): + super(CommandManager, self).__init__(bot, "commands", "Command", + BaseCommand) def check(self, hook, data): """Given an IRC event, check if there's anything we can respond to.""" - with self._command_access_lock: - for command in self._commands.values(): + with self._resource_access_lock: + for command in self._resources.values(): if hook in command.hooks: if command.check(data): try: @@ -120,24 +134,10 @@ class CommandManager(_BaseManager): self.logger.exception(e.format(data.command)) break - def get(self, command_name): - """Return the class instance associated with a certain command name. - - Will raise KeyError if the command is not found. - """ - return self._command[command_name] - class TaskManager(_BaseManager): def __init__(self, bot): - self.bot = bot - self.logger = bot.logger.getChild("tasks") - self._tasks = {} - self._task_access_lock = Lock() - - def __iter__(self): - for name in self._tasks: - yield name + super(TaskManager, self).__init__(bot, "tasks", "Task", BaseTask) def _wrapper(self, task, **kwargs): """Wrapper for task classes: run the task and catch any errors.""" @@ -150,74 +150,14 @@ class TaskManager(_BaseManager): msg = "Task '{0}' finished without error" self.logger.info(msg.format(task.name)) - def _load_task(self, name, path): - """Load a specific task from a module, identified by name and path. - - We'll first try to import it using imp magic, and if that works, make - an instance of the 'Task' class inside (assuming it is an instance of - BaseTask), add it to self._tasks, and log the addition. Any problems - along the way will either be ignored or logged. - """ - f, path, desc = imp.find_module(name, [path]) - try: - module = imp.load_module(name, f, path, desc) - except Exception: - e = "Couldn't load module {0} from {1}" - self.logger.exception(e.format(name, path)) - return - finally: - f.close() - - try: - task_class = module.Task - except AttributeError: - return # No task in this module - try: - task = task_class(self.bot) - except Exception: - e = "Error initializing Task() class in {0} (from {1})" - self.logger.exception(e.format(name, path)) - return - if not isinstance(task, BaseTask): - return - - self._tasks[task.name] = task - self.logger.debug("Loaded task {0}".format(task.name)) - - def _load_directory(self, dir): - """Load all valid tasks in a given directory.""" - processed = [] - for name in listdir(dir): - if not name.endswith(".py") and not name.endswith(".pyc"): - continue - if name.startswith("_") or name.startswith("."): - continue - modname = sub("\.pyc?$", "", name) # Remove extension - if modname not in processed: - self._load_task(modname, dir) - processed.append(modname) - - def load(self): - """Load (or reload) all valid tasks into self._tasks.""" - with self._task_access_lock: - self._tasks.clear() - builtin_dir = path.join(path.dirname(__file__), "tasks") - plugins_dir = path.join(self.bot.config.root_dir, "tasks") - self._load_directory(builtin_dir) # Built-in tasks - self._load_directory(plugins_dir) # Custom tasks, aka plugins - - msg = "Loaded {0} tasks: {1}" - tasks = ', '.join(self._tasks.keys()) - self.logger.info(msg.format(len(self._tasks), tasks)) - def start(self, task_name, **kwargs): """Start a given task in a new thread. kwargs are passed to task.run""" msg = "Starting task '{0}' in a new thread" self.logger.info(msg.format(task_name)) - with self._task_access_lock: + with self._resource_access_lock: try: - task = self._tasks[task_name] + task = self._resources[task_name] except KeyError: e = "Couldn't find task '{0}':" self.logger.error(e.format(task_name)) @@ -241,10 +181,3 @@ class TaskManager(_BaseManager): self.start(task[0], **task[1]) # so pass those to start else: # Otherwise, just pass task_name self.start(task) - - def get(self, task_name): - """Return the class instance associated with a certain task name. - - Will raise KeyError if the task is not found. - """ - return self._tasks[task_name] diff --git a/earwigbot/wiki/sitesdb.py b/earwigbot/wiki/sitesdb.py index 7ad8bf8..6178564 100644 --- a/earwigbot/wiki/sitesdb.py +++ b/earwigbot/wiki/sitesdb.py @@ -55,7 +55,7 @@ class SitesDB(object): def __init__(self, config): """Set up the manager with an attribute for the BotConfig object.""" self.config = config - self._sitesdb = path.join(config.root_dir, "sitesdb") + self._sitesdb = path.join(config.root_dir, "sites.db") self._cookie_file = path.join(config.root_dir, ".cookies") self._cookiejar = None From eead11ffb6dc8701d42fc7bdc231a07c31b54520 Mon Sep 17 00:00:00 2001 From: Ben Kurtovic Date: Sat, 7 Apr 2012 19:40:44 -0400 Subject: [PATCH 17/26] Docstrings and cleanup in earwigbot.managers --- earwigbot/managers.py | 60 ++++++++++++++++++++++++++++++++++++++------------- 1 file changed, 45 insertions(+), 15 deletions(-) diff --git a/earwigbot/managers.py b/earwigbot/managers.py index d3fbfca..0802516 100644 --- a/earwigbot/managers.py +++ b/earwigbot/managers.py @@ -32,7 +32,24 @@ from earwigbot.tasks import BaseTask __all__ = ["CommandManager", "TaskManager"] -class _BaseManager(object): +class _ResourceManager(object): + """ + EarwigBot's Base Resource Manager + + Resources are essentially objects dynamically loaded by the bot, both + packaged with it (built-in resources) and created by users (plugins, aka + custom resources). Currently, the only two types of resources are IRC + commands and bot tasks. These are both loaded from two locations: the + earwigbot.commands and earwigbot.tasks packages, and the commands/ and + tasks/ directories within the bot's working directory. + + This class handles the low-level tasks of (re)loading resources via load(), + retrieving specific resources via get(), and iterating over all resources + via __iter__(). If iterating over resources, it is recommended to acquire + self.lock beforehand and release it afterwards (alternatively, wrap your + code in a `with` statement) so an attempt at reloading resources in another + thread won't disrupt your iteration. + """ def __init__(self, bot, name, attribute, base): self.bot = bot self.logger = bot.logger.getChild(name) @@ -43,6 +60,10 @@ class _BaseManager(object): self._resource_base = base # e.g. BaseCommand or BaseTask self._resource_access_lock = Lock() + @property + def lock(self): + return self._resource_access_lock + def __iter__(self): for name in self._resources: yield name @@ -97,7 +118,7 @@ class _BaseManager(object): def load(self): """Load (or reload) all valid resources into self._resources.""" name = self._resource_name # e.g. "commands" or "tasks" - with self._resource_access_lock: + with self.lock: self._resources.clear() builtin_dir = path.join(path.dirname(__file__), name) plugins_dir = path.join(self.bot.config.root_dir, name) @@ -116,15 +137,20 @@ class _BaseManager(object): return self._resources[key] -class CommandManager(_BaseManager): +class CommandManager(_ResourceManager): + """ + EarwigBot's IRC Command Manager + + Manages (i.e., loads, reloads, and calls) IRC commands. + """ def __init__(self, bot): - super(CommandManager, self).__init__(bot, "commands", "Command", - BaseCommand) + base = super(CommandManager, self) + base.__init__(bot, "commands", "Command", BaseCommand) def check(self, hook, data): """Given an IRC event, check if there's anything we can respond to.""" - with self._resource_access_lock: - for command in self._resources.values(): + with self.lock: + for command in self._resources.itervalues(): if hook in command.hooks: if command.check(data): try: @@ -135,7 +161,12 @@ class CommandManager(_BaseManager): break -class TaskManager(_BaseManager): +class TaskManager(_ResourceManager): + """ + EarwigBot's Bot Task Manager + + Manages (i.e., loads, reloads, schedules, and runs) wiki bot tasks. + """ def __init__(self, bot): super(TaskManager, self).__init__(bot, "tasks", "Task", BaseTask) @@ -155,13 +186,12 @@ class TaskManager(_BaseManager): msg = "Starting task '{0}' in a new thread" self.logger.info(msg.format(task_name)) - with self._resource_access_lock: - try: - task = self._resources[task_name] - except KeyError: - e = "Couldn't find task '{0}':" - self.logger.error(e.format(task_name)) - return + try: + task = self.get(task_name) + except KeyError: + e = "Couldn't find task '{0}':" + self.logger.error(e.format(task_name)) + return task_thread = Thread(target=self._wrapper, args=(task,), kwargs=kwargs) start_time = strftime("%b %d %H:%M:%S") From 6373eea1f7d1b32b926d9d59eb8accc0c40a2a41 Mon Sep 17 00:00:00 2001 From: Ben Kurtovic Date: Sat, 7 Apr 2012 22:21:32 -0400 Subject: [PATCH 18/26] docstrings in Bot; daemonize wiki_scheduler --- earwigbot/bot.py | 35 ++++++++++++++++++++++++++--------- earwigbot/irc/connection.py | 4 ++-- 2 files changed, 28 insertions(+), 11 deletions(-) diff --git a/earwigbot/bot.py b/earwigbot/bot.py index 071cca2..9a1bc79 100644 --- a/earwigbot/bot.py +++ b/earwigbot/bot.py @@ -69,6 +69,7 @@ class Bot(object): self.tasks.load() def _start_irc_components(self): + """Start the IRC frontend/watcher in separate threads if enabled.""" if self.config.components.get("irc_frontend"): self.logger.info("Starting IRC frontend") self.frontend = Frontend(self) @@ -80,6 +81,7 @@ class Bot(object): Thread(name="irc_watcher", target=self.watcher.loop).start() def _start_wiki_scheduler(self): + """Start the wiki scheduler in a separate thread if enabled.""" def wiki_scheduler(): while self._keep_looping: time_start = time() @@ -91,15 +93,27 @@ class Bot(object): if self.config.components.get("wiki_scheduler"): self.logger.info("Starting wiki scheduler") - Thread(name="wiki_scheduler", target=wiki_scheduler).start() + thread = Thread(name="wiki_scheduler", target=wiki_scheduler) + thread.daemon = True # Stop if other threads stop + thread.start() def _stop_irc_components(self): + """Request the IRC frontend and watcher to stop if enabled.""" if self.frontend: self.frontend.stop() if self.watcher: self.watcher.stop() - def _loop(self): + def run(self): + """Main entry point into running the bot. + + Starts all config-enabled components and then enters an idle loop, + ensuring that all components remain online and restarting components + that get disconnected from their servers. + """ + self.logger.info("Starting bot") + self._start_irc_components() + self._start_wiki_scheduler() while self._keep_looping: with self.component_lock: if self.frontend and self.frontend.is_stopped(): @@ -110,15 +124,17 @@ class Bot(object): self.logger.warn("IRC watcher has stopped; restarting") self.watcher = Watcher(self) Thread(name=name, target=self.watcher.loop).start() - sleep(3) - - def run(self): - self.logger.info("Starting bot") - self._start_irc_components() - self._start_wiki_scheduler() - self._loop() + sleep(2) def restart(self): + """Reload config, commands, tasks, and safely restart IRC components. + + This is thread-safe, and it will gracefully stop IRC components before + reloading anything. Note that you can safely reload commands or tasks + without restarting the bot with bot.commands.load() or + bot.tasks.load(). These should not interfere with running components + or tasks. + """ self.logger.info("Restarting bot per request from owner") with self.component_lock: self._stop_irc_components() @@ -128,6 +144,7 @@ class Bot(object): self._start_irc_components() def stop(self): + """Gracefully stop all bot components.""" self.logger.info("Shutting down bot") with self.component_lock: self._stop_irc_components() diff --git a/earwigbot/irc/connection.py b/earwigbot/irc/connection.py index 7e2bb19..62b0455 100644 --- a/earwigbot/irc/connection.py +++ b/earwigbot/irc/connection.py @@ -147,10 +147,10 @@ class IRCConnection(object): self._close() break - def stop(self): + def stop(self, msg=None): """Request the IRC connection to close at earliest convenience.""" if self._is_running: - self.quit() + self.quit(msg) self._is_running = False def is_stopped(self): From cc6a4c6b9903fe43617bee46005649f58c8ffddb Mon Sep 17 00:00:00 2001 From: Ben Kurtovic Date: Sat, 7 Apr 2012 23:05:44 -0400 Subject: [PATCH 19/26] Release manager's _resource_access_lock before processing a command so it can reload itself --- earwigbot/commands/__init__.py | 16 ++++++++++++++-- earwigbot/managers.py | 17 +++++++---------- 2 files changed, 21 insertions(+), 12 deletions(-) diff --git a/earwigbot/commands/__init__.py b/earwigbot/commands/__init__.py index 33604f1..f50094e 100644 --- a/earwigbot/commands/__init__.py +++ b/earwigbot/commands/__init__.py @@ -57,10 +57,22 @@ class BaseCommand(object): self.config = bot.config self.logger = bot.commands.logger.getChild(self.name) + def _wrap_check(self, data): + """Check whether this command should be called, catching errors.""" + try: + return self.check(data) + except Exception: + e = "Error checking command '{0}' with data: {1}:" + self.logger.exception(e.format(self.name, data)) + def _wrap_process(self, data): - """Make a quick connection alias and then process() the message.""" + """Make a connection alias, process() the message, and catch errors.""" self.connection = self.bot.frontend - self.process(data) + try: + self.process(data) + except Exception: + e = "Error executing command '{0}':" + self.logger.exception(e.format(data.command)) def check(self, data): """Return whether this command should be called in response to 'data'. diff --git a/earwigbot/managers.py b/earwigbot/managers.py index 0802516..28da7d6 100644 --- a/earwigbot/managers.py +++ b/earwigbot/managers.py @@ -149,16 +149,13 @@ class CommandManager(_ResourceManager): def check(self, hook, data): """Given an IRC event, check if there's anything we can respond to.""" - with self.lock: - for command in self._resources.itervalues(): - if hook in command.hooks: - if command.check(data): - try: - command._wrap_process(data) - except Exception: - e = "Error executing command '{0}':" - self.logger.exception(e.format(data.command)) - break + self.lock.acquire() + for command in self._resources.itervalues(): + if hook in command.hooks and command._wrap_check(data): + self.lock.release() + command._wrap_process(data) + return + self.lock.release() class TaskManager(_ResourceManager): From fbada5c69aacee09f2582f5790c35d97e65156ef Mon Sep 17 00:00:00 2001 From: Ben Kurtovic Date: Sat, 7 Apr 2012 23:08:39 -0400 Subject: [PATCH 20/26] Minor cleanup, additions, and bugfixes --- earwigbot/bot.py | 21 +++++++++++++-------- earwigbot/commands/restart.py | 9 +++++++-- earwigbot/commands/threads.py | 2 +- earwigbot/irc/connection.py | 2 +- 4 files changed, 22 insertions(+), 12 deletions(-) diff --git a/earwigbot/bot.py b/earwigbot/bot.py index 9a1bc79..1f23d5b 100644 --- a/earwigbot/bot.py +++ b/earwigbot/bot.py @@ -97,12 +97,12 @@ class Bot(object): thread.daemon = True # Stop if other threads stop thread.start() - def _stop_irc_components(self): + def _stop_irc_components(self, msg): """Request the IRC frontend and watcher to stop if enabled.""" if self.frontend: - self.frontend.stop() + self.frontend.stop(msg) if self.watcher: - self.watcher.stop() + self.watcher.stop(msg) def run(self): """Main entry point into running the bot. @@ -126,7 +126,7 @@ class Bot(object): Thread(name=name, target=self.watcher.loop).start() sleep(2) - def restart(self): + def restart(self, msg=None): """Reload config, commands, tasks, and safely restart IRC components. This is thread-safe, and it will gracefully stop IRC components before @@ -134,18 +134,23 @@ class Bot(object): without restarting the bot with bot.commands.load() or bot.tasks.load(). These should not interfere with running components or tasks. + + If given, 'msg' will be used as our quit message. """ self.logger.info("Restarting bot per request from owner") with self.component_lock: - self._stop_irc_components() + self._stop_irc_components(msg) self.config.load() self.commands.load() self.tasks.load() self._start_irc_components() - def stop(self): - """Gracefully stop all bot components.""" + def stop(self, msg=None): + """Gracefully stop all bot components. + + If given, 'msg' will be used as our quit message. + """ self.logger.info("Shutting down bot") with self.component_lock: - self._stop_irc_components() + self._stop_irc_components(msg) self._keep_looping = False diff --git a/earwigbot/commands/restart.py b/earwigbot/commands/restart.py index 8756fe1..2e15c9c 100644 --- a/earwigbot/commands/restart.py +++ b/earwigbot/commands/restart.py @@ -38,9 +38,14 @@ class Command(BaseCommand): if data.command == "restart": self.logger.info("Restarting bot per owner request") - self.bot.restart() + if data.args: + self.bot.restart(" ".join(data.args)) + else: + self.bot.restart() elif data.command == "reload": self.logger.info("Reloading IRC commands") self.bot.commands.load() - self.connection.reply("IRC commands reloaded.") + self.logger.info("Reloading bot tasks") + self.bot.tasks.load() + self.connection.reply("IRC commands and bot tasks reloaded.") diff --git a/earwigbot/commands/threads.py b/earwigbot/commands/threads.py index bfe894b..4f66cb6 100644 --- a/earwigbot/commands/threads.py +++ b/earwigbot/commands/threads.py @@ -76,7 +76,7 @@ class Command(BaseCommand): for thread in threads: tname = thread.name if tname == "MainThread": - t = "\x0302MainThread\x0301 (id {1})" + t = "\x0302MainThread\x0301 (id {0})" normal_threads.append(t.format(thread.ident)) elif tname in self.config.components: t = "\x0302{0}\x0301 (id {1})" diff --git a/earwigbot/irc/connection.py b/earwigbot/irc/connection.py index 62b0455..c733df1 100644 --- a/earwigbot/irc/connection.py +++ b/earwigbot/irc/connection.py @@ -124,7 +124,7 @@ class IRCConnection(object): def quit(self, msg=None): """Issue a quit message to the server.""" if msg: - self._send("QUIT {0}".format(msg)) + self._send("QUIT :{0}".format(msg)) else: self._send("QUIT") From c3fa92269d20a7bdf80d6dbcf142ac632e526035 Mon Sep 17 00:00:00 2001 From: Ben Kurtovic Date: Sun, 8 Apr 2012 00:39:05 -0400 Subject: [PATCH 21/26] Quitting works completely now; bugfixes --- earwigbot/commands/ctcp.py | 4 +-- earwigbot/commands/{restart.py => quit.py} | 48 ++++++++++++++++++++---------- earwigbot/commands/threads.py | 4 +-- earwigbot/config.py | 1 + earwigbot/irc/connection.py | 16 +++++----- earwigbot/util.py | 5 +++- 6 files changed, 50 insertions(+), 28 deletions(-) rename earwigbot/commands/{restart.py => quit.py} (52%) diff --git a/earwigbot/commands/ctcp.py b/earwigbot/commands/ctcp.py index 6641d7e..824cb4f 100644 --- a/earwigbot/commands/ctcp.py +++ b/earwigbot/commands/ctcp.py @@ -27,8 +27,8 @@ from earwigbot import __version__ from earwigbot.commands import BaseCommand class Command(BaseCommand): - """Not an actual command, this module is used to respond to the CTCP - commands PING, TIME, and VERSION.""" + """Not an actual command; this module implements responses to the CTCP + requests PING, TIME, and VERSION.""" name = "ctcp" hooks = ["msg_private"] diff --git a/earwigbot/commands/restart.py b/earwigbot/commands/quit.py similarity index 52% rename from earwigbot/commands/restart.py rename to earwigbot/commands/quit.py index 2e15c9c..9b8cbc3 100644 --- a/earwigbot/commands/restart.py +++ b/earwigbot/commands/quit.py @@ -23,11 +23,12 @@ from earwigbot.commands import BaseCommand class Command(BaseCommand): - """Restart the bot. Only the owner can do this.""" - name = "restart" + """Quit, restart, or reload components from the bot. Only the owners can + run this command.""" + name = "quit" def check(self, data): - commands = ["restart", "reload"] + commands = ["quit", "restart", "reload"] return data.is_command and data.command in commands def process(self, data): @@ -35,17 +36,34 @@ class Command(BaseCommand): msg = "you must be a bot owner to use this command." self.connection.reply(data, msg) return + if data.command == "quit": + self.do_quit(data) + elif data.command == "restart": + self.do_restart(data) + else: + self.do_reload(data) - if data.command == "restart": - self.logger.info("Restarting bot per owner request") - if data.args: - self.bot.restart(" ".join(data.args)) - else: - self.bot.restart() + def do_quit(self, data): + nick = self.config.irc.frontend["nick"] + if not data.args or data.args[0].lower() != nick.lower(): + self.connection.reply(data, "to confirm this action, the first argument must be my nickname.") + return + if data.args[1:]: + msg = " ".join(data.args[1:]) + self.bot.stop("Stopped by {0}: {1}".format(data.nick, msg)) + else: + self.bot.stop("Stopped by {0}".format(data.nick)) + + def do_restart(self, data): + self.logger.info("Restarting bot per owner request") + if data.args: + msg = " ".join(data.args) + self.bot.restart("Restarted by {0}: {1}".format(data.nick, msg)) + else: + self.bot.restart("Restarted by {0}".format(data.nick)) - elif data.command == "reload": - self.logger.info("Reloading IRC commands") - self.bot.commands.load() - self.logger.info("Reloading bot tasks") - self.bot.tasks.load() - self.connection.reply("IRC commands and bot tasks reloaded.") + def do_reload(self, data): + self.logger.info("Reloading IRC commands and bot tasks") + self.bot.commands.load() + self.bot.tasks.load() + self.connection.reply(data, "IRC commands and bot tasks reloaded.") diff --git a/earwigbot/commands/threads.py b/earwigbot/commands/threads.py index 4f66cb6..46858a5 100644 --- a/earwigbot/commands/threads.py +++ b/earwigbot/commands/threads.py @@ -117,9 +117,9 @@ class Command(BaseCommand): t = "\x0302{0}\x0301 (\x02active\x0F as ids {1})" tasklist.append(t.format(task, ', '.join(ids))) - tasklist = ", ".join(tasklist) + tasks = ", ".join(tasklist) - msg = "{0} tasks loaded: {1}.".format(len(all_tasks), tasklist) + msg = "{0} tasks loaded: {1}.".format(len(tasklist), tasks) self.connection.reply(self.data, msg) def do_start(self): diff --git a/earwigbot/config.py b/earwigbot/config.py index 962f8d3..035ee09 100644 --- a/earwigbot/config.py +++ b/earwigbot/config.py @@ -98,6 +98,7 @@ class BotConfig(object): """Configures the logging module so it works the way we want it to.""" log_dir = self._log_dir logger = logging.getLogger("earwigbot") + logger.handlers = [] # Remove any handlers already attached to us logger.setLevel(logging.DEBUG) if self.metadata.get("enableLogging"): diff --git a/earwigbot/irc/connection.py b/earwigbot/irc/connection.py index c733df1..a14284d 100644 --- a/earwigbot/irc/connection.py +++ b/earwigbot/irc/connection.py @@ -81,6 +81,13 @@ class IRCConnection(object): self._sock.sendall(msg + "\r\n") self.logger.debug(msg) + def _quit(self, msg=None): + """Issue a quit message to the server.""" + if msg: + self._send("QUIT :{0}".format(msg)) + else: + self._send("QUIT") + def say(self, target, msg): """Send a private message to a target on the server.""" msg = "PRIVMSG {0} :{1}".format(target, msg) @@ -121,13 +128,6 @@ class IRCConnection(object): msg = "PONG {0}".format(target) self._send(msg) - def quit(self, msg=None): - """Issue a quit message to the server.""" - if msg: - self._send("QUIT :{0}".format(msg)) - else: - self._send("QUIT") - def loop(self): """Main loop for the IRC connection.""" self._is_running = True @@ -150,7 +150,7 @@ class IRCConnection(object): def stop(self, msg=None): """Request the IRC connection to close at earliest convenience.""" if self._is_running: - self.quit(msg) + self._quit(msg) self._is_running = False def is_stopped(self): diff --git a/earwigbot/util.py b/earwigbot/util.py index 442bc6e..51ad463 100755 --- a/earwigbot/util.py +++ b/earwigbot/util.py @@ -67,8 +67,11 @@ def main(): bot.tasks.start(args.task) else: bot.run() + except KeyboardInterrupt: + pass finally: - bot.stop() + if not bot._keep_looping: # Indicates bot has already been stopped + bot.stop() if __name__ == "__main__": main() From 08fb35eefe3a2316b3620df6cd6d69c3cf38a5f0 Mon Sep 17 00:00:00 2001 From: Ben Kurtovic Date: Sun, 8 Apr 2012 00:53:05 -0400 Subject: [PATCH 22/26] Fix small bug in shutdown code --- earwigbot/bot.py | 2 +- earwigbot/util.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/earwigbot/bot.py b/earwigbot/bot.py index 1f23d5b..6f98690 100644 --- a/earwigbot/bot.py +++ b/earwigbot/bot.py @@ -150,7 +150,7 @@ class Bot(object): If given, 'msg' will be used as our quit message. """ - self.logger.info("Shutting down bot") + self.logger.info("Stopping bot") with self.component_lock: self._stop_irc_components(msg) self._keep_looping = False diff --git a/earwigbot/util.py b/earwigbot/util.py index 51ad463..d5f48cf 100755 --- a/earwigbot/util.py +++ b/earwigbot/util.py @@ -70,7 +70,7 @@ def main(): except KeyboardInterrupt: pass finally: - if not bot._keep_looping: # Indicates bot has already been stopped + if bot._keep_looping: # Indicates bot hasn't already been stopped bot.stop() if __name__ == "__main__": From 30e9d30f9f25092032e7a1f9e80b3ffb6f828bd8 Mon Sep 17 00:00:00 2001 From: Ben Kurtovic Date: Sun, 8 Apr 2012 01:22:40 -0400 Subject: [PATCH 23/26] Using function aliases whenever possible; !join and !part --- earwigbot/commands/__init__.py | 13 ++++++++++-- earwigbot/commands/afc_report.py | 11 +++++----- earwigbot/commands/afc_status.py | 18 ++++++++-------- earwigbot/commands/calc.py | 6 +++--- earwigbot/commands/chanops.py | 43 +++++++++++++++++++++++++------------- earwigbot/commands/crypt.py | 18 ++++++++-------- earwigbot/commands/ctcp.py | 8 +++---- earwigbot/commands/editcount.py | 4 ++-- earwigbot/commands/git.py | 38 ++++++++++++++++----------------- earwigbot/commands/help.py | 6 +++--- earwigbot/commands/link.py | 6 +++--- earwigbot/commands/praise.py | 4 ++-- earwigbot/commands/quit.py | 7 +++---- earwigbot/commands/registration.py | 6 +++--- earwigbot/commands/remind.py | 10 ++++----- earwigbot/commands/replag.py | 2 +- earwigbot/commands/rights.py | 4 ++-- earwigbot/commands/test.py | 5 +++-- earwigbot/commands/threads.py | 18 ++++++++-------- earwigbot/irc/connection.py | 4 ++-- 20 files changed, 126 insertions(+), 105 deletions(-) diff --git a/earwigbot/commands/__init__.py b/earwigbot/commands/__init__.py index f50094e..4b4049f 100644 --- a/earwigbot/commands/__init__.py +++ b/earwigbot/commands/__init__.py @@ -57,6 +57,16 @@ class BaseCommand(object): self.config = bot.config self.logger = bot.commands.logger.getChild(self.name) + # Convenience functions: + self.say = lambda target, msg: self.bot.frontend.say(target, msg) + self.reply = lambda data, msg: self.bot.frontend.reply(data, msg) + self.action = lambda target, msg: self.bot.frontend.action(target, msg) + self.notice = lambda target, msg: self.bot.frontend.notice(target, msg) + self.join = lambda chan: self.bot.frontend.join(chan) + self.part = lambda chan: self.bot.frontend.part(chan) + self.mode = lambda t, level, msg: self.bot.frontend.mode(t, level, msg) + self.pong = lambda target: self.bot.frontend.pong(target) + def _wrap_check(self, data): """Check whether this command should be called, catching errors.""" try: @@ -66,8 +76,7 @@ class BaseCommand(object): self.logger.exception(e.format(self.name, data)) def _wrap_process(self, data): - """Make a connection alias, process() the message, and catch errors.""" - self.connection = self.bot.frontend + """process() the message, catching and reporting any errors.""" try: self.process(data) except Exception: diff --git a/earwigbot/commands/afc_report.py b/earwigbot/commands/afc_report.py index 9803a33..aa1b332 100644 --- a/earwigbot/commands/afc_report.py +++ b/earwigbot/commands/afc_report.py @@ -43,7 +43,7 @@ class Command(BaseCommand): if not data.args: msg = "what submission do you want me to give information about?" - self.connection.reply(data, msg) + self.reply(data, msg) return title = " ".join(data.args) @@ -67,8 +67,7 @@ class Command(BaseCommand): if page: return self.report(page) - msg = "submission \x0302{0}\x0301 not found.".format(title) - self.connection.reply(data, msg) + self.reply(data, "submission \x0302{0}\x0301 not found.".format(title)) def get_page(self, title): page = self.site.get_page(title, follow_redirects=False) @@ -89,9 +88,9 @@ class Command(BaseCommand): if status == "accepted": msg3 = "Reviewed by \x0302{0}\x0301 ({1})" - self.connection.reply(self.data, msg1.format(short, url)) - self.connection.say(self.data.chan, msg2.format(status)) - self.connection.say(self.data.chan, msg3.format(user_name, user_url)) + self.reply(self.data, msg1.format(short, url)) + self.say(self.data.chan, msg2.format(status)) + self.say(self.data.chan, msg3.format(user_name, user_url)) def get_status(self, page): if page.is_redirect(): diff --git a/earwigbot/commands/afc_status.py b/earwigbot/commands/afc_status.py index 08333a9..49d2a78 100644 --- a/earwigbot/commands/afc_status.py +++ b/earwigbot/commands/afc_status.py @@ -49,7 +49,7 @@ class Command(BaseCommand): if data.line[1] == "JOIN": status = " ".join(("\x02Current status:\x0F", self.get_status())) - self.connection.notice(data.nick, status) + self.notice(data.nick, status) return if data.args: @@ -57,17 +57,17 @@ class Command(BaseCommand): if action.startswith("sub") or action == "s": subs = self.count_submissions() msg = "there are \x0305{0}\x0301 pending AfC submissions (\x0302WP:AFC\x0301)." - self.connection.reply(data, msg.format(subs)) + self.reply(data, msg.format(subs)) elif action.startswith("redir") or action == "r": redirs = self.count_redirects() msg = "there are \x0305{0}\x0301 open redirect requests (\x0302WP:AFC/R\x0301)." - self.connection.reply(data, msg.format(redirs)) + self.reply(data, msg.format(redirs)) elif action.startswith("file") or action == "f": files = self.count_redirects() msg = "there are \x0305{0}\x0301 open file upload requests (\x0302WP:FFU\x0301)." - self.connection.reply(data, msg.format(files)) + self.reply(data, msg.format(files)) elif action.startswith("agg") or action == "a": try: @@ -78,21 +78,21 @@ class Command(BaseCommand): agg_num = self.get_aggregate_number(agg_data) except ValueError: msg = "\x0303{0}\x0301 isn't a number!" - self.connection.reply(data, msg.format(data.args[1])) + self.reply(data, msg.format(data.args[1])) return aggregate = self.get_aggregate(agg_num) msg = "aggregate is \x0305{0}\x0301 (AfC {1})." - self.connection.reply(data, msg.format(agg_num, aggregate)) + self.reply(data, msg.format(agg_num, aggregate)) elif action.startswith("nocolor") or action == "n": - self.connection.reply(data, self.get_status(color=False)) + self.reply(data, self.get_status(color=False)) else: msg = "unknown argument: \x0303{0}\x0301. Valid args are 'subs', 'redirs', 'files', 'agg', 'nocolor'." - self.connection.reply(data, msg.format(data.args[0])) + self.reply(data, msg.format(data.args[0])) else: - self.connection.reply(data, self.get_status()) + self.reply(data, self.get_status()) def get_status(self, color=True): subs = self.count_submissions() diff --git a/earwigbot/commands/calc.py b/earwigbot/commands/calc.py index f6d3177..1f7f834 100644 --- a/earwigbot/commands/calc.py +++ b/earwigbot/commands/calc.py @@ -32,7 +32,7 @@ class Command(BaseCommand): def process(self, data): if not data.args: - self.connection.reply(data, "what do you want me to calculate?") + self.reply(data, "what do you want me to calculate?") return query = ' '.join(data.args) @@ -47,7 +47,7 @@ class Command(BaseCommand): match = r_result.search(result) if not match: - self.connection.reply(data, "Calculation error.") + self.reply(data, "Calculation error.") return result = match.group(1) @@ -62,7 +62,7 @@ class Command(BaseCommand): result += " " + query.split(" in ", 1)[1] res = "%s = %s" % (query, result) - self.connection.reply(data, res) + self.reply(data, res) def cleanup(self, query): fixes = [ diff --git a/earwigbot/commands/chanops.py b/earwigbot/commands/chanops.py index 1bec2d6..f429a95 100644 --- a/earwigbot/commands/chanops.py +++ b/earwigbot/commands/chanops.py @@ -23,32 +23,45 @@ from earwigbot.commands import BaseCommand class Command(BaseCommand): - """Voice, devoice, op, or deop users in the channel.""" + """Voice, devoice, op, or deop users in the channel, or join or part from + other channels.""" name = "chanops" def check(self, data): - commands = ["chanops", "voice", "devoice", "op", "deop"] + commands = ["chanops", "voice", "devoice", "op", "deop", "join", "part"] if data.is_command and data.command in commands: return True return False def process(self, data): if data.command == "chanops": - msg = "available commands are !voice, !devoice, !op, and !deop." - self.connection.reply(data, msg) + msg = "available commands are !voice, !devoice, !op, !deop, !join, and !part." + self.reply(data, msg) return - if data.host not in self.config.irc["permissions"]["admins"]: - msg = "you must be a bot admin to use this command." - self.connection.reply(data, msg) + self.reply(data, "you must be a bot admin to use this command.") return - # If it is just !op/!devoice/whatever without arguments, assume they - # want to do this to themselves: - if not data.args: - target = data.nick - else: - target = data.args[0] + if data.command in ["voice", "devoice", "op", "deop"]: + # If it is just !op/!devoice/whatever without arguments, assume they + # want to do this to themselves: + if not data.args: + target = data.nick + else: + target = data.args[0] - msg = " ".join((data.command, data.chan, target)) - self.connection.say("ChanServ", msg) + msg = " ".join((data.command, data.chan, target)) + self.say("ChanServ", msg) + + else: + if not data.args: + msg = "you must specify a channel to join or part from." + self.reply(data, msg) + return + channel = data.args[0] + if not channel.startswith("#"): + channel = "#" + channel + if data.command == "join": + self.join(channel) + else: + self.part(channel) diff --git a/earwigbot/commands/crypt.py b/earwigbot/commands/crypt.py index e71e139..cf7c369 100644 --- a/earwigbot/commands/crypt.py +++ b/earwigbot/commands/crypt.py @@ -39,12 +39,12 @@ class Command(BaseCommand): def process(self, data): if data.command == "crypt": msg = "available commands are !hash, !encrypt, and !decrypt." - self.connection.reply(data, msg) + self.reply(data, msg) return if not data.args: msg = "what do you want me to {0}?".format(data.command) - self.connection.reply(data, msg) + self.reply(data, msg) return if data.command == "hash": @@ -52,14 +52,14 @@ class Command(BaseCommand): if algo == "list": algos = ', '.join(hashlib.algorithms) msg = algos.join(("supported algorithms: ", ".")) - self.connection.reply(data, msg) + self.reply(data, msg) elif algo in hashlib.algorithms: string = ' '.join(data.args[1:]) result = getattr(hashlib, algo)(string).hexdigest() - self.connection.reply(data, result) + self.reply(data, result) else: msg = "unknown algorithm: '{0}'.".format(algo) - self.connection.reply(data, msg) + self.reply(data, msg) else: key = data.args[0] @@ -67,14 +67,14 @@ class Command(BaseCommand): if not text: msg = "a key was provided, but text to {0} was not." - self.connection.reply(data, msg.format(data.command)) + self.reply(data, msg.format(data.command)) return try: if data.command == "encrypt": - self.connection.reply(data, blowfish.encrypt(key, text)) + self.reply(data, blowfish.encrypt(key, text)) else: - self.connection.reply(data, blowfish.decrypt(key, text)) + self.reply(data, blowfish.decrypt(key, text)) except blowfish.BlowfishError as error: msg = "{0}: {1}.".format(error.__class__.__name__, error) - self.connection.reply(data, msg) + self.reply(data, msg) diff --git a/earwigbot/commands/ctcp.py b/earwigbot/commands/ctcp.py index 824cb4f..45455b7 100644 --- a/earwigbot/commands/ctcp.py +++ b/earwigbot/commands/ctcp.py @@ -52,17 +52,17 @@ class Command(BaseCommand): if command == "PING": msg = " ".join(data.line[4:]) if msg: - self.connection.notice(target, "\x01PING {0}\x01".format(msg)) + self.notice(target, "\x01PING {0}\x01".format(msg)) else: - self.connection.notice(target, "\x01PING\x01") + self.notice(target, "\x01PING\x01") elif command == "TIME": ts = time.strftime("%a, %d %b %Y %H:%M:%S %Z", time.localtime()) - self.connection.notice(target, "\x01TIME {0}\x01".format(ts)) + self.notice(target, "\x01TIME {0}\x01".format(ts)) elif command == "VERSION": default = "EarwigBot - $1 - Python/$2 https://github.com/earwig/earwigbot" vers = self.config.irc.get("version", default) vers = vers.replace("$1", __version__) vers = vers.replace("$2", platform.python_version()) - self.connection.notice(target, "\x01VERSION {0}\x01".format(vers)) + self.notice(target, "\x01VERSION {0}\x01".format(vers)) diff --git a/earwigbot/commands/editcount.py b/earwigbot/commands/editcount.py index 92341b0..b503d8a 100644 --- a/earwigbot/commands/editcount.py +++ b/earwigbot/commands/editcount.py @@ -49,10 +49,10 @@ class Command(BaseCommand): count = user.editcount() except wiki.UserNotFoundError: msg = "the user \x0302{0}\x0301 does not exist." - self.connection.reply(data, msg.format(name)) + self.reply(data, msg.format(name)) return safe = quote_plus(user.name()) url = "http://toolserver.org/~tparis/pcount/index.php?name={0}&lang=en&wiki=wikipedia" msg = "\x0302{0}\x0301 has {1} edits ({2})." - self.connection.reply(data, msg.format(name, count, url.format(safe))) + self.reply(data, msg.format(name, count, url.format(safe))) diff --git a/earwigbot/commands/git.py b/earwigbot/commands/git.py index c86ff70..c22d719 100644 --- a/earwigbot/commands/git.py +++ b/earwigbot/commands/git.py @@ -35,7 +35,7 @@ class Command(BaseCommand): self.data = data if data.host not in self.config.irc["permissions"]["owners"]: msg = "you must be a bot owner to use this command." - self.connection.reply(data, msg) + self.reply(data, msg) return if not data.args: @@ -65,7 +65,7 @@ class Command(BaseCommand): else: # They asked us to do something we don't know msg = "unknown argument: \x0303{0}\x0301.".format(data.args[0]) - self.connection.reply(data, msg) + self.reply(data, msg) def exec_shell(self, command): """Execute a shell command and get the output.""" @@ -89,13 +89,13 @@ class Command(BaseCommand): for key in sorted(help.keys()): msg += "\x0303{0}\x0301 ({1}), ".format(key, help[key]) msg = msg[:-2] # Trim last comma and space - self.connection.reply(self.data, "sub-commands are: {0}.".format(msg)) + self.reply(self.data, "sub-commands are: {0}.".format(msg)) def do_branch(self): """Get our current branch.""" branch = self.exec_shell("git name-rev --name-only HEAD") msg = "currently on branch \x0302{0}\x0301.".format(branch) - self.connection.reply(self.data, msg) + self.reply(self.data, msg) def do_branches(self): """Get a list of branches.""" @@ -106,14 +106,14 @@ class Command(BaseCommand): branches = branches.replace('\n ', ', ') branches = branches.strip() msg = "branches: \x0302{0}\x0301.".format(branches) - self.connection.reply(self.data, msg) + self.reply(self.data, msg) def do_checkout(self): """Switch branches.""" try: branch = self.data.args[1] except IndexError: # no branch name provided - self.connection.reply(self.data, "switch to which branch?") + self.reply(self.data, "switch to which branch?") return current_branch = self.exec_shell("git name-rev --name-only HEAD") @@ -122,51 +122,51 @@ class Command(BaseCommand): result = self.exec_shell("git checkout %s" % branch) if "Already on" in result: msg = "already on \x0302{0}\x0301!".format(branch) - self.connection.reply(self.data, msg) + self.reply(self.data, msg) else: ms = "switched from branch \x0302{1}\x0301 to \x0302{1}\x0301." msg = ms.format(current_branch, branch) - self.connection.reply(self.data, msg) + self.reply(self.data, msg) except subprocess.CalledProcessError: # Git couldn't switch branches; assume the branch doesn't exist: msg = "branch \x0302{0}\x0301 doesn't exist!".format(branch) - self.connection.reply(self.data, msg) + self.reply(self.data, msg) def do_delete(self): """Delete a branch, while making sure that we are not already on it.""" try: delete_branch = self.data.args[1] except IndexError: # no branch name provided - self.connection.reply(self.data, "delete which branch?") + self.reply(self.data, "delete which branch?") return current_branch = self.exec_shell("git name-rev --name-only HEAD") if current_branch == delete_branch: msg = "you're currently on this branch; please checkout to a different branch before deleting." - self.connection.reply(self.data, msg) + self.reply(self.data, msg) return try: self.exec_shell("git branch -d %s" % delete_branch) msg = "branch \x0302{0}\x0301 has been deleted locally." - self.connection.reply(self.data, msg.format(delete_branch)) + self.reply(self.data, msg.format(delete_branch)) except subprocess.CalledProcessError: # Git couldn't switch branches; assume the branch doesn't exist: msg = "branch \x0302{0}\x0301 doesn't exist!".format(delete_branch) - self.connection.reply(self.data, msg) + self.reply(self.data, msg) def do_pull(self): """Pull from our remote repository.""" branch = self.exec_shell("git name-rev --name-only HEAD") msg = "pulling from remote (currently on \x0302{0}\x0301)..." - self.connection.reply(self.data, msg.format(branch)) + self.reply(self.data, msg.format(branch)) result = self.exec_shell("git pull") if "Already up-to-date." in result: - self.connection.reply(self.data, "done; no new changes.") + self.reply(self.data, "done; no new changes.") else: regex = "\s*((.*?)\sfile(.*?)tions?\(-\))" changes = re.findall(regex, result)[0][0] @@ -176,11 +176,11 @@ class Command(BaseCommand): cmnd_url = "git config --get remote.{0}.url".format(remote) url = self.exec_shell(cmnd_url) msg = "done; {0} [from {1}].".format(changes, url) - self.connection.reply(self.data, msg) + self.reply(self.data, msg) except subprocess.CalledProcessError: # Something in .git/config is not specified correctly, so we # cannot get the remote's URL. However, pull was a success: - self.connection.reply(self.data, "done; %s." % changes) + self.reply(self.data, "done; %s." % changes) def do_status(self): """Check whether we have anything to pull.""" @@ -188,7 +188,7 @@ class Command(BaseCommand): result = self.exec_shell("git fetch --dry-run") if not result: # Nothing was fetched, so remote and local are equal msg = "last commit was {0}. Local copy is \x02up-to-date\x0F with remote." - self.connection.reply(self.data, msg.format(last)) + self.reply(self.data, msg.format(last)) else: msg = "last local commit was {0}. Remote is \x02ahead\x0F of local copy." - self.connection.reply(self.data, msg.format(last)) + self.reply(self.data, msg.format(last)) diff --git a/earwigbot/commands/help.py b/earwigbot/commands/help.py index 95d9e39..6897484 100644 --- a/earwigbot/commands/help.py +++ b/earwigbot/commands/help.py @@ -40,7 +40,7 @@ class Command(BaseCommand): msg = "Hi, I'm a bot! I have {0} commands loaded: {1}. You can get help for any command with '!help '." cmnds = sorted(self.bot.commands) msg = msg.format(len(cmnds), ', '.join(cmnds)) - self.connection.reply(data, msg) + self.reply(data, msg) def do_command_help(self, data): """Give the user help for a specific command.""" @@ -60,9 +60,9 @@ class Command(BaseCommand): doc = cmnd.__doc__.replace("\n", "") doc = re.sub("\s\s+", " ", doc) msg = "help for command \x0303{0}\x0301: \"{1}\"" - self.connection.reply(data, msg.format(command, doc)) + self.reply(data, msg.format(command, doc)) return break msg = "sorry, no help for \x0303{0}\x0301.".format(command) - self.connection.reply(data, msg) + self.reply(data, msg) diff --git a/earwigbot/commands/link.py b/earwigbot/commands/link.py index 675096e..6f84d44 100644 --- a/earwigbot/commands/link.py +++ b/earwigbot/commands/link.py @@ -43,15 +43,15 @@ class Command(BaseCommand): if re.search("(\[\[(.*?)\]\])|(\{\{(.*?)\}\})", msg): links = self.parse_line(msg) links = " , ".join(links) - self.connection.reply(data, links) + self.reply(data, links) elif data.command == "link": if not data.args: - self.connection.reply(data, "what do you want me to link to?") + self.reply(data, "what do you want me to link to?") return pagename = ' '.join(data.args) link = self.parse_link(pagename) - self.connection.reply(data, link) + self.reply(data, link) def parse_line(self, line): results = [] diff --git a/earwigbot/commands/praise.py b/earwigbot/commands/praise.py index c9e3950..fb611f5 100644 --- a/earwigbot/commands/praise.py +++ b/earwigbot/commands/praise.py @@ -45,7 +45,7 @@ class Command(BaseCommand): msg = "You use this command to praise certain people. Who they are is a secret." else: msg = "You're doing it wrong." - self.connection.reply(data, msg) + self.reply(data, msg) return - self.connection.say(data.chan, msg) + self.say(data.chan, msg) diff --git a/earwigbot/commands/quit.py b/earwigbot/commands/quit.py index 9b8cbc3..db3f168 100644 --- a/earwigbot/commands/quit.py +++ b/earwigbot/commands/quit.py @@ -33,8 +33,7 @@ class Command(BaseCommand): def process(self, data): if data.host not in self.config.irc["permissions"]["owners"]: - msg = "you must be a bot owner to use this command." - self.connection.reply(data, msg) + self.reply(data, "you must be a bot owner to use this command.") return if data.command == "quit": self.do_quit(data) @@ -46,7 +45,7 @@ class Command(BaseCommand): def do_quit(self, data): nick = self.config.irc.frontend["nick"] if not data.args or data.args[0].lower() != nick.lower(): - self.connection.reply(data, "to confirm this action, the first argument must be my nickname.") + self.reply(data, "to confirm this action, the first argument must be my nickname.") return if data.args[1:]: msg = " ".join(data.args[1:]) @@ -66,4 +65,4 @@ class Command(BaseCommand): self.logger.info("Reloading IRC commands and bot tasks") self.bot.commands.load() self.bot.tasks.load() - self.connection.reply(data, "IRC commands and bot tasks reloaded.") + self.reply(data, "IRC commands and bot tasks reloaded.") diff --git a/earwigbot/commands/registration.py b/earwigbot/commands/registration.py index 6db8775..ad42269 100644 --- a/earwigbot/commands/registration.py +++ b/earwigbot/commands/registration.py @@ -30,7 +30,7 @@ class Command(BaseCommand): name = "registration" def check(self, data): - commands = ["registration", "age"] + commands = ["registration", "reg", "age"] if data.is_command and data.command in commands: return True return False @@ -49,7 +49,7 @@ class Command(BaseCommand): reg = user.registration() except wiki.UserNotFoundError: msg = "the user \x0302{0}\x0301 does not exist." - self.connection.reply(data, msg.format(name)) + self.reply(data, msg.format(name)) return date = time.strftime("%b %d, %Y at %H:%M:%S UTC", reg) @@ -64,7 +64,7 @@ class Command(BaseCommand): gender = "They're" msg = "\x0302{0}\x0301 registered on {1}. {2} {3} old." - self.connection.reply(data, msg.format(name, date, gender, age)) + self.reply(data, msg.format(name, date, gender, age)) def get_diff(self, t1, t2): parts = {"years": 31536000, "days": 86400, "hours": 3600, diff --git a/earwigbot/commands/remind.py b/earwigbot/commands/remind.py index 115cb4c..930deda 100644 --- a/earwigbot/commands/remind.py +++ b/earwigbot/commands/remind.py @@ -37,19 +37,19 @@ class Command(BaseCommand): def process(self, data): if not data.args: msg = "please specify a time (in seconds) and a message in the following format: !remind