From 27848087cc5e5c33f8e5900c08b9c24b6d71c467 Mon Sep 17 00:00:00 2001 From: Ben Kurtovic Date: Sun, 8 Apr 2012 04:17:38 -0400 Subject: [PATCH] Daemonize task threads; clean up logging --- earwigbot/bot.py | 28 +++++++++++++++++++++++++++- earwigbot/commands/afc_report.py | 4 +++- earwigbot/config.py | 20 +++++++++++++++++++- earwigbot/irc/connection.py | 2 +- earwigbot/managers.py | 10 ++++++++-- earwigbot/tasks/afc_statistics.py | 2 +- earwigbot/util.py | 17 ++++++++++++++--- earwigbot/wiki/site.py | 2 +- 8 files changed, 74 insertions(+), 11 deletions(-) diff --git a/earwigbot/bot.py b/earwigbot/bot.py index e6c0be6..bc8fbc4 100644 --- a/earwigbot/bot.py +++ b/earwigbot/bot.py @@ -21,7 +21,7 @@ # SOFTWARE. import logging -from threading import Lock, Thread +from threading import Lock, Thread, enumerate as enumerate_threads from time import sleep, time from earwigbot.config import BotConfig @@ -104,6 +104,31 @@ class Bot(object): if self.watcher: self.watcher.stop(msg) + def _stop_task_threads(self): + """Notify the user of which task threads are going to be killed. + + Unfortunately, there is no method right now of stopping task threads + safely. This is because there is no way to tell them to stop like the + IRC components can be told; furthermore, they are run as daemons, and + daemon threads automatically stop without calling any __exit__ or + try/finally code when all non-daemon threads stop. They were originally + implemented as regular non-daemon threads, but this meant there was no + way to completely stop the bot if tasks were running, because all other + threads would exit and threading would absorb KeyboardInterrupts. + + The advantage of this is that stopping the bot is truly guarenteed to + *stop* the bot, while the disadvantage is that the tasks are given no + advance warning of their forced shutdown. + """ + tasks = [] + non_tasks = self.config.components.keys() + ["MainThread", "reminder"] + for thread in enumerate_threads(): + if thread.name not in non_tasks and thread.is_alive(): + tasks.append(thread.name) + if tasks: + log = "The following tasks will be killed: {0}" + self.logger.warn(log.format(" ".join(tasks))) + def run(self): """Main entry point into running the bot. @@ -160,3 +185,4 @@ class Bot(object): with self.component_lock: self._stop_irc_components(msg) self._keep_looping = False + self._stop_task_threads() diff --git a/earwigbot/commands/afc_report.py b/earwigbot/commands/afc_report.py index aa1b332..f44ae2b 100644 --- a/earwigbot/commands/afc_report.py +++ b/earwigbot/commands/afc_report.py @@ -37,8 +37,10 @@ class Command(BaseCommand): try: self.statistics = self.bot.tasks.get("afc_statistics") except KeyError: - e = "Cannot run command: requires afc_statistics task (from earwigbot_plugins)." + e = "Cannot run command: requires afc_statistics task (from earwigbot_plugins)" self.logger.error(e) + msg = "command requires afc_statistics task (from earwigbot_plugins)" + self.reply(data, msg) return if not data.args: diff --git a/earwigbot/config.py b/earwigbot/config.py index 035ee09..59392a1 100644 --- a/earwigbot/config.py +++ b/earwigbot/config.py @@ -301,7 +301,7 @@ class BotConfig(object): class _ConfigNode(object): def __iter__(self): - for key in self.__dict__.iterkeys(): + for key in self.__dict__: yield key def __getitem__(self, item): @@ -330,6 +330,24 @@ class _ConfigNode(object): def get(self, *args, **kwargs): return self.__dict__.get(*args, **kwargs) + def keys(self): + return self.__dict__.keys() + + def values(self): + return self.__dict__.values() + + def items(self): + return self.__dict__.items() + + def iterkeys(self): + return self.__dict__.iterkeys() + + def itervalues(self): + return self.__dict__.itervalues() + + def iteritems(self): + return self.__dict__.iteritems() + class _BotFormatter(logging.Formatter): def __init__(self, color=False): diff --git a/earwigbot/irc/connection.py b/earwigbot/irc/connection.py index 3a00383..f819534 100644 --- a/earwigbot/irc/connection.py +++ b/earwigbot/irc/connection.py @@ -53,7 +53,7 @@ class IRCConnection(object): try: self._sock.connect((self.host, self.port)) except socket.error: - self.logger.exception("Couldn't connect to IRC server") + self.logger.exception("Couldn't connect to IRC server; retrying") sleep(8) self._connect() self._send("NICK {0}".format(self.nick)) diff --git a/earwigbot/managers.py b/earwigbot/managers.py index 28da7d6..293894c 100644 --- a/earwigbot/managers.py +++ b/earwigbot/managers.py @@ -179,21 +179,27 @@ class TaskManager(_ResourceManager): self.logger.info(msg.format(task.name)) def start(self, task_name, **kwargs): - """Start a given task in a new thread. kwargs are passed to task.run""" + """Start a given task in a new daemon thread, and return the thread. + + kwargs are passed to task.run(). If the task is not found, None will be + returned. + """ msg = "Starting task '{0}' in a new thread" self.logger.info(msg.format(task_name)) try: task = self.get(task_name) except KeyError: - e = "Couldn't find task '{0}':" + e = "Couldn't find task '{0}'" self.logger.error(e.format(task_name)) return task_thread = Thread(target=self._wrapper, args=(task,), kwargs=kwargs) start_time = strftime("%b %d %H:%M:%S") task_thread.name = "{0} ({1})".format(task_name, start_time) + task_thread.daemon = True task_thread.start() + return task_thread def schedule(self, now=None): """Start all tasks that are supposed to be run at a given time.""" diff --git a/earwigbot/tasks/afc_statistics.py b/earwigbot/tasks/afc_statistics.py index 6601243..c92d517 100644 --- a/earwigbot/tasks/afc_statistics.py +++ b/earwigbot/tasks/afc_statistics.py @@ -207,7 +207,7 @@ class Task(BaseTask): replag = self.site.get_replag() self.logger.debug("Server replag is {0}".format(replag)) if replag > 600 and not kwargs.get("ignore_replag"): - msg = "Sync canceled as replag ({0} secs) is greater than ten minutes." + msg = "Sync canceled as replag ({0} secs) is greater than ten minutes" self.logger.warn(msg.format(replag)) return diff --git a/earwigbot/util.py b/earwigbot/util.py index a4d40db..bac47e2 100755 --- a/earwigbot/util.py +++ b/earwigbot/util.py @@ -26,9 +26,10 @@ This is EarwigBot's command-line utility, enabling you to easily start the bot or run specific tasks. """ -import argparse +from argparse import ArgumentParser import logging from os import path +from time import sleep from earwigbot import __version__ from earwigbot.bot import Bot @@ -37,7 +38,7 @@ __all__ = ["main"] def main(): version = "EarwigBot v{0}".format(__version__) - parser = argparse.ArgumentParser(description=__doc__) + parser = ArgumentParser(description=__doc__) parser.add_argument("path", nargs="?", metavar="PATH", default=path.curdir, help="path to the bot's working directory, which will be created if it doesn't exist; current directory assumed if not specified") parser.add_argument("-v", "--version", action="version", version=version) @@ -63,7 +64,17 @@ def main(): bot = Bot(path.abspath(args.path), level=level) if args.task: - bot.tasks.start(args.task) + thread = bot.tasks.start(args.task) + if not thread: + return + try: + while thread.is_alive(): # Keep it alive; it's a daemon + sleep(1) + except KeyboardInterrupt: + pass + finally: + if thread.is_alive(): + bot.tasks.logger.warn("The task is will be killed") else: try: bot.run() diff --git a/earwigbot/wiki/site.py b/earwigbot/wiki/site.py index f5a3aca..1f4d277 100644 --- a/earwigbot/wiki/site.py +++ b/earwigbot/wiki/site.py @@ -242,7 +242,7 @@ class Site(object): e = "Maximum number of retries reached ({0})." raise SiteAPIError(e.format(self._max_retries)) tries += 1 - msg = 'Server says: "{0}". Retrying in {1} seconds ({2}/{3}).' + msg = 'Server says "{0}"; retrying in {1} seconds ({2}/{3})' logger.info(msg.format(info, wait, tries, self._max_retries)) sleep(wait) return self._api_query(params, tries=tries, wait=wait*3)