From b0da4531b2ef2cc3e525a37926c5d649c05ee841 Mon Sep 17 00:00:00 2001 From: Ben Kurtovic Date: Sun, 7 Aug 2011 01:33:52 -0400 Subject: [PATCH] tons of improvements, import fixes, cleanup, etc --- bot/classes/data.py | 6 +-- bot/commands/__init__.py | 60 +++++++++++---------- bot/config.py | 5 +- bot/frontend.py | 134 +++++++++++++++++++++++++---------------------- bot/main.py | 20 +++---- bot/tasks/__init__.py | 81 ++++++++++++++-------------- bot/watcher.py | 87 +++++++++++++++++------------- bot/watcher_logic.py | 20 +++---- earwigbot.py | 6 +-- 9 files changed, 220 insertions(+), 199 deletions(-) diff --git a/bot/classes/data.py b/bot/classes/data.py index 781c2ed..1075fc3 100644 --- a/bot/classes/data.py +++ b/bot/classes/data.py @@ -11,9 +11,9 @@ class KwargParseException(Exception): pass class Data(object): - def __init__(self): - """store data from an individual line received on IRC""" - self.line = str() + def __init__(self, line): + """Store data from an individual line received on IRC.""" + self.line = line self.chan = str() self.nick = str() self.ident = str() diff --git a/bot/commands/__init__.py b/bot/commands/__init__.py index b187185..fc25de5 100644 --- a/bot/commands/__init__.py +++ b/bot/commands/__init__.py @@ -5,29 +5,11 @@ import os import traceback -commands = [] +__all__ = ["load", "get_all", "check"] -def load_commands(connection): - """load all valid command classes from irc/commmands/ into the commands variable""" - files = os.listdir(os.path.join("irc", "commands")) # get all files in irc/commands/ - files.sort() # alphabetically sort list of files +_commands = [] - for f in files: - if f.startswith("_") or not f.endswith(".py"): # ignore non-python files or files beginning with "_" - continue - module = f[:-3] # strip .py from end - try: - exec "from irc.commands import %s" % module - except: # importing the file failed for some reason... - print "Couldn't load file %s:" % f - traceback.print_exc() - continue - process_module(connection, eval(module)) # 'module' is a string, so get the actual object for processing by eval-ing it - - pretty_cmnds = map(lambda c: c.__class__.__name__, commands) - print "Found %s command classes: %s." % (len(commands), ', '.join(pretty_cmnds)) - -def process_module(connection, module): +def _process_module(connection, module): """go through all objects in a module and add valid command classes to the commands variable""" global commands objects = dir(module) @@ -43,19 +25,41 @@ def process_module(connection, module): for base in bases: if base.__name__ == "BaseCommand": # this inherits BaseCommand, so it must be a command class command = obj(connection) # initialize a new command object - commands.append(command) + _commands.append(command) print "Added command class %s from %s..." % (this_obj, module.__name__) continue -def get_commands(): - """get our commands""" - return commands +def load(connection): + """load all valid command classes from irc/commmands/ into the commands variable""" + files = os.listdir(os.path.join("irc", "commands")) # get all files in irc/commands/ + files.sort() # alphabetically sort list of files + + for f in files: + if f.startswith("_") or not f.endswith(".py"): # ignore non-python files or files beginning with "_" + continue + module = f[:-3] # strip .py from end + try: + exec "from irc.commands import %s" % module + except: # importing the file failed for some reason... + print "Couldn't load file %s:" % f + traceback.print_exc() + continue + process_module(connection, eval(module)) # 'module' is a string, so get the actual object for processing by eval-ing it + + pretty_cmnds = map(lambda c: c.__class__.__name__, commands) + print "Found %s command classes: %s." % (len(commands), ', '.join(pretty_cmnds)) + +def get_all(): + """Return our list of all commands.""" + return _commands def check(hook, data): - """given an event on IRC, check if there's anything we can respond to by calling each command class""" - data.parse_args() # parse command arguments into data.command and data.args + """Given an event on IRC, check if there's anything we can respond to by + calling each command class""" + # parse command arguments into data.command and data.args + data.parse_args() - for command in commands: + for command in _commands: if hook in command.get_hooks(): if command.check(data): try: diff --git a/bot/config.py b/bot/config.py index b2c7cdf..82969b6 100644 --- a/bot/config.py +++ b/bot/config.py @@ -18,9 +18,9 @@ from within config's three global variables and one function: """ import json -from os import makedirs, path +from os import path -from lib import blowfish +import blowfish script_dir = path.dirname(path.abspath(__file__)) root_dir = path.split(script_dir)[0] @@ -149,7 +149,6 @@ def schedule(minute, hour, month_day, month, week_day): def make_new_config(): """Make a new config file based on the user's input.""" - makedirs(config_dir) encrypt = raw_input("Would you like to encrypt passwords stored in " + "config.json? [y/n] ") diff --git a/bot/frontend.py b/bot/frontend.py index 2a8537d..b3a6307 100644 --- a/bot/frontend.py +++ b/bot/frontend.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- """ -EarwigBot's IRC Front-end Component +EarwigBot's IRC Frontend Component The IRC frontend runs on a normal IRC server and expects users to interact with it and give it commands. Commands are stored as "command classes", subclasses @@ -9,13 +9,16 @@ of BaseCommand in irc/base_command.py. All command classes are automatically imported by irc/command_handler.py if they are in irc/commands. """ -from re import findall +import re -from core import config -from irc import command_handler -from irc.classes import Connection, Data, BrokenSocketException +import config +import commands +from classes import Connection, Data, BrokenSocketException + +__all__ = ["get_connection", "startup", "main"] connection = None +sender_regex = re.compile(":(.*?)!(.*?)@(.*?)\Z") def get_connection(): """Return a new Connection() instance with information about our server @@ -31,16 +34,18 @@ def startup(conn): command_handler, and then establish a connection with the IRC server.""" global connection connection = conn - command_handler.load_commands(connection) + commands.load(connection) connection.connect() def main(): - """Main loop for the Frontend IRC Bot component. get_connection() and - startup() should have already been called.""" + """Main loop for the frontend component. + + get_connection() and startup() should have already been called before this. + """ read_buffer = str() while 1: - try: + try: read_buffer = read_buffer + connection.get() except BrokenSocketException: print "Socket has broken on front-end; restarting bot..." @@ -48,57 +53,60 @@ def main(): lines = read_buffer.split("\n") read_buffer = lines.pop() - - for line in lines: # handle a single message from IRC - line = line.strip().split() - data = Data() # new Data() instance to store info about this line - data.line = line - - if line[1] == "JOIN": - data.nick, data.ident, data.host = findall( - ":(.*?)!(.*?)@(.*?)\Z", line[0])[0] - data.chan = line[2][1:] - command_handler.check("join", data) # check for 'join' hooks in - # our commands - - if line[1] == "PRIVMSG": - data.nick, data.ident, data.host = findall( - ":(.*?)!(.*?)@(.*?)\Z", line[0])[0] - data.msg = ' '.join(line[3:])[1:] - data.chan = line[2] - - if data.chan == config.irc["frontend"]["nick"]: - # this is a privmsg to us, so set 'chan' as the nick of the - # sender, then check for private-only command hooks - data.chan = data.nick - command_handler.check("msg_private", data) - else: - # check for public-only command hooks - command_handler.check("msg_public", data) - - # check for command hooks that apply to all messages - command_handler.check("msg", data) - - # hardcode the !restart command (we can't restart from within - # an ordinary command) - if data.msg in ["!restart", ".restart"]: - if data.host in config.irc["permissions"]["owners"]: - print "Restarting bot per owner request..." - return - - if line[0] == "PING": # if we are pinged, pong back to the server - connection.send("PONG %s" % line[1]) - - if line[1] == "376": # we've successfully connected to the network - try: # if we're supposed to auth to nickserv, do that - ns_username = config.irc["frontend"]["nickservUsername"] - ns_password = config.irc["frontend"]["nickservPassword"] - except KeyError: - pass - else: - connection.say("NickServ", "IDENTIFY {0} {1}".format( - ns_username, ns_password)) - - # join all of our startup channels - for chan in config.irc["frontend"]["channels"]: - connection.join(chan) + for line in lines: + _process_message(line) + +def _process_message(line): + """Process a single message from IRC.""" + line = line.strip().split() + data = Data(line) # new Data instance to store info about this line + + if line[1] == "JOIN": + data.nick, data.ident, data.host = sender_regex.findall(line[0])[0] + data.chan = line[2][1:] + # Check for 'join' hooks in our commands: + commands.check("join", data) + + elif line[1] == "PRIVMSG": + data.nick, data.ident, data.host = sender_regex.findall(line[0])[0] + data.msg = ' '.join(line[3:])[1:] + data.chan = line[2] + + if data.chan == config.irc["frontend"]["nick"]: + # This is a privmsg to us, so set 'chan' as the nick of the, sender + # then check for private-only command hooks: + data.chan = data.nick + commands.check("msg_private", data) + else: + # Check for public-only command hooks: + commands.check("msg_public", data) + + # Check for command hooks that apply to all messages: + commands.check("msg", data) + + # Hardcode the !restart command (we can't restart from within an + # ordinary command): + if data.msg in ["!restart", ".restart"]: + if data.host in config.irc["permissions"]["owners"]: + print "Restarting bot per owner request..." + return + + # If we are pinged, pong back to the server: + if line[0] == "PING": + connection.send("PONG %s" % line[1]) + + # On successful connection to the server: + if line[1] == "376": + # If we're supposed to auth to NickServ, do that: + try: + username = config.irc["frontend"]["nickservUsername"] + password = config.irc["frontend"]["nickservPassword"] + except KeyError: + pass + else: + msg = " ".join(("IDENTIFY", username, password)) + connection.say("NickServ", msg) + + # Join all of our startup channels: + for chan in config.irc["frontend"]["channels"]: + connection.join(chan) diff --git a/bot/main.py b/bot/main.py index 0b49842..eb892de 100644 --- a/bot/main.py +++ b/bot/main.py @@ -33,17 +33,11 @@ Else, the bot will stop, as no components are enabled. import threading import time import traceback -import sys -import os -script_dir = os.path.dirname(os.path.abspath(__file__)) -root_dir = os.path.split(script_dir)[0] # the bot's "root" directory relative - # to its different components -sys.path.append(root_dir) # make sure we look in the root dir for modules - -from core import config -from irc import frontend, watcher -from wiki import task_manager +import config +import frontend +import tasks +import watcher f_conn = None w_conn = None @@ -70,7 +64,7 @@ def wiki_scheduler(): time_start = time.time() now = time.gmtime(time_start) - task_manager.start_tasks(now) + tasks.schedule(now) time_end = time.time() time_diff = time_start - time_end @@ -89,7 +83,7 @@ def irc_frontend(): if "wiki_schedule" in config.components: print "\nStarting wiki scheduler..." - task_manager.load_tasks() + tasks.load() t_scheduler = threading.Thread(target=wiki_scheduler) t_scheduler.name = "wiki-scheduler" t_scheduler.daemon = True @@ -123,7 +117,7 @@ def run(): elif "wiki_schedule" in enabled: # run the scheduler on the main print "Starting wiki scheduler..." # thread, but also run the IRC - task_manager.load_tasks() # watcher on another thread iff it + tasks.load() # watcher on another thread iff it if "irc_watcher" in enabled: # is enabled print "\nStarting IRC watcher..." t_watcher = threading.Thread(target=irc_watcher, args=()) diff --git a/bot/tasks/__init__.py b/bot/tasks/__init__.py index 61e8386..502813a 100644 --- a/bot/tasks/__init__.py +++ b/bot/tasks/__init__.py @@ -1,9 +1,10 @@ # -*- coding: utf-8 -*- """ -EarwigBot's Wiki Bot Task Manager +EarwigBot's Wiki Task Manager -This module provides some functions to run and load bot tasks from wiki/tasks/. +This package provides the wiki bot "tasks" EarwigBot runs. Here in __init__, +you can find some functions used to load and run these tasks. """ import time @@ -11,28 +12,17 @@ import traceback import threading import os -from core import config +import config + +__all__ = ["load", "schedule", "start"] # store loaded tasks as a dict where the key is the task name and the value is # an instance of the task class (wiki.tasks.task_file.Task()) -task_list = dict() +_tasks = dict() -def load_tasks(): - """Load all valid task classes from wiki/tasks/, and add them to the - task_list variable.""" - files = os.listdir(os.path.join("wiki", "tasks")) - files.sort() # alphabetically sort all files in wiki/tasks/ - for f in files: - if not os.path.isfile(os.path.join("wiki", "tasks", f)): - continue # ignore non-files - if f.startswith("_") or not f.endswith(".py"): - continue # ignore non-python files or files beginning with an _ - load_class_from_file(f) - print "Found %s tasks: %s." % (len(task_list), ', '.join(task_list.keys())) - -def load_class_from_file(f): +def _load_class_from_file(f): """Look in a given file for the task class.""" - global task_list + global _tasks module = f[:-3] # strip .py from end try: @@ -48,45 +38,58 @@ def load_class_from_file(f): traceback.print_exc() return task_name = task_class.task_name - task_list[task_name] = task_class() - print "Added task %s from wiki/tasks/%s..." % (task_name, f) + _tasks[task_name] = task_class() + print "Added task %s from bot/tasks/%s..." % (task_name, f) + +def _wrapper(task, **kwargs): + """Wrapper for task classes: run the task and catch any errors.""" + try: + task.run(**kwargs) + except: + print "Task '{0}' raised an exception and had to stop:".format(task.task_name) + traceback.print_exc() + else: + print "Task '{0}' finished without error.".format(task.task_name) -def start_tasks(now=time.gmtime()): +def load(): + """Load all valid task classes from bot/tasks/, and add them to the + _tasks variable.""" + files = os.listdir(os.path.join("bot", "tasks")) + files.sort() # alphabetically sort all files in wiki/tasks/ + for f in files: + if not os.path.isfile(os.path.join("bot", "tasks", f)): + continue # ignore non-files + if f.startswith("_") or not f.endswith(".py"): + continue # ignore non-python files or files beginning with an _ + load_class_from_file(f) + print "Found %s tasks: %s." % (len(_tasks), ', '.join(_tasks.keys())) + +def schedule(now=time.gmtime()): """Start all tasks that are supposed to be run at a given time.""" tasks = config.schedule(now.tm_min, now.tm_hour, now.tm_mday, now.tm_mon, now.tm_wday) # get list of tasks to run this turn for task in tasks: - if isinstance(task, list): # they've specified kwargs - start_task(task[0], **task[1]) # so pass those to start_task + if isinstance(task, list): # they've specified kwargs + start(task[0], **task[1]) # so pass those to start_task else: # otherwise, just pass task_name - start_task(task) + start(task) -def start_task(task_name, **kwargs): +def start(task_name, **kwargs): """Start a given task in a new thread. Pass args to the task's run() function.""" print "Starting task '{0}' in a new thread...".format(task_name) try: - task = task_list[task_name] + task = _tasks[task_name] except KeyError: - print ("Couldn't find task '{0}': wiki/tasks/{0}.py does not exist.").format(task_name) + print ("Couldn't find task '{0}': bot/tasks/{0}.py does not exist.").format(task_name) return - task_thread = threading.Thread(target=lambda: task_wrapper(task, **kwargs)) + task_thread = threading.Thread(target=lambda: _wrapper(task, **kwargs)) task_thread.name = "{0} ({1})".format(task_name, time.strftime("%b %d %H:%M:%S")) # stop bot task threads automagically if the main bot stops task_thread.daemon = True task_thread.start() - -def task_wrapper(task, **kwargs): - """Wrapper for task classes: run the task and catch any errors.""" - try: - task.run(**kwargs) - except: - print "Task '{0}' raised an exception and had to stop:".format(task.task_name) - traceback.print_exc() - else: - print "Task '{0}' finished without error.".format(task.task_name) diff --git a/bot/watcher.py b/bot/watcher.py index b031b20..2e43099 100644 --- a/bot/watcher.py +++ b/bot/watcher.py @@ -10,25 +10,30 @@ wiki bot tasks being started (listed in wiki/tasks/) or messages being sent to channels in the IRC frontend. """ -from core import config -from irc.classes import Connection, RC, BrokenSocketException -from irc import watcher_logic +import config +from classes import Connection, RC, BrokenSocketException +import watcher_logic as logic frontend_conn = None def get_connection(): - """Return a new Connection() instance with information about our server - connection, but don't actually connect yet.""" + """Return a new Connection() instance with connection information. + + Don't actually connect yet. + """ cf = config.irc["watcher"] connection = Connection(cf["host"], cf["port"], cf["nick"], cf["ident"], cf["realname"]) return connection def main(connection, f_conn=None): - """Main loop for the Watcher IRC Bot component. get_connection() should - have already been called and the connection should have been started with - connection.connect(). Accept the frontend connection as well as an optional - parameter in order to send messages directly to frontend IRC channels.""" + """Main loop for the Watcher IRC Bot component. + + get_connection() should have already been called and the connection should + have been started with connection.connect(). Accept the frontend connection + as well as an optional parameter in order to send messages directly to + frontend IRC channels. + """ global frontend_conn frontend_conn = f_conn read_buffer = str() @@ -43,34 +48,42 @@ def main(connection, f_conn=None): read_buffer = lines.pop() for line in lines: - line = line.strip().split() - - if line[1] == "PRIVMSG": - chan = line[2] - - # ignore messages originating from channels not in our list, to - # prevent someone PMing us false data - if chan not in config.irc["watcher"]["channels"]: - continue - - msg = ' '.join(line[3:])[1:] - rc = RC(msg) # new RC object to store this event's data - rc.parse() # parse a message into pagenames, usernames, etc. - process(rc) # report to frontend channels or start tasks - - if line[0] == "PING": # if we are pinged, pong back to the server - connection.send("PONG %s" % line[1]) - - # when we've finished starting up, join all watcher channels - if line[1] == "376": - for chan in config.irc["watcher"]["channels"]: - connection.join(chan) - -def process(rc): - """Process a message from IRC (technically, an RC object). The actual - processing is configurable, so we don't have that hard-coded here. We - simply call irc/watcher_logic.py's process() function and expect a list of - channels back, which we report the event data to.""" + _process_message(line) + +def _process_message(line): + """Process a single message from IRC.""" + line = line.strip().split() + + if line[1] == "PRIVMSG": + chan = line[2] + + # Ignore messages originating from channels not in our list, to prevent + # someone PMing us false data: + if chan not in config.irc["watcher"]["channels"]: + continue + + msg = ' '.join(line[3:])[1:] + rc = RC(msg) # new RC object to store this event's data + rc.parse() # parse a message into pagenames, usernames, etc. + process_rc(rc) # report to frontend channels or start tasks + + # If we are pinged, pong back to the server: + elif line[0] == "PING": + msg = " ".join(("PONG", line[1])) + connection.send(msg) + + # When we've finished starting up, join all watcher channels: + elif line[1] == "376": + for chan in config.irc["watcher"]["channels"]: + connection.join(chan) + +def process_rc(rc): + """Process a recent change event from IRC (or, an RC object). + + The actual processing is configurable, so we don't have that hard-coded + here. We simply call watcher_logic's process() function and expect a list + of channels back, which we report the event data to. + """ chans = watcher_logic.process(rc) if chans and frontend_conn: pretty = rc.get_pretty() diff --git a/bot/watcher_logic.py b/bot/watcher_logic.py index a7a6476..c86ac9e 100644 --- a/bot/watcher_logic.py +++ b/bot/watcher_logic.py @@ -12,7 +12,7 @@ sense for this sort of thing... so... import re -from wiki import task_manager as tasks +import tasks afc_prefix = "wikipedia( talk)?:(wikiproject )?articles for creation" @@ -39,8 +39,8 @@ def process(rc): chans.update(("##earwigbot", "#wikipedia-en-afc")) if r_page.search(page_name): - tasks.start_task("afc_statistics", action="process_edit", page=rc.page) - tasks.start_task("afc_copyvios", action="process_edit", page=rc.page) + tasks.start("afc_statistics", action="process_edit", page=rc.page) + tasks.start("afc_copyvios", action="process_edit", page=rc.page) chans.add("#wikipedia-en-afc") elif r_ffu.match(page_name): @@ -50,22 +50,22 @@ def process(rc): chans.add("#wikipedia-en-afc") elif rc.flags == "move" and (r_move1.match(comment) or - r_move2.match(comment)): + r_move2.match(comment)): p = r_moved_pages.findall(rc.comment)[0] - tasks.start_task("afc_statistics", action="process_move", pages=p) - tasks.start_task("afc_copyvios", action="process_move", pages=p) + tasks.start("afc_statistics", action="process_move", pages=p) + tasks.start("afc_copyvios", action="process_move", pages=p) chans.add("#wikipedia-en-afc") elif rc.flags == "delete" and r_delete.match(comment): p = r_deleted_page.findall(rc.comment)[0] - tasks.start_task("afc_statistics", action="process_delete", page=p) - tasks.start_task("afc_copyvios", action="process_delete", page=p) + tasks.start("afc_statistics", action="process_delete", page=p) + tasks.start("afc_copyvios", action="process_delete", page=p) chans.add("#wikipedia-en-afc") elif rc.flags == "restore" and r_restore.match(comment): p = r_restored_page.findall(rc.comment)[0] - tasks.start_task("afc_statistics", action="process_restore", page=p) - tasks.start_task("afc_copyvios", action="process_restore", page=p) + tasks.start("afc_statistics", action="process_restore", page=p) + tasks.start("afc_copyvios", action="process_restore", page=p) chans.add("#wikipedia-en-afc") elif rc.flags == "protect" and r_protect.match(comment): diff --git a/earwigbot.py b/earwigbot.py index 9706378..a98f961 100755 --- a/earwigbot.py +++ b/earwigbot.py @@ -20,7 +20,7 @@ from os import path from sys import executable from time import sleep -from core.config import verify_config +from bot import config __author__ = "Ben Kurtovic" __copyright__ = "Copyright (C) 2009, 2010, 2011 by Ben Kurtovic" @@ -28,12 +28,12 @@ __license__ = "MIT License" __version__ = "0.1-dev" __email__ = "ben.kurtovic@verizon.net" -bot_script = path.join(path.dirname(path.abspath(__file__)), "core", "main.py") +bot_script = path.join(path.dirname(path.abspath(__file__)), "bot", "main.py") def main(): print "EarwigBot v{0}\n".format(__version__) - is_encrypted = verify_config() + is_encrypted = config.verify_config() if is_encrypted: # passwords in the config file are encrypted key = getpass("Enter key to unencrypt bot passwords: ") else: