From 19397290d19821483f39ffddca62e075efe4db12 Mon Sep 17 00:00:00 2001 From: Ben Kurtovic Date: Sat, 24 Mar 2012 17:51:41 -0400 Subject: [PATCH 1/5] JSON -> YAML --- .gitignore | 1 + earwigbot/config.py | 15 ++++++++------- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/.gitignore b/.gitignore index edef0ae..f3cb28b 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,7 @@ # Ignore bot-specific config file: config.json +config.yml # Ignore logs directory: logs/ diff --git a/earwigbot/config.py b/earwigbot/config.py index 9358509..486b72a 100644 --- a/earwigbot/config.py +++ b/earwigbot/config.py @@ -21,7 +21,7 @@ # SOFTWARE. """ -EarwigBot's JSON Config File Parser +EarwigBot's YAML Config File Parser This handles all tasks involving reading and writing to our config file, including encrypting and decrypting passwords and making a new config file from @@ -45,11 +45,12 @@ Additionally, _BotConfig has some functions used in config loading: variables; won't work if passwords aren't encrypted """ -import json import logging import logging.handlers from os import mkdir, path +import yaml + from earwigbot import blowfish __all__ = ["config"] @@ -90,7 +91,7 @@ class _BotConfig(object): def __init__(self): self._script_dir = path.dirname(path.abspath(__file__)) self._root_dir = path.split(self._script_dir)[0] - self._config_path = path.join(self._root_dir, "config.json") + self._config_path = path.join(self._root_dir, "config.yml") self._log_dir = path.join(self._root_dir, "logs") self._decryption_key = None self._data = None @@ -105,12 +106,12 @@ class _BotConfig(object): self._metadata] def _load(self): - """Load data from our JSON config file (config.json) into _config.""" + """Load data from our JSON config file (config.yml) into _config.""" filename = self._config_path with open(filename, 'r') as fp: try: - self._data = json.load(fp) - except ValueError as error: + self._data = yml.load(fp) + except yaml.YAMLError as error: print "Error parsing config file {0}:".format(filename) print error exit(1) @@ -158,7 +159,7 @@ class _BotConfig(object): def _make_new(self): """Make a new config file based on the user's input.""" - encrypt = raw_input("Would you like to encrypt passwords stored in config.json? [y/n] ") + encrypt = raw_input("Would you like to encrypt passwords stored in config.yml? [y/n] ") if encrypt.lower().startswith("y"): is_encrypted = True else: From 202464d9fb51e2660f202c1bbd97e720b9d12753 Mon Sep 17 00:00:00 2001 From: Ben Kurtovic Date: Sat, 24 Mar 2012 17:51:41 -0400 Subject: [PATCH 2/5] JSON -> YAML --- .gitignore | 1 + earwigbot/config.py | 15 ++++++++------- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/.gitignore b/.gitignore index edef0ae..f3cb28b 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,7 @@ # Ignore bot-specific config file: config.json +config.yml # Ignore logs directory: logs/ diff --git a/earwigbot/config.py b/earwigbot/config.py index 9358509..486b72a 100644 --- a/earwigbot/config.py +++ b/earwigbot/config.py @@ -21,7 +21,7 @@ # SOFTWARE. """ -EarwigBot's JSON Config File Parser +EarwigBot's YAML Config File Parser This handles all tasks involving reading and writing to our config file, including encrypting and decrypting passwords and making a new config file from @@ -45,11 +45,12 @@ Additionally, _BotConfig has some functions used in config loading: variables; won't work if passwords aren't encrypted """ -import json import logging import logging.handlers from os import mkdir, path +import yaml + from earwigbot import blowfish __all__ = ["config"] @@ -90,7 +91,7 @@ class _BotConfig(object): def __init__(self): self._script_dir = path.dirname(path.abspath(__file__)) self._root_dir = path.split(self._script_dir)[0] - self._config_path = path.join(self._root_dir, "config.json") + self._config_path = path.join(self._root_dir, "config.yml") self._log_dir = path.join(self._root_dir, "logs") self._decryption_key = None self._data = None @@ -105,12 +106,12 @@ class _BotConfig(object): self._metadata] def _load(self): - """Load data from our JSON config file (config.json) into _config.""" + """Load data from our JSON config file (config.yml) into _config.""" filename = self._config_path with open(filename, 'r') as fp: try: - self._data = json.load(fp) - except ValueError as error: + self._data = yml.load(fp) + except yaml.YAMLError as error: print "Error parsing config file {0}:".format(filename) print error exit(1) @@ -158,7 +159,7 @@ class _BotConfig(object): def _make_new(self): """Make a new config file based on the user's input.""" - encrypt = raw_input("Would you like to encrypt passwords stored in config.json? [y/n] ") + encrypt = raw_input("Would you like to encrypt passwords stored in config.yml? [y/n] ") if encrypt.lower().startswith("y"): is_encrypted = True else: From 8f193c9953580548843fbeef204e62dbfd9c8dc2 Mon Sep 17 00:00:00 2001 From: Ben Kurtovic Date: Sat, 24 Mar 2012 18:10:30 -0400 Subject: [PATCH 3/5] Fixing references to the config file --- earwigbot/config.py | 2 +- earwigbot/runner.py | 2 +- earwigbot/wiki/functions.py | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/earwigbot/config.py b/earwigbot/config.py index 486b72a..d86a909 100644 --- a/earwigbot/config.py +++ b/earwigbot/config.py @@ -110,7 +110,7 @@ class _BotConfig(object): filename = self._config_path with open(filename, 'r') as fp: try: - self._data = yml.load(fp) + self._data = yaml.load(fp) except yaml.YAMLError as error: print "Error parsing config file {0}:".format(filename) print error diff --git a/earwigbot/runner.py b/earwigbot/runner.py index 882b35f..2e03dfc 100644 --- a/earwigbot/runner.py +++ b/earwigbot/runner.py @@ -44,7 +44,7 @@ def run(): from earwigbot import main root_dir = raw_input() - config_path = path.join(root_dir, "config.json") + config_path = path.join(root_dir, "config.yml") log_dir = path.join(root_dir, "logs") is_encrypted = config.load(config_path, log_dir) if is_encrypted: diff --git a/earwigbot/wiki/functions.py b/earwigbot/wiki/functions.py index a870ee4..5444af2 100644 --- a/earwigbot/wiki/functions.py +++ b/earwigbot/wiki/functions.py @@ -63,8 +63,8 @@ def _get_cookiejar(): one is returned every time. The .cookies file is located in the project root, same directory as - config.json and earwigbot.py. If it doesn't exist, we will create the file - and set it to be readable and writeable only by us. If it exists but the + config.yml and bot.py. If it doesn't exist, we will create the file and set + it to be readable and writeable only by us. If it exists but the information inside is bogus, we will ignore it. This is normally called by _get_site_object_from_dict() (in turn called by From 76d1af164c63232489b4e2000ae73ed5146615cd Mon Sep 17 00:00:00 2001 From: Ben Kurtovic Date: Sat, 24 Mar 2012 20:48:42 -0400 Subject: [PATCH 4/5] str->int key conversion no longer necessary --- .gitignore | 1 - earwigbot/wiki/functions.py | 8 -------- 2 files changed, 9 deletions(-) diff --git a/.gitignore b/.gitignore index f3cb28b..5c965b9 100644 --- a/.gitignore +++ b/.gitignore @@ -2,7 +2,6 @@ *.pyc # Ignore bot-specific config file: -config.json config.yml # Ignore logs directory: diff --git a/earwigbot/wiki/functions.py b/earwigbot/wiki/functions.py index 5444af2..5504306 100644 --- a/earwigbot/wiki/functions.py +++ b/earwigbot/wiki/functions.py @@ -116,14 +116,6 @@ def _get_site_object_from_dict(name, d): user_agent = user_agent.replace("$1", earwigbot.__version__) user_agent = user_agent.replace("$2", platform.python_version()) - for key, value in namespaces.items(): # Convert string keys to integers - del namespaces[key] - try: - namespaces[int(key)] = value - except ValueError: # Data is broken, ignore it - namespaces = None - break - return Site(name=name, project=project, lang=lang, base_url=base_url, article_path=article_path, script_path=script_path, sql=sql, namespaces=namespaces, login=login, cookiejar=cookiejar, From 65a01c66b52f519946a45b2327a02462674a1f97 Mon Sep 17 00:00:00 2001 From: Ben Kurtovic Date: Sat, 24 Mar 2012 22:12:01 -0400 Subject: [PATCH 5/5] Removed earwigbot.rules; moved its contents to config.yml --- earwigbot/__init__.py | 2 +- earwigbot/config.py | 5 +++ earwigbot/irc/watcher.py | 40 +++++++++++++++++++---- earwigbot/rules.py | 85 ------------------------------------------------ 4 files changed, 39 insertions(+), 93 deletions(-) delete mode 100644 earwigbot/rules.py diff --git a/earwigbot/__init__.py b/earwigbot/__init__.py index d77092a..4dab7da 100644 --- a/earwigbot/__init__.py +++ b/earwigbot/__init__.py @@ -32,5 +32,5 @@ __version__ = "0.1.dev" __email__ = "ben.kurtovic@verizon.net" from earwigbot import ( - blowfish, commands, config, irc, main, rules, runner, tasks, tests, wiki + blowfish, commands, config, irc, main, runner, tasks, tests, wiki ) diff --git a/earwigbot/config.py b/earwigbot/config.py index d86a909..e0ef26a 100644 --- a/earwigbot/config.py +++ b/earwigbot/config.py @@ -182,6 +182,11 @@ class _BotConfig(object): @property def log_dir(self): return self._log_dir + + @property + def data(self): + """The entire config file.""" + return self._data @property def components(self): diff --git a/earwigbot/irc/watcher.py b/earwigbot/irc/watcher.py index be7328a..8d9cb4c 100644 --- a/earwigbot/irc/watcher.py +++ b/earwigbot/irc/watcher.py @@ -22,7 +22,6 @@ import logging -from earwigbot import rules from earwigbot.irc import IRCConnection, RC, BrokenSocketException from earwigbot.config import config @@ -34,7 +33,7 @@ class Watcher(IRCConnection): The IRC watcher runs on a wiki recent-changes server and listens for edits. Users cannot interact with this part of the bot. When an event - occurs, we run it through rules.py's process() function, which can result + occurs, we run it through some rules stored in our config, which can result in wiki bot tasks being started (located in tasks/) or messages being sent to channels on the IRC frontend. """ @@ -46,6 +45,7 @@ class Watcher(IRCConnection): base.__init__(cf["host"], cf["port"], cf["nick"], cf["ident"], cf["realname"], self.logger) self.frontend = frontend + self._prepare_process_hook() self._connect() def _process_message(self, line): @@ -63,7 +63,7 @@ class Watcher(IRCConnection): msg = " ".join(line[3:])[1:] rc = RC(msg) # New RC object to store this event's data rc.parse() # Parse a message into pagenames, usernames, etc. - self._process_rc(rc) # Report to frontend channels or start tasks + self._process_rc_event(rc) # If we are pinged, pong back: elif line[0] == "PING": @@ -74,14 +74,40 @@ class Watcher(IRCConnection): for chan in config.irc["watcher"]["channels"]: self.join(chan) - def _process_rc(self, rc): + def _prepare_process_hook(self): + """Create our RC event process hook from information in config. + + This will get put in the function self._process_hook, which takes an RC + object and returns a list of frontend channels to report this event to. + """ + # Default RC process hook does nothing: + self._process_hook = lambda rc: () + try: + rules = config.data["rules"] + except KeyError: + return + try: + module = compile(rules, config.config_path, "exec") + except Exception: + e = "Could not compile config file's RC event rules" + self.logger.exception(e) + return + try: + self._process_hook = module.process + except AttributeError: + e = "RC event rules compiled correctly, but no process(rc) function was found" + self.logger.error(e) + return + + def _process_rc_event(self, rc): """Process a recent change event from IRC (or, an RC object). The actual processing is configurable, so we don't have that hard-coded - here. We simply call rules's process() function and expect a list of - channels back, which we report the event data to. + here. We simply call our process hook (self._process_hook), created by + self._prepare_process_hook() from information in the "rules" section of + our config. """ - chans = rules.process(rc) + chans = self._process_hook(rc) if chans and self.frontend: pretty = rc.prettify() for chan in chans: diff --git a/earwigbot/rules.py b/earwigbot/rules.py deleted file mode 100644 index 8b58b3b..0000000 --- a/earwigbot/rules.py +++ /dev/null @@ -1,85 +0,0 @@ -# -*- coding: utf-8 -*- -# -# Copyright (C) 2009-2012 by Ben Kurtovic -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. - -""" -EarwigBot's IRC Watcher Rules - -This file contains (configurable!) rules that EarwigBot's watcher uses after it -recieves an event from IRC. -""" - -import re - -from earwigbot.tasks import task_manager - -afc_prefix = "wikipedia( talk)?:(wikiproject )?articles for creation" - -# compile some regexps used when finding specific events -r_page = re.compile(afc_prefix) -r_ffu = re.compile("wikipedia( talk)?:files for upload") -r_move1 = re.compile("moved \[\[{}".format(afc_prefix)) -r_move2 = re.compile("moved \[\[(.*?)\]\] to \[\[{}".format(afc_prefix)) -r_moved_pages = re.compile("^moved \[\[(.*?)\]\] to \[\[(.*?)\]\]") -r_delete = re.compile("deleted \"\[\[{}".format(afc_prefix)) -r_deleted_page = re.compile("^deleted \"\[\[(.*?)\]\]") -r_restore = re.compile("restored \"\[\[{}".format(afc_prefix)) -r_restored_page = re.compile("^restored \"\[\[(.*?)\]\]") -r_protect = re.compile("protected \"\[\[{}".format(afc_prefix)) - -def process(rc): - """Given an RC() object, return a list of channels to report this event to. - Also, start any wiki bot tasks within this function if necessary.""" - chans = set() # channels to report this message to - page_name = rc.page.lower() - comment = rc.comment.lower() - - if "!earwigbot" in rc.msg.lower(): - chans.update(("##earwigbot", "#wikipedia-en-afc-feed")) - - if r_page.search(page_name): - #task_manager.start("afc_copyvios", page=rc.page) - chans.add("#wikipedia-en-afc-feed") - - elif r_ffu.match(page_name): - chans.add("#wikipedia-en-afc-feed") - - elif page_name.startswith("template:afc submission"): - chans.add("#wikipedia-en-afc-feed") - - elif rc.flags == "move" and (r_move1.match(comment) or - r_move2.match(comment)): - p = r_moved_pages.findall(rc.comment)[0] - chans.add("#wikipedia-en-afc-feed") - - elif rc.flags == "delete" and r_delete.match(comment): - p = r_deleted_page.findall(rc.comment)[0] - chans.add("#wikipedia-en-afc-feed") - - elif rc.flags == "restore" and r_restore.match(comment): - p = r_restored_page.findall(rc.comment)[0] - #task_manager.start("afc_copyvios", page=p) - chans.add("#wikipedia-en-afc-feed") - - elif rc.flags == "protect" and r_protect.match(comment): - chans.add("#wikipedia-en-afc-feed") - - return chans