From 93240c9b6959f7840a2a4ed992f7e98044076291 Mon Sep 17 00:00:00 2001 From: Ben Kurtovic Date: Fri, 10 Aug 2012 00:04:16 -0400 Subject: [PATCH 01/14] Starting work on configure script. --- earwigbot/config/__init__.py | 23 +++---------- earwigbot/config/script.py | 77 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 82 insertions(+), 18 deletions(-) create mode 100644 earwigbot/config/script.py diff --git a/earwigbot/config/__init__.py b/earwigbot/config/__init__.py index be29d29..c3a1e1a 100644 --- a/earwigbot/config/__init__.py +++ b/earwigbot/config/__init__.py @@ -44,6 +44,7 @@ except ImportError: from earwigbot.config.formatter import BotFormatter from earwigbot.config.node import ConfigNode from earwigbot.config.permissions import PermissionsDB +from earwigbot.config.script import ConfigScript from earwigbot.exceptions import NoConfigError __all__ = ["BotConfig"] @@ -168,20 +169,6 @@ class BotConfig(object): print "Error decrypting passwords:" raise - def _make_new(self): - """Make a new config file based on the user's input.""" - #m = "Would you like to encrypt passwords stored in config.yml? [y/n] " - #encrypt = raw_input(m) - #if encrypt.lower().startswith("y"): - # is_encrypted = True - #else: - # is_encrypted = False - raise NotImplementedError() - # yaml.dumps() config.yml file (self._config_path) - # Create root_dir/, root_dir/commands/, root_dir/tasks/ - # Give a reasonable message after config has been created regarding - # what to do next... - @property def root_dir(self): """The bot's root directory containing its config file and more.""" @@ -268,11 +255,11 @@ class BotConfig(object): """ if not path.exists(self._config_path): print "Config file not found:", self._config_path - choice = raw_input("Would you like to create a config file now? [y/n] ") - if choice.lower().startswith("y"): - self._make_new() - else: + choice = raw_input("Would you like to create a config file now? [Y/n] ") + if choice.lower().startswith("n"): raise NoConfigError() + else: + ConfigScript(self).make_new() self._load() data = self._data diff --git a/earwigbot/config/script.py b/earwigbot/config/script.py new file mode 100644 index 0000000..233593d --- /dev/null +++ b/earwigbot/config/script.py @@ -0,0 +1,77 @@ +# -*- coding: utf-8 -*- +# +# Copyright (C) 2009-2012 Ben Kurtovic +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +from getpass import getpass +import re + +try: + import bcrypt +except ImportError: + bcrypt = None + +try: + import yaml +except ImportError: + yaml = None + +__all__ = ["ConfigScript"] + +class ConfigScript(object): + """A script to guide a user through the creation of a new config file.""" + BCRYPT_ROUNDS = 12 + + def __init__(self, config): + self.config = config + self.data = {} + + def _prnt(self, msg): + pass + + def _ask_bool(self, text, default=True): + pass + + def make_new(self): + """Make a new config file based on the user's input.""" + print + self.data["metadata"] = {"version": 1} + self._print("""I can encrypt passwords stored in your config file in + addition to preventing other users on your system from + reading the file. Encryption is recommended is the bot + is to run on a public computer like the Toolserver, but + otherwise the need to enter a key everytime you start + the bot may be annoying.""") + if self._ask_bool("Encrypt stored passwords?"): + self.data["metadata"]["encryptPasswords"] = True + key = getpass("> Enter an encryption key: ") + print "Running {0} rounds of bcrypt...".format(self.BCRYPT_ROUNDS), + signature = bcrypt.hashpw(key, bcrypt.gensalt(self.BCRYPT_ROUNDS)) + self.data["metadata"]["signature"] = signature + print " done." + else: + self.data["metadata"]["encryptPasswords"] = False + + self._print("""The bot can temporarily store its logs in the logs/ + subdirectory. Error logs are kept for a month whereas + normal logs are kept for a week. If you disable this, + the bot will still print logs to stdout.""") + question = "Enable logging?" + self.data["metadata"]["enableLogging"] = self._ask_bool(question) From 69299c7b47e954a15c8e12d2b2746364650187f5 Mon Sep 17 00:00:00 2001 From: Ben Kurtovic Date: Fri, 10 Aug 2012 00:38:40 -0400 Subject: [PATCH 02/14] More of the skeleton. --- earwigbot/config/script.py | 51 ++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 47 insertions(+), 4 deletions(-) diff --git a/earwigbot/config/script.py b/earwigbot/config/script.py index 233593d..9618039 100644 --- a/earwigbot/config/script.py +++ b/earwigbot/config/script.py @@ -41,7 +41,15 @@ class ConfigScript(object): def __init__(self, config): self.config = config - self.data = {} + self.data = { + "metadata": None, + "components": None, + "wiki": None, + "irc": None, + "commands": None, + "tasks": None, + "schedule": None, + } def _prnt(self, msg): pass @@ -49,9 +57,7 @@ class ConfigScript(object): def _ask_bool(self, text, default=True): pass - def make_new(self): - """Make a new config file based on the user's input.""" - print + def _set_metadata(self): self.data["metadata"] = {"version": 1} self._print("""I can encrypt passwords stored in your config file in addition to preventing other users on your system from @@ -75,3 +81,40 @@ class ConfigScript(object): the bot will still print logs to stdout.""") question = "Enable logging?" self.data["metadata"]["enableLogging"] = self._ask_bool(question) + + def _set_components(self): + pass + + def _set_wiki(self): + pass + + def _set_irc(self): + pass + + def _set_commands(self): + pass + + def _set_tasks(self): + pass + + def _set_schedule(self): + pass + + def _save(self): + with open(self.config.path, "w") as fp: + yaml.dump(self.data, stream=fp, default_flow_style=False) + + def make_new(self): + """Make a new config file based on the user's input.""" + print + self._set_metadata() + self._set_components() + self._set_wiki() + components = self.data["components"] + if components["irc_frontend"] or components["irc_watcher"]: + self._set_irc() + self._set_commands() + self._set_tasks() + if components["wiki_scheduler"]: + self._set_schedule() + self._save() From c8418ab0b996620653b5c034dfcdd488f1d07ec2 Mon Sep 17 00:00:00 2001 From: Ben Kurtovic Date: Fri, 10 Aug 2012 01:27:23 -0400 Subject: [PATCH 03/14] Implement _print and _ask_bool. --- earwigbot/config/script.py | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/earwigbot/config/script.py b/earwigbot/config/script.py index 9618039..04d06b3 100644 --- a/earwigbot/config/script.py +++ b/earwigbot/config/script.py @@ -22,6 +22,7 @@ from getpass import getpass import re +from textwrap import fill, wrap try: import bcrypt @@ -37,6 +38,7 @@ __all__ = ["ConfigScript"] class ConfigScript(object): """A script to guide a user through the creation of a new config file.""" + WIDTH = 79 BCRYPT_ROUNDS = 12 def __init__(self, config): @@ -51,11 +53,26 @@ class ConfigScript(object): "schedule": None, } - def _prnt(self, msg): - pass + def _print(self, msg): + print fill(re.sub("\s\s+", " ", msg), self.WIDTH) def _ask_bool(self, text, default=True): - pass + text = "> " + text + if default: + text += " [Y/n] " + else: + text += " [y/N] " + lines = wrap(re.sub("\s\s+", " ", msg), self.WIDTH) + if len(lines) > 1: + print "\n".join(lines[:-1]) + while True: + answer = raw_input(lines[-1]).lower() + if not answer: + return default + if answer.startswith("y"): + return True + if answer.startswith("n"): + return False def _set_metadata(self): self.data["metadata"] = {"version": 1} From dc2d69dd36d65f2c602058a59c15311a7c1ffdac Mon Sep 17 00:00:00 2001 From: Ben Kurtovic Date: Fri, 10 Aug 2012 02:39:31 -0400 Subject: [PATCH 04/14] Implement _set_components(). --- earwigbot/config/script.py | 52 +++++++++++++++++++++++++++++++++------------- 1 file changed, 37 insertions(+), 15 deletions(-) diff --git a/earwigbot/config/script.py b/earwigbot/config/script.py index 04d06b3..e5a4c08 100644 --- a/earwigbot/config/script.py +++ b/earwigbot/config/script.py @@ -20,6 +20,7 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. +from collections import OrderedDict from getpass import getpass import re from textwrap import fill, wrap @@ -43,15 +44,15 @@ class ConfigScript(object): def __init__(self, config): self.config = config - self.data = { - "metadata": None, - "components": None, - "wiki": None, - "irc": None, - "commands": None, - "tasks": None, - "schedule": None, - } + self.data = OrderedDict( + ("metadata", OrderedDict()), + ("components", OrderedDict()), + ("wiki", OrderedDict()), + ("irc", OrderedDict()), + ("commands", OrderedDict()), + ("tasks", OrderedDict()), + ("schedule", []) + ) def _print(self, msg): print fill(re.sub("\s\s+", " ", msg), self.WIDTH) @@ -59,14 +60,14 @@ class ConfigScript(object): def _ask_bool(self, text, default=True): text = "> " + text if default: - text += " [Y/n] " + text += " [Y/n]" else: - text += " [y/N] " + text += " [y/N]" lines = wrap(re.sub("\s\s+", " ", msg), self.WIDTH) if len(lines) > 1: print "\n".join(lines[:-1]) while True: - answer = raw_input(lines[-1]).lower() + answer = raw_input(lines[-1] + " ").lower() if not answer: return default if answer.startswith("y"): @@ -75,7 +76,8 @@ class ConfigScript(object): return False def _set_metadata(self): - self.data["metadata"] = {"version": 1} + print + self.data["metadata"] = OrderedDict(("version", 1)) self._print("""I can encrypt passwords stored in your config file in addition to preventing other users on your system from reading the file. Encryption is recommended is the bot @@ -100,7 +102,28 @@ class ConfigScript(object): self.data["metadata"]["enableLogging"] = self._ask_bool(question) def _set_components(self): - pass + print + self._print("""The bot contains three separate components that can run + independently of each other.""") + self._print("""- The IRC front-end runs on a normal IRC server, like + freenode, and expects users to interact with it through + commands.""") + self._print("""- The IRC watcher runs on a wiki recent-changes server, + like irc.wikimedia.org, and listens for edits. Users + cannot interact with this component. It can detect + specific events and report them to "feed" channels on + the front-end, or start bot tasks.""") + self._pritn("""- The wiki task scheduler runs wiki-editing bot tasks in + separate threads at user-defined times through a + cron-like interface. Tasks which are not scheduled can + be started by the IRC watcher manually through the IRC + front-end.""") + frontend = self._ask_bool("Enable the IRC front-end?") + watcher = self._ask_bool("Enable the IRC watcher?") + scheduler = self._ask_bool("Enable the wiki task scheduler?") + self.data["components"]["irc_frontend"] = frontend + self.data["components"]["irc_watcher"] = watcher + self.data["components"]["wiki_scheduler"] = scheduler def _set_wiki(self): pass @@ -123,7 +146,6 @@ class ConfigScript(object): def make_new(self): """Make a new config file based on the user's input.""" - print self._set_metadata() self._set_components() self._set_wiki() From 15193b0f634a6bd9a65032ab10fc5519f22ca3cd Mon Sep 17 00:00:00 2001 From: Ben Kurtovic Date: Fri, 10 Aug 2012 03:36:01 -0400 Subject: [PATCH 05/14] Parts of _set_wiki(); expanding. --- earwigbot/config/script.py | 41 +++++++++++++++++++++++++++++++++++++++-- 1 file changed, 39 insertions(+), 2 deletions(-) diff --git a/earwigbot/config/script.py b/earwigbot/config/script.py index e5a4c08..d268163 100644 --- a/earwigbot/config/script.py +++ b/earwigbot/config/script.py @@ -113,7 +113,7 @@ class ConfigScript(object): cannot interact with this component. It can detect specific events and report them to "feed" channels on the front-end, or start bot tasks.""") - self._pritn("""- The wiki task scheduler runs wiki-editing bot tasks in + self._print("""- The wiki task scheduler runs wiki-editing bot tasks in separate threads at user-defined times through a cron-like interface. Tasks which are not scheduled can be started by the IRC watcher manually through the IRC @@ -126,15 +126,44 @@ class ConfigScript(object): self.data["components"]["wiki_scheduler"] = scheduler def _set_wiki(self): - pass + print + wmf = self._ask_bool("""Will this bot run on Wikimedia Foundation + wikis, like Wikipedia?""") + if wmf: + sitename = ? # setup sites.db + else: + sitename = ? # setup sites.db + self.data["wiki"]["username"] = raw_input("> Bot username: ") + self.data["wiki"]["password"] = getpass("> Bot password: ") + self.data["wiki"]["userAgent"] = "EarwigBot/$1 (Python/$2; https://github.com/earwig/earwigbot)" + self.data["wiki"]["summary"] = "([[WP:BOT|Bot]]): $2" + shutoff + self.data["wiki"]["useHTTPS"] = True + self.data["wiki"]["assert"] = "user" + self.data["wiki"]["maxlag"] = 10 + self.data["wiki"]["waitTime"] = 2 + self.data["wiki"]["defaultSite"] = sitename + ts = self._ask_bool("Will this bot run from the Wikimedia Toolserver?") + if ts: + args = (("host", "$1-p.rrdb.toolserver.org"), ("db": "$1_p")) + self.data["wiki"]["sql"] = OrderedDict(args) + else: + self.data["wiki"]["sql"] = {} + self.data["wiki"]["search"] = {} def _set_irc(self): + # create permissions.db with us if frontend + # create rules.py if watcher pass def _set_commands(self): + # disable: True if no IRC frontend or prompted + # create commands/ pass def _set_tasks(self): + # disable: True if prompted + # create tasks/ pass def _set_schedule(self): @@ -156,4 +185,12 @@ class ConfigScript(object): self._set_tasks() if components["wiki_scheduler"]: self._set_schedule() + self._print("""I am now saving config.yml with your settings. YAML is a + relatively straightforward format and you should be able + to update these settings in the future when necessary. + I will start the bot at your signal. Feel free to + contact me at wikipedia.earwig at gmail.com if you have + any questions.""") self._save() + if not self._ask_bool("Start the bot now?"): + exit() From 7d7205265eee300946c31e9406a5a13295962c17 Mon Sep 17 00:00:00 2001 From: Ben Kurtovic Date: Fri, 10 Aug 2012 04:39:07 -0400 Subject: [PATCH 06/14] Implement the rest of _set_wiki(); _ask(); other tweaks. --- earwigbot/bot.py | 2 +- earwigbot/config/__init__.py | 8 +++- earwigbot/config/script.py | 88 ++++++++++++++++++++++++++++++++++++++------ earwigbot/wiki/sitesdb.py | 2 +- 4 files changed, 85 insertions(+), 15 deletions(-) diff --git a/earwigbot/bot.py b/earwigbot/bot.py index bb6b213..56042a3 100644 --- a/earwigbot/bot.py +++ b/earwigbot/bot.py @@ -60,7 +60,7 @@ class Bot(object): """ def __init__(self, root_dir, level=logging.INFO): - self.config = BotConfig(root_dir, level) + self.config = BotConfig(self, root_dir, level) self.logger = logging.getLogger("earwigbot") self.commands = CommandManager(self) self.tasks = TaskManager(self) diff --git a/earwigbot/config/__init__.py b/earwigbot/config/__init__.py index c3a1e1a..2b8ea3c 100644 --- a/earwigbot/config/__init__.py +++ b/earwigbot/config/__init__.py @@ -76,7 +76,8 @@ class BotConfig(object): - :py:meth:`decrypt`: decrypts an object in the config tree """ - def __init__(self, root_dir, level): + def __init__(self, bot, root_dir, level): + self._bot = bot self._root_dir = root_dir self._logging_level = level self._config_path = path.join(self.root_dir, "config.yml") @@ -170,6 +171,11 @@ class BotConfig(object): raise @property + def bot(self): + """The config's Bot object.""" + return self._bot + + @property def root_dir(self): """The bot's root directory containing its config file and more.""" return self._root_dir diff --git a/earwigbot/config/script.py b/earwigbot/config/script.py index d268163..40e8ff2 100644 --- a/earwigbot/config/script.py +++ b/earwigbot/config/script.py @@ -35,6 +35,8 @@ try: except ImportError: yaml = None +from earwigbot import exceptions + __all__ = ["ConfigScript"] class ConfigScript(object): @@ -57,6 +59,15 @@ class ConfigScript(object): def _print(self, msg): print fill(re.sub("\s\s+", " ", msg), self.WIDTH) + def _ask(self, text, default=None): + text = "> " + text + if default: + text += " [{0}]".format(default) + lines = wrap(re.sub("\s\s+", " ", msg), self.WIDTH) + if len(lines) > 1: + print "\n".join(lines[:-1]) + return raw_input(lines[-1] + " ") or default + def _ask_bool(self, text, default=True): text = "> " + text if default: @@ -125,30 +136,83 @@ class ConfigScript(object): self.data["components"]["irc_watcher"] = watcher self.data["components"]["wiki_scheduler"] = scheduler + def _login(self, kwargs): + self.config.wiki._load(self.data["wiki"]) + print "Trying to login to the site...", + try: + site = self.config.bot.wiki.add_site(**kargs) + except exceptions.APIError: + print " API error!" + question = "Would you like to re-enter the site information?" + if self._ask_bool(question): + return self._set_wiki() + question = "This will cancel the setup process. Are you sure?" + if self._ask_bool(question, default=False): + raise exceptions.NoConfigError() + return self._set_wiki() + except exceptions.LoginError: + print " login error!" + question = "Would you like to re-enter your login information?" + if self._ask_bool(question): + self.data["wiki"]["username"] = self._ask("Bot username:") + self.data["wiki"]["password"] = getpass("> Bot password: ") + return self._login(kwargs) + question = "Would you like to re-enter the site information?" + if self._ask_bool(question): + return self._set_wiki() + self._print("""Moving on. You can modify the login information + stored in the bot's config in the future.""") + else: + print " success." + return site + def _set_wiki(self): print wmf = self._ask_bool("""Will this bot run on Wikimedia Foundation wikis, like Wikipedia?""") if wmf: - sitename = ? # setup sites.db + msg = "Site project (e.g. 'wikipedia', 'wiktionary', 'wikimedia'):" + project = self._ask(msg, default="wikipedia").lower() + msg = "Site language code (e.g. 'en', 'fr', 'commons'):" + lang = self._ask(msg, default="en").lower() + kwargs = {"project": project, "lang": lang} else: - sitename = ? # setup sites.db - self.data["wiki"]["username"] = raw_input("> Bot username: ") + msg = "Site base URL, without the script path and trailing slash;" + msg += " can be protocol-insensitive (e.g. '//en.wikipedia.org'):" + url = self._ask(msg) + script = self._ask("Site script path:", default="/w") + kwargs = {"base_url": url, "script_path": script, "sql": sql} + + self.data["wiki"]["username"] = self._ask("Bot username:") self.data["wiki"]["password"] = getpass("> Bot password: ") self.data["wiki"]["userAgent"] = "EarwigBot/$1 (Python/$2; https://github.com/earwig/earwigbot)" self.data["wiki"]["summary"] = "([[WP:BOT|Bot]]): $2" - shutoff self.data["wiki"]["useHTTPS"] = True self.data["wiki"]["assert"] = "user" self.data["wiki"]["maxlag"] = 10 - self.data["wiki"]["waitTime"] = 2 - self.data["wiki"]["defaultSite"] = sitename - ts = self._ask_bool("Will this bot run from the Wikimedia Toolserver?") - if ts: - args = (("host", "$1-p.rrdb.toolserver.org"), ("db": "$1_p")) - self.data["wiki"]["sql"] = OrderedDict(args) - else: - self.data["wiki"]["sql"] = {} + self.data["wiki"]["waitTime"] = 3 + self.data["wiki"]["defaultSite"] = self._login(kwargs).name + self.data["wiki"]["sql"] = {} + + if wmf: + msg = "Will this bot run from the Wikimedia Toolserver?" + toolserver = self._ask_bool(msg, default=False) + if toolserver: + args = (("host", "$1-p.rrdb.toolserver.org"), ("db": "$1_p")) + self.data["wiki"]["sql"] = OrderedDict(args) + + self.data["wiki"]["shutoff"] = {} + msg = "Would you like to enable an automatic shutoff page for the bot?" + if self._ask_bool(msg): + self._print("""The page title can contain two wildcards: $1 will be + substituted with the bot's username, and $2 with the + current task number. This can be used to implement a + separate shutoff page for each task.""") + page = self._ask("Page title:", default="User:$1/Shutoff") + disabled = self._ask("Page content when *not* shut off:", "run") + args = (("page", page), ("disabled", disabled)) + self.data["wiki"]["shutoff"] = OrderedDict(args) + self.data["wiki"]["search"] = {} def _set_irc(self): diff --git a/earwigbot/wiki/sitesdb.py b/earwigbot/wiki/sitesdb.py index 00d07b1..3724d4b 100644 --- a/earwigbot/wiki/sitesdb.py +++ b/earwigbot/wiki/sitesdb.py @@ -386,7 +386,7 @@ class SitesDB(object): config = self.config login = (config.wiki.get("username"), config.wiki.get("password")) user_agent = config.wiki.get("userAgent") - use_https = config.wiki.get("useHTTPS", False) + use_https = config.wiki.get("useHTTPS", True) assert_edit = config.wiki.get("assert") maxlag = config.wiki.get("maxlag") wait_between_queries = config.wiki.get("waitTime", 3) From 987d140ef3eb50cfa77f34bbfdf2e51e530e6b34 Mon Sep 17 00:00:00 2001 From: Ben Kurtovic Date: Fri, 10 Aug 2012 16:56:35 -0400 Subject: [PATCH 07/14] Most of _set_irc(). --- earwigbot/config/script.py | 106 +++++++++++++++++++++++++++++++++++++-------- 1 file changed, 88 insertions(+), 18 deletions(-) diff --git a/earwigbot/config/script.py b/earwigbot/config/script.py index 40e8ff2..f4b912a 100644 --- a/earwigbot/config/script.py +++ b/earwigbot/config/script.py @@ -22,7 +22,9 @@ from collections import OrderedDict from getpass import getpass +from os import chmod import re +import stat from textwrap import fill, wrap try: @@ -56,14 +58,18 @@ class ConfigScript(object): ("schedule", []) ) - def _print(self, msg): - print fill(re.sub("\s\s+", " ", msg), self.WIDTH) + self.wmf = False + self.proj = None + self.lang = None + + def _print(self, text): + print fill(re.sub("\s\s+", " ", text), self.WIDTH) def _ask(self, text, default=None): text = "> " + text if default: text += " [{0}]".format(default) - lines = wrap(re.sub("\s\s+", " ", msg), self.WIDTH) + lines = wrap(re.sub("\s\s+", " ", text), self.WIDTH) if len(lines) > 1: print "\n".join(lines[:-1]) return raw_input(lines[-1] + " ") or default @@ -74,7 +80,7 @@ class ConfigScript(object): text += " [Y/n]" else: text += " [y/N]" - lines = wrap(re.sub("\s\s+", " ", msg), self.WIDTH) + lines = wrap(re.sub("\s\s+", " ", text), self.WIDTH) if len(lines) > 1: print "\n".join(lines[:-1]) while True: @@ -86,6 +92,9 @@ class ConfigScript(object): if answer.startswith("n"): return False + def _ask_list(self, text): + pass + def _set_metadata(self): print self.data["metadata"] = OrderedDict(("version", 1)) @@ -140,7 +149,7 @@ class ConfigScript(object): self.config.wiki._load(self.data["wiki"]) print "Trying to login to the site...", try: - site = self.config.bot.wiki.add_site(**kargs) + site = self.config.bot.wiki.add_site(**kwargs) except exceptions.APIError: print " API error!" question = "Would you like to re-enter the site information?" @@ -168,20 +177,20 @@ class ConfigScript(object): def _set_wiki(self): print - wmf = self._ask_bool("""Will this bot run on Wikimedia Foundation - wikis, like Wikipedia?""") - if wmf: + self.wmf = self._ask_bool("""Will this bot run on Wikimedia Foundation + wikis, like Wikipedia?""") + if self.wmf: msg = "Site project (e.g. 'wikipedia', 'wiktionary', 'wikimedia'):" - project = self._ask(msg, default="wikipedia").lower() + self.proj = project = self._ask(msg, default="wikipedia").lower() msg = "Site language code (e.g. 'en', 'fr', 'commons'):" - lang = self._ask(msg, default="en").lower() + self.lang = lang = self._ask(msg, default="en").lower() kwargs = {"project": project, "lang": lang} else: msg = "Site base URL, without the script path and trailing slash;" msg += " can be protocol-insensitive (e.g. '//en.wikipedia.org'):" url = self._ask(msg) script = self._ask("Site script path:", default="/w") - kwargs = {"base_url": url, "script_path": script, "sql": sql} + kwargs = {"base_url": url, "script_path": script} self.data["wiki"]["username"] = self._ask("Bot username:") self.data["wiki"]["password"] = getpass("> Bot password: ") @@ -194,11 +203,11 @@ class ConfigScript(object): self.data["wiki"]["defaultSite"] = self._login(kwargs).name self.data["wiki"]["sql"] = {} - if wmf: + if self.wmf: msg = "Will this bot run from the Wikimedia Toolserver?" toolserver = self._ask_bool(msg, default=False) if toolserver: - args = (("host", "$1-p.rrdb.toolserver.org"), ("db": "$1_p")) + args = (("host", "$1-p.rrdb.toolserver.org"), ("db", "$1_p")) self.data["wiki"]["sql"] = OrderedDict(args) self.data["wiki"]["shutoff"] = {} @@ -216,9 +225,68 @@ class ConfigScript(object): self.data["wiki"]["search"] = {} def _set_irc(self): - # create permissions.db with us if frontend - # create rules.py if watcher - pass + if self.data["components"]["irc_frontend"]: + print + frontend = self.data["irc"]["frontend"] = OrderedDict() + msg = "Hostname of the frontend's IRC server, without 'irc://'" + frontend["host"] = self._ask(msg, "irc.freenode.net") + frontend["port"] = self._ask("Frontend port", 6667) + frontend["nick"] = self._ask("Frontend bot's nickname") + frontend["ident"] = self._ask("Frontend bot's ident", + frontend["nick"].lower()) + question = "Frontend bot's real name (gecos)" + frontend["realname"] = self._ask(question) + if self._ask_bool("Should the bot identify to NickServ?"): + frontend["nickservUsername"] = self._ask("NickServ username", + frontend["nick"]) + frontend["nickservPassword"] = getpass("> Nickserv password: ") + chan_question = "Frontend channels to join by default" + frontend["channels"] = self._ask_list(chan_question) + self._print("""The bot keeps a database of its admins (users who + can use certain sensitive commands) and owners + (users who can quit the bot and modify its access + list), identified by nick, ident, and/or hostname. + Hostname is most secure since it cannot be easily + spoofed. If you have a cloak, it will probably look + like 'wikipedia/Username' or + 'unaffiliated/nickname'.""") + host = self._ask("Your hostname on the IRC frontend") + if host: + self.config._permissions.load() + self.config._permissions.add_owner(host=host) + self.config._permissions.add_admin(host=host) + else: + frontend = {} + + if self.data["components"]["irc_watcher"]: + print + watcher = self.data["irc"]["watcher"] = OrderedDict() + if self.wmf: + watcher["host"] = "irc.wikimedia.org" + watcher["port"] = 6667 + else: + msg = "Hostname of the watcher's IRC server, without 'irc://'" + watcher["host"] = self._ask(msg) + watcher["port"] = self._ask("Watcher port", 6667) + watcher["nick"] = self._ask("Watcher bot's nickname", + frontend.get("nick")) + watcher["ident"] = self._ask("Watcher bot's ident", + watcher["nick"].lower()) + question = "Watcher bot's real name (gecos)" + watcher["realname"] = self._ask(question, frontend.get("realname")) + watcher_ns = "Should the bot identify to NickServ?" + if not self.wmf and self._ask_bool(watcher_ns): + watcher["nickservUsername"] = self._ask("NickServ username", + watcher["nick"]) + watcher["nickservPassword"] = getpass("> Nickserv password: ") + if self.wmf: + watcher["channels"] = ["#{0}.{1}".format(self.lang, self.proj)] + else: + chan_question = "Watcher channels to join" + watcher["channels"] = self._ask_list(chan_question) + # create rules.py + + self.data["irc"]["version"] = "EarwigBot - $1 - Python/$2 https://github.com/earwig/earwigbot" def _set_commands(self): # disable: True if no IRC frontend or prompted @@ -234,8 +302,10 @@ class ConfigScript(object): pass def _save(self): - with open(self.config.path, "w") as fp: - yaml.dump(self.data, stream=fp, default_flow_style=False) + open(self.config.path, "w").close() + chmod(self.config.path, stat.S_IRUSR|stat.S_IWUSR) + with open(self.config.path, "w") as stream: + yaml.dump(self.data, stream=stream, default_flow_style=False) def make_new(self): """Make a new config file based on the user's input.""" From 5856ce3a2e8ec0780d4baecb3bea403b3651ddd0 Mon Sep 17 00:00:00 2001 From: Ben Kurtovic Date: Fri, 10 Aug 2012 17:12:34 -0400 Subject: [PATCH 08/14] Create 'rules.py' template; do a check before we start that we can write to the config file. --- earwigbot/config/__init__.py | 3 ++- earwigbot/config/script.py | 33 +++++++++++++++++++++++++++------ 2 files changed, 29 insertions(+), 7 deletions(-) diff --git a/earwigbot/config/__init__.py b/earwigbot/config/__init__.py index 2b8ea3c..ecefd16 100644 --- a/earwigbot/config/__init__.py +++ b/earwigbot/config/__init__.py @@ -25,6 +25,7 @@ from hashlib import sha256 import logging import logging.handlers from os import mkdir, path +import stat try: from Crypto.Cipher import Blowfish @@ -139,7 +140,7 @@ class BotConfig(object): if not path.isdir(log_dir): if not path.exists(log_dir): - mkdir(log_dir, 0700) + mkdir(log_dir, stat.S_IWUSR|stat.S_IRUSR|stat.S_IXUSR) else: msg = "log_dir ({0}) exists but is not a directory!" print msg.format(log_dir) diff --git a/earwigbot/config/script.py b/earwigbot/config/script.py index f4b912a..bdf6335 100644 --- a/earwigbot/config/script.py +++ b/earwigbot/config/script.py @@ -22,7 +22,7 @@ from collections import OrderedDict from getpass import getpass -from os import chmod +from os import chmod, path import re import stat from textwrap import fill, wrap @@ -41,6 +41,15 @@ from earwigbot import exceptions __all__ = ["ConfigScript"] +RULES_TEMPLATE = """"# -*- coding: utf-8 -*- + +def process(bot, rc): + \"\"\"Given a Bot() object and an RC() object, return a list of channels + to report this event to. Also, start any wiki bot tasks within this + function if necessary.\"\"\" + pass +""" + class ConfigScript(object): """A script to guide a user through the creation of a new config file.""" WIDTH = 79 @@ -118,8 +127,8 @@ class ConfigScript(object): subdirectory. Error logs are kept for a month whereas normal logs are kept for a week. If you disable this, the bot will still print logs to stdout.""") - question = "Enable logging?" - self.data["metadata"]["enableLogging"] = self._ask_bool(question) + logging = self._ask_bool("Enable logging?") + self.data["metadata"]["enableLogging"] = logging def _set_components(self): print @@ -284,7 +293,15 @@ class ConfigScript(object): else: chan_question = "Watcher channels to join" watcher["channels"] = self._ask_list(chan_question) - # create rules.py + self._print("""I am now creating a blank 'rules.py' file, which + will determine how the bot handles messages received + from the IRC watcher. It contains a process() + function that takes a Bot object allowing you to + start tasks and an RC object that holds the message + from the watcher. See the documentation for + details.""") + with open(path.join(self.config.root_dir, "rules.py"), "w") as fp: + fp.write(RULES_TEMPLATE) self.data["irc"]["version"] = "EarwigBot - $1 - Python/$2 https://github.com/earwig/earwigbot" @@ -302,13 +319,17 @@ class ConfigScript(object): pass def _save(self): - open(self.config.path, "w").close() - chmod(self.config.path, stat.S_IRUSR|stat.S_IWUSR) with open(self.config.path, "w") as stream: yaml.dump(self.data, stream=stream, default_flow_style=False) def make_new(self): """Make a new config file based on the user's input.""" + try: + open(self.config.path, "w").close() + chmod(self.config.path, stat.S_IRUSR|stat.S_IWUSR) + except IOError: + print "I can't seem to write to the config file:" + raise self._set_metadata() self._set_components() self._set_wiki() From f3f372f550ae1d19c49aeb85c6d07613e2eec74c Mon Sep 17 00:00:00 2001 From: Ben Kurtovic Date: Fri, 10 Aug 2012 18:00:29 -0400 Subject: [PATCH 09/14] Finishing everything; some other tweaks. --- earwigbot/config/script.py | 93 +++++++++++++++++++++++++++++++++------------- earwigbot/managers.py | 8 +++- 2 files changed, 73 insertions(+), 28 deletions(-) diff --git a/earwigbot/config/script.py b/earwigbot/config/script.py index bdf6335..972def8 100644 --- a/earwigbot/config/script.py +++ b/earwigbot/config/script.py @@ -22,7 +22,7 @@ from collections import OrderedDict from getpass import getpass -from os import chmod, path +from os import chmod, mkdir, path import re import stat from textwrap import fill, wrap @@ -74,6 +74,9 @@ class ConfigScript(object): def _print(self, text): print fill(re.sub("\s\s+", " ", text), self.WIDTH) + def _pause(self): + raw_input("> Press enter to continue: ") + def _ask(self, text, default=None): text = "> " + text if default: @@ -102,7 +105,15 @@ class ConfigScript(object): return False def _ask_list(self, text): - pass + print fill(re.sub("\s\s+", " ", "> " + text), self.WIDTH) + print "[one item per line; blank line to end]:" + result = [] + while True: + line = raw_input("> ") + if line: + result.append(line) + else: + return result def _set_metadata(self): print @@ -237,19 +248,19 @@ class ConfigScript(object): if self.data["components"]["irc_frontend"]: print frontend = self.data["irc"]["frontend"] = OrderedDict() - msg = "Hostname of the frontend's IRC server, without 'irc://'" + msg = "Hostname of the frontend's IRC server, without 'irc://':" frontend["host"] = self._ask(msg, "irc.freenode.net") - frontend["port"] = self._ask("Frontend port", 6667) - frontend["nick"] = self._ask("Frontend bot's nickname") - frontend["ident"] = self._ask("Frontend bot's ident", + frontend["port"] = self._ask("Frontend port:", 6667) + frontend["nick"] = self._ask("Frontend bot's nickname:") + frontend["ident"] = self._ask("Frontend bot's ident:", frontend["nick"].lower()) - question = "Frontend bot's real name (gecos)" + question = "Frontend bot's real name (gecos):" frontend["realname"] = self._ask(question) if self._ask_bool("Should the bot identify to NickServ?"): - frontend["nickservUsername"] = self._ask("NickServ username", + frontend["nickservUsername"] = self._ask("NickServ username:", frontend["nick"]) frontend["nickservPassword"] = getpass("> Nickserv password: ") - chan_question = "Frontend channels to join by default" + chan_question = "Frontend channels to join by default:" frontend["channels"] = self._ask_list(chan_question) self._print("""The bot keeps a database of its admins (users who can use certain sensitive commands) and owners @@ -259,7 +270,7 @@ class ConfigScript(object): spoofed. If you have a cloak, it will probably look like 'wikipedia/Username' or 'unaffiliated/nickname'.""") - host = self._ask("Your hostname on the IRC frontend") + host = self._ask("Your hostname on the IRC frontend:") if host: self.config._permissions.load() self.config._permissions.add_owner(host=host) @@ -274,24 +285,24 @@ class ConfigScript(object): watcher["host"] = "irc.wikimedia.org" watcher["port"] = 6667 else: - msg = "Hostname of the watcher's IRC server, without 'irc://'" + msg = "Hostname of the watcher's IRC server, without 'irc://':" watcher["host"] = self._ask(msg) - watcher["port"] = self._ask("Watcher port", 6667) - watcher["nick"] = self._ask("Watcher bot's nickname", + watcher["port"] = self._ask("Watcher port:", 6667) + watcher["nick"] = self._ask("Watcher bot's nickname:", frontend.get("nick")) - watcher["ident"] = self._ask("Watcher bot's ident", + watcher["ident"] = self._ask("Watcher bot's ident:", watcher["nick"].lower()) - question = "Watcher bot's real name (gecos)" + question = "Watcher bot's real name (gecos):" watcher["realname"] = self._ask(question, frontend.get("realname")) watcher_ns = "Should the bot identify to NickServ?" if not self.wmf and self._ask_bool(watcher_ns): - watcher["nickservUsername"] = self._ask("NickServ username", + watcher["nickservUsername"] = self._ask("NickServ username:", watcher["nick"]) watcher["nickservPassword"] = getpass("> Nickserv password: ") if self.wmf: watcher["channels"] = ["#{0}.{1}".format(self.lang, self.proj)] else: - chan_question = "Watcher channels to join" + chan_question = "Watcher channels to join by default:" watcher["channels"] = self._ask_list(chan_question) self._print("""I am now creating a blank 'rules.py' file, which will determine how the bot handles messages received @@ -302,21 +313,51 @@ class ConfigScript(object): details.""") with open(path.join(self.config.root_dir, "rules.py"), "w") as fp: fp.write(RULES_TEMPLATE) + self._pause() self.data["irc"]["version"] = "EarwigBot - $1 - Python/$2 https://github.com/earwig/earwigbot" def _set_commands(self): - # disable: True if no IRC frontend or prompted - # create commands/ - pass + print + msg = """Would you like to disable the default IRC commands? You can + fine-tune which commands are disabled later on.""" + if (not self.data["components"]["irc_frontend"] or + self._ask_bool(msg, default=False)): + self.data["commands"]["disable"] = True + self._print("""I am now creating the 'commands/' directory, where you + can place custom IRC commands and plugins. Creating your + own commands is described in the documentation.""") + mkdir(path.join(self.config.root_dir, "commands")) + self._pause() def _set_tasks(self): - # disable: True if prompted - # create tasks/ - pass + print + self._print("""I am now creating the 'tasks/' directory, where you can + place custom bot tasks and plugins. Creating your own + tasks is described in the documentation.""") + mkdir(path.join(self.config.root_dir, "tasks")) + self._pause() def _set_schedule(self): - pass + self._print("""The final section of your config file, 'schedule', is a + list of bot tasks to be started by the wiki scheduler. + Each entry contains cron-like time quantifiers and a + list of tasks. For example, the following starts the + 'foobot' task every hour on the half-hour:""") + print "schedule:" + print " - minute: 30" + print " tasks:" + print " - foobot" + self._print("""The following starts the 'barbot' task with the keyword + arguments 'action="baz"' every Monday at 05:00 UTC:""") + print " - week_day: 1" + print " hour: 5" + print " tasks:" + print ' - ["barbot", {"action": "baz"}]' + self._print("""The full list of quantifiers is minute, hour, month_day, + month, and week_day. See the documentation for more + details.""") + self._pause() def _save(self): with open(self.config.path, "w") as stream: @@ -344,8 +385,8 @@ class ConfigScript(object): relatively straightforward format and you should be able to update these settings in the future when necessary. I will start the bot at your signal. Feel free to - contact me at wikipedia.earwig at gmail.com if you have - any questions.""") + contact me at wikipedia.earwig@gmail.com if you have any + questions.""") self._save() if not self._ask_bool("Start the bot now?"): exit() diff --git a/earwigbot/managers.py b/earwigbot/managers.py index 02917df..c700e19 100644 --- a/earwigbot/managers.py +++ b/earwigbot/managers.py @@ -149,11 +149,15 @@ class _ResourceManager(object): builtin_dir = path.join(path.dirname(__file__), name) plugins_dir = path.join(self.bot.config.root_dir, name) if getattr(self.bot.config, name).get("disable") is True: - log = "Skipping disabled builtins directory {0}" + log = "Skipping disabled builtins directory: {0}" self.logger.debug(log.format(builtin_dir)) else: self._load_directory(builtin_dir) # Built-in resources - self._load_directory(plugins_dir) # Custom resources, aka plugins + if path.exists(plugins_dir) and path.isdir(plugins_dir): + self._load_directory(plugins_dir) # Custom resources, plugins + else: + log = "Skipping nonexistent plugins directory: {0}" + self.logger.debug(log.format(plugins_dir)) if self._resources: msg = "Loaded {0} {1}: {2}" From 2d4b31cde97ab44e8d4f0b1de0e7a9c91c4a7aa3 Mon Sep 17 00:00:00 2001 From: Ben Kurtovic Date: Fri, 10 Aug 2012 20:27:32 -0400 Subject: [PATCH 10/14] OrderedLoader/OrderedDumper to... preserve order... plus some cleanup. --- earwigbot/config/__init__.py | 16 +++--- earwigbot/config/node.py | 6 ++- earwigbot/config/ordered_yaml.py | 107 +++++++++++++++++++++++++++++++++++++++ earwigbot/config/script.py | 12 +++-- earwigbot/wiki/sitesdb.py | 5 +- 5 files changed, 130 insertions(+), 16 deletions(-) create mode 100644 earwigbot/config/ordered_yaml.py diff --git a/earwigbot/config/__init__.py b/earwigbot/config/__init__.py index ecefd16..3022981 100644 --- a/earwigbot/config/__init__.py +++ b/earwigbot/config/__init__.py @@ -20,6 +20,7 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. +from collections import OrderedDict from getpass import getpass from hashlib import sha256 import logging @@ -44,6 +45,7 @@ except ImportError: from earwigbot.config.formatter import BotFormatter from earwigbot.config.node import ConfigNode +from earwigbot.config.ordered_yaml import OrderedLoader from earwigbot.config.permissions import PermissionsDB from earwigbot.config.script import ConfigScript from earwigbot.exceptions import NoConfigError @@ -120,7 +122,7 @@ class BotConfig(object): filename = self._config_path with open(filename, 'r') as fp: try: - self._data = yaml.load(fp) + self._data = yaml.load(fp, OrderedLoader) except yaml.YAMLError: print "Error parsing config file {0}:".format(filename) raise @@ -270,12 +272,12 @@ class BotConfig(object): self._load() data = self._data - self.components._load(data.get("components", {})) - self.wiki._load(data.get("wiki", {})) - self.irc._load(data.get("irc", {})) - self.commands._load(data.get("commands", {})) - self.tasks._load(data.get("tasks", {})) - self.metadata._load(data.get("metadata", {})) + self.components._load(data.get("components", OrderedDict())) + self.wiki._load(data.get("wiki", OrderedDict())) + self.irc._load(data.get("irc", OrderedDict())) + self.commands._load(data.get("commands", OrderedDict())) + self.tasks._load(data.get("tasks", OrderedDict())) + self.metadata._load(data.get("metadata", OrderedDict())) self._setup_logging() if self.is_encrypted(): diff --git a/earwigbot/config/node.py b/earwigbot/config/node.py index ee045bf..82ffaa7 100644 --- a/earwigbot/config/node.py +++ b/earwigbot/config/node.py @@ -20,11 +20,13 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. +from collections import OrderedDict + __all__ = ["ConfigNode"] class ConfigNode(object): def __init__(self): - self._data = {} + self._data = OrderedDict() def __repr__(self): return self._data @@ -99,4 +101,4 @@ class ConfigNode(object): return self._data.itervalues() def iteritems(self): - return self.__dict__.iteritems() + return self._data.iteritems() diff --git a/earwigbot/config/ordered_yaml.py b/earwigbot/config/ordered_yaml.py new file mode 100644 index 0000000..dfe4590 --- /dev/null +++ b/earwigbot/config/ordered_yaml.py @@ -0,0 +1,107 @@ +# -*- coding: utf-8 -*- +# +# Copyright (C) 2009-2012 Ben Kurtovic +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +""" +Based on: + * https://gist.github.com/844388 + * http://pyyaml.org/attachment/ticket/161/use_ordered_dict.py +with modifications. +""" + +from collections import OrderedDict + +try: + import yaml +except ImportError: + yaml = None + +__all__ = ["OrderedLoader", "OrderedDumper"] + +class OrderedLoader(yaml.Loader): + """A YAML loader that loads mappings into ordered dictionaries.""" + + def __init__(self, *args, **kwargs): + super(OrderedLoader, self).__init__(*args, **kwargs) + constructor = type(self).construct_yaml_map + self.add_constructor(u"tag:yaml.org,2002:map", constructor) + self.add_constructor(u"tag:yaml.org,2002:omap", constructor) + + def construct_yaml_map(self, node): + data = OrderedDict() + yield data + value = self.construct_mapping(node) + data.update(value) + + def construct_mapping(self, node, deep=False): + if isinstance(node, yaml.MappingNode): + self.flatten_mapping(node) + else: + raise yaml.constructor.ConstructorError(None, None, + "expected a mapping node, but found {0}".format(node.id), + node.start_mark) + + mapping = OrderedDict() + for key_node, value_node in node.value: + key = self.construct_object(key_node, deep=deep) + try: + hash(key) + except TypeError, exc: + raise yaml.constructor.ConstructorError( + "while constructing a mapping", node.start_mark, + "found unacceptable key ({0})".format(exc), + key_node.start_mark) + value = self.construct_object(value_node, deep=deep) + mapping[key] = value + return mapping + + +class OrderedDumper(yaml.Dumper): + """A YAML dumper that dumps mappings into ordered dictionaries.""" + + def __init__(self, *args, **kwargs): + super(OrderedDumper, self).__init__(*args, **kwargs) + self.add_representer(OrderedDict, type(self).represent_dict) + + def represent_mapping(self, tag, mapping, flow_style=None): + value = [] + node = yaml.MappingNode(tag, value, flow_style=flow_style) + if self.alias_key is not None: + self.represented_objects[self.alias_key] = node + best_style = True + if hasattr(mapping, "items"): + mapping = list(mapping.items()) + for item_key, item_value in mapping: + node_key = self.represent_data(item_key) + node_value = self.represent_data(item_value) + if not (isinstance(node_key, yaml.ScalarNode) and not + node_key.style): + best_style = False + if not (isinstance(node_value, yaml.ScalarNode) and not + node_value.style): + best_style = False + value.append((node_key, node_value)) + if flow_style is None: + if self.default_flow_style is not None: + node.flow_style = self.default_flow_style + else: + node.flow_style = best_style + return node diff --git a/earwigbot/config/script.py b/earwigbot/config/script.py index 972def8..d4f9bf9 100644 --- a/earwigbot/config/script.py +++ b/earwigbot/config/script.py @@ -38,6 +38,7 @@ except ImportError: yaml = None from earwigbot import exceptions +from earwigbot.config.ordered_yaml import OrderedDumper __all__ = ["ConfigScript"] @@ -272,9 +273,10 @@ class ConfigScript(object): 'unaffiliated/nickname'.""") host = self._ask("Your hostname on the IRC frontend:") if host: - self.config._permissions.load() - self.config._permissions.add_owner(host=host) - self.config._permissions.add_admin(host=host) + permdb = self.config._permissions + permdb.load() + permdb.add_owner(host=host) + permdb.add_admin(host=host) else: frontend = {} @@ -360,8 +362,8 @@ class ConfigScript(object): self._pause() def _save(self): - with open(self.config.path, "w") as stream: - yaml.dump(self.data, stream=stream, default_flow_style=False) + with open(self.config.path, "w") as strm: + yaml.dump(self.data, strm, OrderedDumper, default_flow_style=False) def make_new(self): """Make a new config file based on the user's input.""" diff --git a/earwigbot/wiki/sitesdb.py b/earwigbot/wiki/sitesdb.py index 3724d4b..d8c2e3b 100644 --- a/earwigbot/wiki/sitesdb.py +++ b/earwigbot/wiki/sitesdb.py @@ -20,6 +20,7 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. +from collections import OrderedDict from cookielib import LWPCookieJar, LoadError import errno from os import chmod, path @@ -192,7 +193,7 @@ class SitesDB(object): maxlag = config.wiki.get("maxlag") wait_between_queries = config.wiki.get("waitTime", 3) logger = self._logger.getChild(name) - search_config = config.wiki.get("search", {}).copy() + search_config = config.wiki.get("search", OrderedDict()).copy() if user_agent: user_agent = user_agent.replace("$1", __version__) @@ -204,7 +205,7 @@ class SitesDB(object): search_config["exclusions_db"] = self._exclusions_db if not sql: - sql = config.wiki.get("sql", {}).copy() + sql = config.wiki.get("sql", OrderedDict()).copy() for key, value in sql.iteritems(): if isinstance(value, basestring) and "$1" in value: sql[key] = value.replace("$1", name) From a3b64a38d98a873aef8777b261a05d018bbe98fc Mon Sep 17 00:00:00 2001 From: Ben Kurtovic Date: Fri, 10 Aug 2012 21:19:52 -0400 Subject: [PATCH 11/14] Miscellaneous tweaks and fixes. --- earwigbot/commands/crypt.py | 3 ++ earwigbot/config/ordered_yaml.py | 2 +- earwigbot/config/script.py | 86 +++++++++++++++++++++++++--------------- 3 files changed, 58 insertions(+), 33 deletions(-) diff --git a/earwigbot/commands/crypt.py b/earwigbot/commands/crypt.py index d77a8c8..0123be1 100644 --- a/earwigbot/commands/crypt.py +++ b/earwigbot/commands/crypt.py @@ -69,6 +69,9 @@ class Crypt(Command): cipher = Blowfish.new(hashlib.sha256(key).digest()) try: if data.command == "encrypt": + if len(text) % 8: + pad = 8 - len(text) % 8 + text = text.ljust(len(text) + pad, "\x00") self.reply(data, cipher.encrypt(text).encode("hex")) else: self.reply(data, cipher.decrypt(text.decode("hex"))) diff --git a/earwigbot/config/ordered_yaml.py b/earwigbot/config/ordered_yaml.py index dfe4590..8901321 100644 --- a/earwigbot/config/ordered_yaml.py +++ b/earwigbot/config/ordered_yaml.py @@ -75,7 +75,7 @@ class OrderedLoader(yaml.Loader): class OrderedDumper(yaml.Dumper): - """A YAML dumper that dumps mappings into ordered dictionaries.""" + """A YAML dumper that dumps ordered dictionaries into mappings.""" def __init__(self, *args, **kwargs): super(OrderedDumper, self).__init__(*args, **kwargs) diff --git a/earwigbot/config/script.py b/earwigbot/config/script.py index d4f9bf9..cd9415a 100644 --- a/earwigbot/config/script.py +++ b/earwigbot/config/script.py @@ -22,12 +22,18 @@ from collections import OrderedDict from getpass import getpass +from hashlib import sha256 from os import chmod, mkdir, path import re import stat from textwrap import fill, wrap try: + from Crypto.Cipher import Blowfish +except ImportError: + Blowfish = None + +try: import bcrypt except ImportError: bcrypt = None @@ -42,7 +48,7 @@ from earwigbot.config.ordered_yaml import OrderedDumper __all__ = ["ConfigScript"] -RULES_TEMPLATE = """"# -*- coding: utf-8 -*- +RULES_TEMPLATE = """# -*- coding: utf-8 -*- def process(bot, rc): \"\"\"Given a Bot() object and an RC() object, return a list of channels @@ -58,7 +64,7 @@ class ConfigScript(object): def __init__(self, config): self.config = config - self.data = OrderedDict( + self.data = OrderedDict([ ("metadata", OrderedDict()), ("components", OrderedDict()), ("wiki", OrderedDict()), @@ -66,11 +72,12 @@ class ConfigScript(object): ("commands", OrderedDict()), ("tasks", OrderedDict()), ("schedule", []) - ) + ]) - self.wmf = False - self.proj = None - self.lang = None + self._cipher = None + self._wmf = False + self._proj = None + self._lang = None def _print(self, text): print fill(re.sub("\s\s+", " ", text), self.WIDTH) @@ -105,6 +112,16 @@ class ConfigScript(object): if answer.startswith("n"): return False + def _ask_pass(self, text): + password = getpass("> " + text + " ") + if self._cipher: + mod = len(password) % 8 + if mod: + password = password.ljust(len(password) + (8 - mod), "\x00") + return self._cipher.encrypt(password).encode("hex") + else: + return password + def _ask_list(self, text): print fill(re.sub("\s\s+", " ", "> " + text), self.WIDTH) print "[one item per line; blank line to end]:" @@ -118,7 +135,7 @@ class ConfigScript(object): def _set_metadata(self): print - self.data["metadata"] = OrderedDict(("version", 1)) + self.data["metadata"] = OrderedDict([("version", 1)]) self._print("""I can encrypt passwords stored in your config file in addition to preventing other users on your system from reading the file. Encryption is recommended is the bot @@ -128,9 +145,10 @@ class ConfigScript(object): if self._ask_bool("Encrypt stored passwords?"): self.data["metadata"]["encryptPasswords"] = True key = getpass("> Enter an encryption key: ") - print "Running {0} rounds of bcrypt...".format(self.BCRYPT_ROUNDS), + print "Running {0} rounds of bcrypt...".format(self.BCRYPT_ROUNDS), # STDOUT signature = bcrypt.hashpw(key, bcrypt.gensalt(self.BCRYPT_ROUNDS)) self.data["metadata"]["signature"] = signature + self._cipher = Blowfish.new(sha256(key).digest()) print " done." else: self.data["metadata"]["encryptPasswords"] = False @@ -185,7 +203,7 @@ class ConfigScript(object): question = "Would you like to re-enter your login information?" if self._ask_bool(question): self.data["wiki"]["username"] = self._ask("Bot username:") - self.data["wiki"]["password"] = getpass("> Bot password: ") + self.data["wiki"]["password"] = self._ask_pass("Bot password:") return self._login(kwargs) question = "Would you like to re-enter the site information?" if self._ask_bool(question): @@ -198,13 +216,13 @@ class ConfigScript(object): def _set_wiki(self): print - self.wmf = self._ask_bool("""Will this bot run on Wikimedia Foundation - wikis, like Wikipedia?""") - if self.wmf: + self._wmf = self._ask_bool("""Will this bot run on Wikimedia Foundation + wikis, like Wikipedia?""") + if self._wmf: msg = "Site project (e.g. 'wikipedia', 'wiktionary', 'wikimedia'):" - self.proj = project = self._ask(msg, default="wikipedia").lower() + self._proj = project = self._ask(msg, default="wikipedia").lower() msg = "Site language code (e.g. 'en', 'fr', 'commons'):" - self.lang = lang = self._ask(msg, default="en").lower() + self._lang = lang = self._ask(msg, default="en").lower() kwargs = {"project": project, "lang": lang} else: msg = "Site base URL, without the script path and trailing slash;" @@ -214,7 +232,7 @@ class ConfigScript(object): kwargs = {"base_url": url, "script_path": script} self.data["wiki"]["username"] = self._ask("Bot username:") - self.data["wiki"]["password"] = getpass("> Bot password: ") + self.data["wiki"]["password"] = self._ask_pass("Bot password:") self.data["wiki"]["userAgent"] = "EarwigBot/$1 (Python/$2; https://github.com/earwig/earwigbot)" self.data["wiki"]["summary"] = "([[WP:BOT|Bot]]): $2" self.data["wiki"]["useHTTPS"] = True @@ -224,11 +242,11 @@ class ConfigScript(object): self.data["wiki"]["defaultSite"] = self._login(kwargs).name self.data["wiki"]["sql"] = {} - if self.wmf: + if self._wmf: msg = "Will this bot run from the Wikimedia Toolserver?" toolserver = self._ask_bool(msg, default=False) if toolserver: - args = (("host", "$1-p.rrdb.toolserver.org"), ("db", "$1_p")) + args = [("host", "$1-p.rrdb.toolserver.org"), ("db", "$1_p")] self.data["wiki"]["sql"] = OrderedDict(args) self.data["wiki"]["shutoff"] = {} @@ -240,7 +258,7 @@ class ConfigScript(object): separate shutoff page for each task.""") page = self._ask("Page title:", default="User:$1/Shutoff") disabled = self._ask("Page content when *not* shut off:", "run") - args = (("page", page), ("disabled", disabled)) + args = [("page", page), ("disabled", disabled)] self.data["wiki"]["shutoff"] = OrderedDict(args) self.data["wiki"]["search"] = {} @@ -258,9 +276,10 @@ class ConfigScript(object): question = "Frontend bot's real name (gecos):" frontend["realname"] = self._ask(question) if self._ask_bool("Should the bot identify to NickServ?"): - frontend["nickservUsername"] = self._ask("NickServ username:", - frontend["nick"]) - frontend["nickservPassword"] = getpass("> Nickserv password: ") + ns_user = self._ask("NickServ username:", frontend["nick"]) + ns_pass = self._ask_pass("Nickserv password:") + frontend["nickservUsername"] = ns_user + frontend["nickservPassword"] = ns_pass chan_question = "Frontend channels to join by default:" frontend["channels"] = self._ask_list(chan_question) self._print("""The bot keeps a database of its admins (users who @@ -283,26 +302,28 @@ class ConfigScript(object): if self.data["components"]["irc_watcher"]: print watcher = self.data["irc"]["watcher"] = OrderedDict() - if self.wmf: + if self._wmf: watcher["host"] = "irc.wikimedia.org" watcher["port"] = 6667 else: msg = "Hostname of the watcher's IRC server, without 'irc://':" watcher["host"] = self._ask(msg) watcher["port"] = self._ask("Watcher port:", 6667) - watcher["nick"] = self._ask("Watcher bot's nickname:", - frontend.get("nick")) - watcher["ident"] = self._ask("Watcher bot's ident:", - watcher["nick"].lower()) + nick = self._ask("Watcher bot's nickname:", frontend.get("nick")) + ident = self._ask("Watcher bot's ident:", watcher["nick"].lower()) + watcher["nick"] = nick + watcher["ident"] = ident question = "Watcher bot's real name (gecos):" watcher["realname"] = self._ask(question, frontend.get("realname")) watcher_ns = "Should the bot identify to NickServ?" - if not self.wmf and self._ask_bool(watcher_ns): - watcher["nickservUsername"] = self._ask("NickServ username:", - watcher["nick"]) - watcher["nickservPassword"] = getpass("> Nickserv password: ") - if self.wmf: - watcher["channels"] = ["#{0}.{1}".format(self.lang, self.proj)] + if not self._wmf and self._ask_bool(watcher_ns): + ns_user = self._ask("NickServ username:", watcher["nick"]) + ns_pass = self._ask_pass("Nickserv password:") + watcher["nickservUsername"] = ns_user + watcher["nickservPassword"] = ns_pass + if self._wmf: + chan = "#{0}.{1}".format(self._lang, self._proj) + watcher["channels"] = [chan] else: chan_question = "Watcher channels to join by default:" watcher["channels"] = self._ask_list(chan_question) @@ -383,6 +404,7 @@ class ConfigScript(object): self._set_tasks() if components["wiki_scheduler"]: self._set_schedule() + print self._print("""I am now saving config.yml with your settings. YAML is a relatively straightforward format and you should be able to update these settings in the future when necessary. From b2b11eadada94541247a033a357f0799a6bc92ce Mon Sep 17 00:00:00 2001 From: Ben Kurtovic Date: Fri, 10 Aug 2012 21:27:47 -0400 Subject: [PATCH 12/14] Fix a watcher bug. --- earwigbot/irc/watcher.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/earwigbot/irc/watcher.py b/earwigbot/irc/watcher.py index d7f5ff8..dbb1150 100644 --- a/earwigbot/irc/watcher.py +++ b/earwigbot/irc/watcher.py @@ -94,7 +94,7 @@ class Watcher(IRCConnection): except ImportError: return try: - module = imp.load_module(name, f, path, desc) + module = imp.load_module("rules", f, path, desc) except Exception: return finally: From 110e83fb6b874a59b5f7694b992e2dc57a0fe369 Mon Sep 17 00:00:00 2001 From: Ben Kurtovic Date: Fri, 10 Aug 2012 22:59:18 -0400 Subject: [PATCH 13/14] A number of fixes; color and stdout flushing. --- earwigbot/config/__init__.py | 19 +++++++++++------- earwigbot/config/script.py | 48 +++++++++++++++++++++++++++++++------------- 2 files changed, 46 insertions(+), 21 deletions(-) diff --git a/earwigbot/config/__init__.py b/earwigbot/config/__init__.py index 3022981..f81bc2f 100644 --- a/earwigbot/config/__init__.py +++ b/earwigbot/config/__init__.py @@ -117,6 +117,15 @@ class BotConfig(object): """Return a nice string representation of the BotConfig.""" return "".format(self.root_dir) + def _handle_missing_config(self): + print "Config file missing or empty:", self._config_path + msg = "Would you like to create a config file now? [Y/n] " + choice = raw_input(msg) + if choice.lower().startswith("n"): + raise NoConfigError() + else: + ConfigScript(self).make_new() + def _load(self): """Load data from our JSON config file (config.yml) into self._data.""" filename = self._config_path @@ -263,15 +272,11 @@ class BotConfig(object): decrypted if they were decrypted earlier. """ if not path.exists(self._config_path): - print "Config file not found:", self._config_path - choice = raw_input("Would you like to create a config file now? [Y/n] ") - if choice.lower().startswith("n"): - raise NoConfigError() - else: - ConfigScript(self).make_new() - + self._handle_missing_config() self._load() data = self._data + if not data: + self._handle_missing_config() self.components._load(data.get("components", OrderedDict())) self.wiki._load(data.get("wiki", OrderedDict())) self.irc._load(data.get("irc", OrderedDict())) diff --git a/earwigbot/config/script.py b/earwigbot/config/script.py index cd9415a..fd0d2dd 100644 --- a/earwigbot/config/script.py +++ b/earwigbot/config/script.py @@ -26,6 +26,7 @@ from hashlib import sha256 from os import chmod, mkdir, path import re import stat +import sys from textwrap import fill, wrap try: @@ -60,6 +61,7 @@ def process(bot, rc): class ConfigScript(object): """A script to guide a user through the creation of a new config file.""" WIDTH = 79 + PROMPT = "\x1b[32m> \x1b[0m" BCRYPT_ROUNDS = 12 def __init__(self, config): @@ -82,24 +84,28 @@ class ConfigScript(object): def _print(self, text): print fill(re.sub("\s\s+", " ", text), self.WIDTH) + def _print_no_nl(self, text): + sys.stdout.write(fill(re.sub("\s\s+", " ", text), self.WIDTH)) + sys.stdout.flush() + def _pause(self): - raw_input("> Press enter to continue: ") + raw_input(self.PROMPT + "Press enter to continue: ") def _ask(self, text, default=None): - text = "> " + text + text = self.PROMPT + text if default: - text += " [{0}]".format(default) + text += " \x1b[33m[{0}]\x1b[0m".format(default) lines = wrap(re.sub("\s\s+", " ", text), self.WIDTH) if len(lines) > 1: print "\n".join(lines[:-1]) return raw_input(lines[-1] + " ") or default def _ask_bool(self, text, default=True): - text = "> " + text + text = self.PROMPT + text if default: - text += " [Y/n]" + text += " \x1b[33m[Y/n]\x1b[0m" else: - text += " [y/N]" + text += " \x1b[33m[y/N]\x1b[0m" lines = wrap(re.sub("\s\s+", " ", text), self.WIDTH) if len(lines) > 1: print "\n".join(lines[:-1]) @@ -113,7 +119,7 @@ class ConfigScript(object): return False def _ask_pass(self, text): - password = getpass("> " + text + " ") + password = getpass(self.PROMPT + text + " ") if self._cipher: mod = len(password) % 8 if mod: @@ -123,11 +129,11 @@ class ConfigScript(object): return password def _ask_list(self, text): - print fill(re.sub("\s\s+", " ", "> " + text), self.WIDTH) + print fill(re.sub("\s\s+", " ", self.PROMPT + text), self.WIDTH) print "[one item per line; blank line to end]:" result = [] while True: - line = raw_input("> ") + line = raw_input(self.PROMPT) if line: result.append(line) else: @@ -144,8 +150,9 @@ class ConfigScript(object): the bot may be annoying.""") if self._ask_bool("Encrypt stored passwords?"): self.data["metadata"]["encryptPasswords"] = True - key = getpass("> Enter an encryption key: ") - print "Running {0} rounds of bcrypt...".format(self.BCRYPT_ROUNDS), # STDOUT + key = getpass(self.PROMPT + "Enter an encryption key: ") + msg = "Running {0} rounds of bcrypt...".format(self.BCRYPT_ROUNDS) + self._print_no_nl(msg) signature = bcrypt.hashpw(key, bcrypt.gensalt(self.BCRYPT_ROUNDS)) self.data["metadata"]["signature"] = signature self._cipher = Blowfish.new(sha256(key).digest()) @@ -153,6 +160,7 @@ class ConfigScript(object): else: self.data["metadata"]["encryptPasswords"] = False + print self._print("""The bot can temporarily store its logs in the logs/ subdirectory. Error logs are kept for a month whereas normal logs are kept for a week. If you disable this, @@ -186,11 +194,12 @@ class ConfigScript(object): def _login(self, kwargs): self.config.wiki._load(self.data["wiki"]) - print "Trying to login to the site...", + self._print_no_nl("Trying to connect to the site...") try: site = self.config.bot.wiki.add_site(**kwargs) - except exceptions.APIError: + except exceptions.APIError as exc: print " API error!" + print "\x1b[31m" + exc.message + "\x1b[0m" question = "Would you like to re-enter the site information?" if self._ask_bool(question): return self._set_wiki() @@ -198,8 +207,9 @@ class ConfigScript(object): if self._ask_bool(question, default=False): raise exceptions.NoConfigError() return self._set_wiki() - except exceptions.LoginError: + except exceptions.LoginError as exc: print " login error!" + print "\x1b[31m" + exc.message + "\x1b[0m" question = "Would you like to re-enter your login information?" if self._ask_bool(question): self.data["wiki"]["username"] = self._ask("Bot username:") @@ -210,6 +220,13 @@ class ConfigScript(object): return self._set_wiki() self._print("""Moving on. You can modify the login information stored in the bot's config in the future.""") + password = self.data["wiki"]["password"] + self.data["wiki"]["password"] = None # Clear so we don't login + self.config.wiki._load(self.data["wiki"]) + self._print_no_nl("Trying to connect to the site...") + site = self.config.bot.wiki.add_site(**kwargs) + print " success." + self.data["wiki"]["password"] = password # Reset original value else: print " success." return site @@ -252,6 +269,7 @@ class ConfigScript(object): self.data["wiki"]["shutoff"] = {} msg = "Would you like to enable an automatic shutoff page for the bot?" if self._ask_bool(msg): + print self._print("""The page title can contain two wildcards: $1 will be substituted with the bot's username, and $2 with the current task number. This can be used to implement a @@ -282,6 +300,7 @@ class ConfigScript(object): frontend["nickservPassword"] = ns_pass chan_question = "Frontend channels to join by default:" frontend["channels"] = self._ask_list(chan_question) + print self._print("""The bot keeps a database of its admins (users who can use certain sensitive commands) and owners (users who can quit the bot and modify its access @@ -327,6 +346,7 @@ class ConfigScript(object): else: chan_question = "Watcher channels to join by default:" watcher["channels"] = self._ask_list(chan_question) + print self._print("""I am now creating a blank 'rules.py' file, which will determine how the bot handles messages received from the IRC watcher. It contains a process() From 8d8703358cbd685c51945df64abb66fe30922eb1 Mon Sep 17 00:00:00 2001 From: Ben Kurtovic Date: Sat, 11 Aug 2012 00:10:50 -0400 Subject: [PATCH 14/14] More fixes and tweaks; cleanup; etc. --- earwigbot/config/ordered_yaml.py | 2 +- earwigbot/config/script.py | 55 ++++++++++++++++++++++++------------- earwigbot/wiki/copyvios/__init__.py | 10 ++++--- 3 files changed, 43 insertions(+), 24 deletions(-) diff --git a/earwigbot/config/ordered_yaml.py b/earwigbot/config/ordered_yaml.py index 8901321..33158ab 100644 --- a/earwigbot/config/ordered_yaml.py +++ b/earwigbot/config/ordered_yaml.py @@ -74,7 +74,7 @@ class OrderedLoader(yaml.Loader): return mapping -class OrderedDumper(yaml.Dumper): +class OrderedDumper(yaml.SafeDumper): """A YAML dumper that dumps ordered dictionaries into mappings.""" def __init__(self, *args, **kwargs): diff --git a/earwigbot/config/script.py b/earwigbot/config/script.py index fd0d2dd..c02ff1e 100644 --- a/earwigbot/config/script.py +++ b/earwigbot/config/script.py @@ -118,8 +118,13 @@ class ConfigScript(object): if answer.startswith("n"): return False - def _ask_pass(self, text): + def _ask_pass(self, text, encrypt=True): password = getpass(self.PROMPT + text + " ") + if encrypt: + return self._encrypt(password) + return password + + def _encrypt(self, password): if self._cipher: mod = len(password) % 8 if mod: @@ -213,14 +218,17 @@ class ConfigScript(object): question = "Would you like to re-enter your login information?" if self._ask_bool(question): self.data["wiki"]["username"] = self._ask("Bot username:") - self.data["wiki"]["password"] = self._ask_pass("Bot password:") + password = self._ask_pass("Bot password:", encrypt=False) + self.data["wiki"]["password"] = password return self._login(kwargs) + else: + password = self.data["wiki"]["password"] question = "Would you like to re-enter the site information?" if self._ask_bool(question): return self._set_wiki() + print self._print("""Moving on. You can modify the login information stored in the bot's config in the future.""") - password = self.data["wiki"]["password"] self.data["wiki"]["password"] = None # Clear so we don't login self.config.wiki._load(self.data["wiki"]) self._print_no_nl("Trying to connect to the site...") @@ -229,6 +237,10 @@ class ConfigScript(object): self.data["wiki"]["password"] = password # Reset original value else: print " success." + + # Remember to store the encrypted password: + password = self._encrypt(self.data["wiki"]["password"]) + self.data["wiki"]["password"] = password return site def _set_wiki(self): @@ -249,7 +261,8 @@ class ConfigScript(object): kwargs = {"base_url": url, "script_path": script} self.data["wiki"]["username"] = self._ask("Bot username:") - self.data["wiki"]["password"] = self._ask_pass("Bot password:") + password = self._ask_pass("Bot password:", encrypt=False) + self.data["wiki"]["password"] = password self.data["wiki"]["userAgent"] = "EarwigBot/$1 (Python/$2; https://github.com/earwig/earwigbot)" self.data["wiki"]["summary"] = "([[WP:BOT|Bot]]): $2" self.data["wiki"]["useHTTPS"] = True @@ -275,7 +288,8 @@ class ConfigScript(object): current task number. This can be used to implement a separate shutoff page for each task.""") page = self._ask("Page title:", default="User:$1/Shutoff") - disabled = self._ask("Page content when *not* shut off:", "run") + msg = "Page content to indicate the bot is *not* shut off:" + disabled = self._ask(msg, "run") args = [("page", page), ("disabled", disabled)] self.data["wiki"]["shutoff"] = OrderedDict(args) @@ -305,9 +319,9 @@ class ConfigScript(object): can use certain sensitive commands) and owners (users who can quit the bot and modify its access list), identified by nick, ident, and/or hostname. - Hostname is most secure since it cannot be easily - spoofed. If you have a cloak, it will probably look - like 'wikipedia/Username' or + Hostname is the most secure option since it cannot + be easily spoofed. If you have a cloak, this will + probably look like 'wikipedia/Username' or 'unaffiliated/nickname'.""") host = self._ask("Your hostname on the IRC frontend:") if host: @@ -329,7 +343,7 @@ class ConfigScript(object): watcher["host"] = self._ask(msg) watcher["port"] = self._ask("Watcher port:", 6667) nick = self._ask("Watcher bot's nickname:", frontend.get("nick")) - ident = self._ask("Watcher bot's ident:", watcher["nick"].lower()) + ident = self._ask("Watcher bot's ident:", nick.lower()) watcher["nick"] = nick watcher["ident"] = ident question = "Watcher bot's real name (gecos):" @@ -350,9 +364,9 @@ class ConfigScript(object): self._print("""I am now creating a blank 'rules.py' file, which will determine how the bot handles messages received from the IRC watcher. It contains a process() - function that takes a Bot object allowing you to - start tasks and an RC object that holds the message - from the watcher. See the documentation for + function that takes a Bot object (allowing you to + start tasks) and an RC object (storing the message + from the watcher). See the documentation for details.""") with open(path.join(self.config.root_dir, "rules.py"), "w") as fp: fp.write(RULES_TEMPLATE) @@ -367,6 +381,7 @@ class ConfigScript(object): if (not self.data["components"]["irc_frontend"] or self._ask_bool(msg, default=False)): self.data["commands"]["disable"] = True + print self._print("""I am now creating the 'commands/' directory, where you can place custom IRC commands and plugins. Creating your own commands is described in the documentation.""") @@ -382,29 +397,31 @@ class ConfigScript(object): self._pause() def _set_schedule(self): + print self._print("""The final section of your config file, 'schedule', is a list of bot tasks to be started by the wiki scheduler. Each entry contains cron-like time quantifiers and a list of tasks. For example, the following starts the 'foobot' task every hour on the half-hour:""") - print "schedule:" + print "\x1b[33mschedule:" print " - minute: 30" print " tasks:" - print " - foobot" + print " - foobot\x1b[0m" self._print("""The following starts the 'barbot' task with the keyword arguments 'action="baz"' every Monday at 05:00 UTC:""") - print " - week_day: 1" + print "\x1b[33m - week_day: 1" print " hour: 5" print " tasks:" - print ' - ["barbot", {"action": "baz"}]' + print ' - ["barbot", {"action": "baz"}]\x1b[0m' self._print("""The full list of quantifiers is minute, hour, month_day, month, and week_day. See the documentation for more - details.""") + information.""") self._pause() def _save(self): - with open(self.config.path, "w") as strm: - yaml.dump(self.data, strm, OrderedDumper, default_flow_style=False) + with open(self.config.path, "w") as stream: + yaml.dump(self.data, stream, OrderedDumper, indent=4, + allow_unicode=True, default_flow_style=False) def make_new(self): """Make a new config file based on the user's input.""" diff --git a/earwigbot/wiki/copyvios/__init__.py b/earwigbot/wiki/copyvios/__init__.py index d01b943..ee0bc9e 100644 --- a/earwigbot/wiki/copyvios/__init__.py +++ b/earwigbot/wiki/copyvios/__init__.py @@ -52,7 +52,7 @@ class CopyvioMixIn(object): def __init__(self, site): self._search_config = site._search_config - self._exclusions_db = self._search_config["exclusions_db"] + self._exclusions_db = self._search_config.get("exclusions_db") self._opener = build_opener() self._opener.addheaders = site._opener.addheaders @@ -137,7 +137,8 @@ class CopyvioMixIn(object): :py:exc:`~earwigbot.exceptions.SearchQueryError`, ...) on errors. """ searcher = self._select_search_engine() - self._exclusions_db.sync(self.site.name) + if self._exclusions_db: + self._exclusions_db.sync(self.site.name) handled_urls = [] best_confidence = 0 best_match = None @@ -163,8 +164,9 @@ class CopyvioMixIn(object): urls = [url for url in urls if url not in handled_urls] for url in urls: handled_urls.append(url) - if self._exclusions_db.check(self.site.name, url): - continue + if self._exclusions_db: + if self._exclusions_db.check(self.site.name, url): + continue conf, chains = self._copyvio_compare_content(article_chain, url) if conf > best_confidence: best_confidence = conf