Merge branch 'feature/confscript' into develop (closes #1)

earwigbot/bot.py

@@ -60,7 +60,7 @@ class Bot(object):
     """
 
     def __init__(self, root_dir, level=logging.INFO):
-        self.config = BotConfig(root_dir, level)
+        self.config = BotConfig(self, root_dir, level)
         self.logger = logging.getLogger("earwigbot")
         self.commands = CommandManager(self)
         self.tasks = TaskManager(self)

earwigbot/commands/crypt.py

@@ -69,6 +69,9 @@ class Crypt(Command):
             cipher = Blowfish.new(hashlib.sha256(key).digest())
             try:
                 if data.command == "encrypt":
+                    if len(text) % 8:
+                        pad = 8 - len(text) % 8
+                        text = text.ljust(len(text) + pad, "\x00")
                     self.reply(data, cipher.encrypt(text).encode("hex"))
                 else:
                     self.reply(data, cipher.decrypt(text.decode("hex")))

earwigbot/config/__init__.py

@@ -20,11 +20,13 @@
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 # SOFTWARE.
 
+from collections import OrderedDict
 from getpass import getpass
 from hashlib import sha256
 import logging
 import logging.handlers
 from os import mkdir, path
+import stat
 
 try:
     from Crypto.Cipher import Blowfish
@@ -43,7 +45,9 @@ except ImportError:
 
 from earwigbot.config.formatter import BotFormatter
 from earwigbot.config.node import ConfigNode
+from earwigbot.config.ordered_yaml import OrderedLoader
 from earwigbot.config.permissions import PermissionsDB
+from earwigbot.config.script import ConfigScript
 from earwigbot.exceptions import NoConfigError
 
 __all__ = ["BotConfig"]
@@ -75,7 +79,8 @@ class BotConfig(object):
     - :py:meth:`decrypt`: decrypts an object in the config tree
     """
 
-    def __init__(self, root_dir, level):
+    def __init__(self, bot, root_dir, level):
+        self._bot = bot
         self._root_dir = root_dir
         self._logging_level = level
         self._config_path = path.join(self.root_dir, "config.yml")
@@ -112,12 +117,21 @@ class BotConfig(object):
         """Return a nice string representation of the BotConfig."""
         return "<BotConfig at {0}>".format(self.root_dir)
 
+    def _handle_missing_config(self):
+        print "Config file missing or empty:", self._config_path
+        msg = "Would you like to create a config file now? [Y/n] "
+        choice = raw_input(msg)
+        if choice.lower().startswith("n"):
+            raise NoConfigError()
+        else:
+            ConfigScript(self).make_new()
+
     def _load(self):
         """Load data from our JSON config file (config.yml) into self._data."""
         filename = self._config_path
         with open(filename, 'r') as fp:
             try:
-                self._data = yaml.load(fp)
+                self._data = yaml.load(fp, OrderedLoader)
             except yaml.YAMLError:
                 print "Error parsing config file {0}:".format(filename)
                 raise
@@ -137,7 +151,7 @@ class BotConfig(object):
 
             if not path.isdir(log_dir):
                 if not path.exists(log_dir):
-                    mkdir(log_dir, 0700)
+                    mkdir(log_dir, stat.S_IWUSR|stat.S_IRUSR|stat.S_IXUSR)
                 else:
                     msg = "log_dir ({0}) exists but is not a directory!"
                     print msg.format(log_dir)
@@ -168,19 +182,10 @@ class BotConfig(object):
             print "Error decrypting passwords:"
             raise
 
-    def _make_new(self):
-        """Make a new config file based on the user's input."""
-        #m = "Would you like to encrypt passwords stored in config.yml? [y/n] "
-        #encrypt = raw_input(m)
-        #if encrypt.lower().startswith("y"):
-        #    is_encrypted = True
-        #else:
-        #    is_encrypted = False
-        raise NotImplementedError()
-        # yaml.dumps() config.yml file (self._config_path)
-        # Create root_dir/, root_dir/commands/, root_dir/tasks/
-        # Give a reasonable message after config has been created regarding
-        # what to do next...
+    @property
+    def bot(self):
+        """The config's Bot object."""
+        return self._bot
 
     @property
     def root_dir(self):
@@ -267,21 +272,17 @@ class BotConfig(object):
         decrypted if they were decrypted earlier.
         """
         if not path.exists(self._config_path):
-            print "Config file not found:", self._config_path
-            choice = raw_input("Would you like to create a config file now? [y/n] ")
-            if choice.lower().startswith("y"):
-                self._make_new()
-            else:
-                raise NoConfigError()
-
+            self._handle_missing_config()
         self._load()
         data = self._data
-        self.components._load(data.get("components", {}))
-        self.wiki._load(data.get("wiki", {}))
-        self.irc._load(data.get("irc", {}))
-        self.commands._load(data.get("commands", {}))
-        self.tasks._load(data.get("tasks", {}))
-        self.metadata._load(data.get("metadata", {}))
+        if not data:
+            self._handle_missing_config()
+        self.components._load(data.get("components", OrderedDict()))
+        self.wiki._load(data.get("wiki", OrderedDict()))
+        self.irc._load(data.get("irc", OrderedDict()))
+        self.commands._load(data.get("commands", OrderedDict()))
+        self.tasks._load(data.get("tasks", OrderedDict()))
+        self.metadata._load(data.get("metadata", OrderedDict()))
 
         self._setup_logging()
         if self.is_encrypted():

earwigbot/config/node.py

@@ -20,11 +20,13 @@
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 # SOFTWARE.
 
+from collections import OrderedDict
+
 __all__ = ["ConfigNode"]
 
 class ConfigNode(object):
     def __init__(self):
-        self._data = {}
+        self._data = OrderedDict()
 
     def __repr__(self):
         return self._data
@@ -99,4 +101,4 @@ class ConfigNode(object):
         return self._data.itervalues()
 
     def iteritems(self):
-        return self.__dict__.iteritems()
+        return self._data.iteritems()

earwigbot/config/ordered_yaml.py

@@ -0,0 +1,107 @@
+# -*- coding: utf-8  -*-
+#
+# Copyright (C) 2009-2012 Ben Kurtovic <ben.kurtovic@verizon.net>
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+"""
+Based on:
+ * https://gist.github.com/844388
+ * http://pyyaml.org/attachment/ticket/161/use_ordered_dict.py
+with modifications.
+"""
+
+from collections import OrderedDict
+
+try:
+    import yaml
+except ImportError:
+    yaml = None
+
+__all__ = ["OrderedLoader", "OrderedDumper"]
+
+class OrderedLoader(yaml.Loader):
+    """A YAML loader that loads mappings into ordered dictionaries."""
+
+    def __init__(self, *args, **kwargs):
+        super(OrderedLoader, self).__init__(*args, **kwargs)
+        constructor = type(self).construct_yaml_map
+        self.add_constructor(u"tag:yaml.org,2002:map", constructor)
+        self.add_constructor(u"tag:yaml.org,2002:omap", constructor)
+
+    def construct_yaml_map(self, node):
+        data = OrderedDict()
+        yield data
+        value = self.construct_mapping(node)
+        data.update(value)
+
+    def construct_mapping(self, node, deep=False):
+        if isinstance(node, yaml.MappingNode):
+            self.flatten_mapping(node)
+        else:
+            raise yaml.constructor.ConstructorError(None, None,
+                "expected a mapping node, but found {0}".format(node.id),
+                node.start_mark)
+
+        mapping = OrderedDict()
+        for key_node, value_node in node.value:
+            key = self.construct_object(key_node, deep=deep)
+            try:
+                hash(key)
+            except TypeError, exc:
+                raise yaml.constructor.ConstructorError(
+                    "while constructing a mapping", node.start_mark,
+                    "found unacceptable key ({0})".format(exc),
+                    key_node.start_mark)
+            value = self.construct_object(value_node, deep=deep)
+            mapping[key] = value
+        return mapping
+
+
+class OrderedDumper(yaml.SafeDumper):
+    """A YAML dumper that dumps ordered dictionaries into mappings."""
+
+    def __init__(self, *args, **kwargs):
+        super(OrderedDumper, self).__init__(*args, **kwargs)
+        self.add_representer(OrderedDict, type(self).represent_dict)
+
+    def represent_mapping(self, tag, mapping, flow_style=None):
+        value = []
+        node = yaml.MappingNode(tag, value, flow_style=flow_style)
+        if self.alias_key is not None:
+            self.represented_objects[self.alias_key] = node
+        best_style = True
+        if hasattr(mapping, "items"):
+            mapping = list(mapping.items())
+        for item_key, item_value in mapping:
+            node_key = self.represent_data(item_key)
+            node_value = self.represent_data(item_value)
+            if not (isinstance(node_key, yaml.ScalarNode) and not
+                    node_key.style):
+                best_style = False
+            if not (isinstance(node_value, yaml.ScalarNode) and not
+                    node_value.style):
+                best_style = False
+            value.append((node_key, node_value))
+        if flow_style is None:
+            if self.default_flow_style is not None:
+                node.flow_style = self.default_flow_style
+            else:
+                node.flow_style = best_style
+        return node

earwigbot/config/script.py

@@ -0,0 +1,453 @@
+# -*- coding: utf-8  -*-
+#
+# Copyright (C) 2009-2012 Ben Kurtovic <ben.kurtovic@verizon.net>
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+from collections import OrderedDict
+from getpass import getpass
+from hashlib import sha256
+from os import chmod, mkdir, path
+import re
+import stat
+import sys
+from textwrap import fill, wrap
+
+try:
+    from Crypto.Cipher import Blowfish
+except ImportError:
+    Blowfish = None
+
+try:
+    import bcrypt
+except ImportError:
+    bcrypt = None
+
+try:
+    import yaml
+except ImportError:
+    yaml = None
+
+from earwigbot import exceptions
+from earwigbot.config.ordered_yaml import OrderedDumper
+
+__all__ = ["ConfigScript"]
+
+RULES_TEMPLATE = """# -*- coding: utf-8  -*-
+
+def process(bot, rc):
+    \"\"\"Given a Bot() object and an RC() object, return a list of channels
+    to report this event to. Also, start any wiki bot tasks within this
+    function if necessary.\"\"\"
+    pass
+"""
+
+class ConfigScript(object):
+    """A script to guide a user through the creation of a new config file."""
+    WIDTH = 79
+    PROMPT = "\x1b[32m> \x1b[0m"
+    BCRYPT_ROUNDS = 12
+
+    def __init__(self, config):
+        self.config = config
+        self.data = OrderedDict([
+            ("metadata", OrderedDict()),
+            ("components", OrderedDict()),
+            ("wiki", OrderedDict()),
+            ("irc", OrderedDict()),
+            ("commands", OrderedDict()),
+            ("tasks", OrderedDict()),
+            ("schedule", [])
+        ])
+
+        self._cipher = None
+        self._wmf = False
+        self._proj = None
+        self._lang = None
+
+    def _print(self, text):
+        print fill(re.sub("\s\s+", " ", text), self.WIDTH)
+
+    def _print_no_nl(self, text):
+        sys.stdout.write(fill(re.sub("\s\s+", " ", text), self.WIDTH))
+        sys.stdout.flush()
+
+    def _pause(self):
+        raw_input(self.PROMPT + "Press enter to continue: ")
+
+    def _ask(self, text, default=None):
+        text = self.PROMPT + text
+        if default:
+            text += " \x1b[33m[{0}]\x1b[0m".format(default)
+        lines = wrap(re.sub("\s\s+", " ", text), self.WIDTH)
+        if len(lines) > 1:
+            print "\n".join(lines[:-1])
+        return raw_input(lines[-1] + " ") or default
+
+    def _ask_bool(self, text, default=True):
+        text = self.PROMPT + text
+        if default:
+            text += " \x1b[33m[Y/n]\x1b[0m"
+        else:
+            text += " \x1b[33m[y/N]\x1b[0m"
+        lines = wrap(re.sub("\s\s+", " ", text), self.WIDTH)
+        if len(lines) > 1:
+            print "\n".join(lines[:-1])
+        while True:
+            answer = raw_input(lines[-1] + " ").lower()
+            if not answer:
+                return default
+            if answer.startswith("y"):
+                return True
+            if answer.startswith("n"):
+                return False
+
+    def _ask_pass(self, text, encrypt=True):
+        password = getpass(self.PROMPT + text + " ")
+        if encrypt:
+            return self._encrypt(password)
+        return password
+
+    def _encrypt(self, password):
+        if self._cipher:
+            mod = len(password) % 8
+            if mod:
+                password = password.ljust(len(password) + (8 - mod), "\x00")
+            return self._cipher.encrypt(password).encode("hex")
+        else:
+            return password
+
+    def _ask_list(self, text):
+        print fill(re.sub("\s\s+", " ", self.PROMPT + text), self.WIDTH)
+        print "[one item per line; blank line to end]:"
+        result = []
+        while True:
+            line = raw_input(self.PROMPT)
+            if line:
+                result.append(line)
+            else:
+                return result
+
+    def _set_metadata(self):
+        print
+        self.data["metadata"] = OrderedDict([("version", 1)])
+        self._print("""I can encrypt passwords stored in your config file in
+                       addition to preventing other users on your system from
+                       reading the file. Encryption is recommended is the bot
+                       is to run on a public computer like the Toolserver, but
+                       otherwise the need to enter a key everytime you start
+                       the bot may be annoying.""")
+        if self._ask_bool("Encrypt stored passwords?"):
+            self.data["metadata"]["encryptPasswords"] = True
+            key = getpass(self.PROMPT + "Enter an encryption key: ")
+            msg = "Running {0} rounds of bcrypt...".format(self.BCRYPT_ROUNDS)
+            self._print_no_nl(msg)
+            signature = bcrypt.hashpw(key, bcrypt.gensalt(self.BCRYPT_ROUNDS))
+            self.data["metadata"]["signature"] = signature
+            self._cipher = Blowfish.new(sha256(key).digest())
+            print " done."
+        else:
+            self.data["metadata"]["encryptPasswords"] = False
+
+        print
+        self._print("""The bot can temporarily store its logs in the logs/
+                       subdirectory. Error logs are kept for a month whereas
+                       normal logs are kept for a week. If you disable this,
+                       the bot will still print logs to stdout.""")
+        logging = self._ask_bool("Enable logging?")
+        self.data["metadata"]["enableLogging"] = logging
+
+    def _set_components(self):
+        print
+        self._print("""The bot contains three separate components that can run
+                       independently of each other.""")
+        self._print("""- The IRC front-end runs on a normal IRC server, like
+                       freenode, and expects users to interact with it through
+                       commands.""")
+        self._print("""- The IRC watcher runs on a wiki recent-changes server,
+                       like irc.wikimedia.org, and listens for edits. Users
+                       cannot interact with this component. It can detect
+                       specific events and report them to "feed" channels on
+                       the front-end, or start bot tasks.""")
+        self._print("""- The wiki task scheduler runs wiki-editing bot tasks in
+                       separate threads at user-defined times through a
+                       cron-like interface. Tasks which are not scheduled can
+                       be started by the IRC watcher manually through the IRC
+                       front-end.""")
+        frontend = self._ask_bool("Enable the IRC front-end?")
+        watcher = self._ask_bool("Enable the IRC watcher?")
+        scheduler  = self._ask_bool("Enable the wiki task scheduler?")
+        self.data["components"]["irc_frontend"] = frontend
+        self.data["components"]["irc_watcher"] = watcher
+        self.data["components"]["wiki_scheduler"] = scheduler
+
+    def _login(self, kwargs):
+        self.config.wiki._load(self.data["wiki"])
+        self._print_no_nl("Trying to connect to the site...")
+        try:
+            site = self.config.bot.wiki.add_site(**kwargs)
+        except exceptions.APIError as exc:
+            print " API error!"
+            print "\x1b[31m" + exc.message + "\x1b[0m"
+            question = "Would you like to re-enter the site information?"
+            if self._ask_bool(question):
+                return self._set_wiki()
+            question = "This will cancel the setup process. Are you sure?"
+            if self._ask_bool(question, default=False):
+                raise exceptions.NoConfigError()
+            return self._set_wiki()
+        except exceptions.LoginError as exc:
+            print " login error!"
+            print "\x1b[31m" + exc.message + "\x1b[0m"
+            question = "Would you like to re-enter your login information?"
+            if self._ask_bool(question):
+                self.data["wiki"]["username"] = self._ask("Bot username:")
+                password = self._ask_pass("Bot password:", encrypt=False)
+                self.data["wiki"]["password"] = password
+                return self._login(kwargs)
+            else:
+                password = self.data["wiki"]["password"]
+            question = "Would you like to re-enter the site information?"
+            if self._ask_bool(question):
+                return self._set_wiki()
+            print
+            self._print("""Moving on. You can modify the login information
+                           stored in the bot's config in the future.""")
+            self.data["wiki"]["password"] = None  # Clear so we don't login
+            self.config.wiki._load(self.data["wiki"])
+            self._print_no_nl("Trying to connect to the site...")
+            site = self.config.bot.wiki.add_site(**kwargs)
+            print " success."
+            self.data["wiki"]["password"] = password  # Reset original value
+        else:
+            print " success."
+
+        # Remember to store the encrypted password:
+        password = self._encrypt(self.data["wiki"]["password"])
+        self.data["wiki"]["password"] = password
+        return site
+
+    def _set_wiki(self):
+        print
+        self._wmf = self._ask_bool("""Will this bot run on Wikimedia Foundation
+                                      wikis, like Wikipedia?""")
+        if self._wmf:
+            msg = "Site project (e.g. 'wikipedia', 'wiktionary', 'wikimedia'):"
+            self._proj = project = self._ask(msg, default="wikipedia").lower()
+            msg = "Site language code (e.g. 'en', 'fr', 'commons'):"
+            self._lang = lang = self._ask(msg, default="en").lower()
+            kwargs = {"project": project, "lang": lang}
+        else:
+            msg = "Site base URL, without the script path and trailing slash;"
+            msg += " can be protocol-insensitive (e.g. '//en.wikipedia.org'):"
+            url = self._ask(msg)
+            script = self._ask("Site script path:", default="/w")
+            kwargs = {"base_url": url, "script_path": script}
+
+        self.data["wiki"]["username"] = self._ask("Bot username:")
+        password = self._ask_pass("Bot password:", encrypt=False)
+        self.data["wiki"]["password"] = password
+        self.data["wiki"]["userAgent"] = "EarwigBot/$1 (Python/$2; https://github.com/earwig/earwigbot)"
+        self.data["wiki"]["summary"] = "([[WP:BOT|Bot]]): $2"
+        self.data["wiki"]["useHTTPS"] = True
+        self.data["wiki"]["assert"] = "user"
+        self.data["wiki"]["maxlag"] = 10
+        self.data["wiki"]["waitTime"] = 3
+        self.data["wiki"]["defaultSite"] = self._login(kwargs).name
+        self.data["wiki"]["sql"] = {}
+
+        if self._wmf:
+            msg = "Will this bot run from the Wikimedia Toolserver?"
+            toolserver = self._ask_bool(msg, default=False)
+            if toolserver:
+                args = [("host", "$1-p.rrdb.toolserver.org"), ("db", "$1_p")]
+                self.data["wiki"]["sql"] = OrderedDict(args)
+
+        self.data["wiki"]["shutoff"] = {}
+        msg = "Would you like to enable an automatic shutoff page for the bot?"
+        if self._ask_bool(msg):
+            print
+            self._print("""The page title can contain two wildcards: $1 will be
+                           substituted with the bot's username, and $2 with the
+                           current task number. This can be used to implement a
+                           separate shutoff page for each task.""")
+            page = self._ask("Page title:", default="User:$1/Shutoff")
+            msg = "Page content to indicate the bot is *not* shut off:"
+            disabled = self._ask(msg, "run")
+            args = [("page", page), ("disabled", disabled)]
+            self.data["wiki"]["shutoff"] = OrderedDict(args)
+
+        self.data["wiki"]["search"] = {}
+
+    def _set_irc(self):
+        if self.data["components"]["irc_frontend"]:
+            print
+            frontend = self.data["irc"]["frontend"] = OrderedDict()
+            msg = "Hostname of the frontend's IRC server, without 'irc://':"
+            frontend["host"] = self._ask(msg, "irc.freenode.net")
+            frontend["port"] = self._ask("Frontend port:", 6667)
+            frontend["nick"] = self._ask("Frontend bot's nickname:")
+            frontend["ident"] = self._ask("Frontend bot's ident:",
+                                          frontend["nick"].lower())
+            question = "Frontend bot's real name (gecos):"
+            frontend["realname"] = self._ask(question)
+            if self._ask_bool("Should the bot identify to NickServ?"):
+                ns_user = self._ask("NickServ username:", frontend["nick"])
+                ns_pass = self._ask_pass("Nickserv password:")
+                frontend["nickservUsername"] = ns_user
+                frontend["nickservPassword"] = ns_pass
+            chan_question = "Frontend channels to join by default:"
+            frontend["channels"] = self._ask_list(chan_question)
+            print
+            self._print("""The bot keeps a database of its admins (users who
+                           can use certain sensitive commands) and owners
+                           (users who can quit the bot and modify its access
+                           list), identified by nick, ident, and/or hostname.
+                           Hostname is the most secure option since it cannot
+                           be easily spoofed. If you have a cloak, this will
+                           probably look like 'wikipedia/Username' or
+                           'unaffiliated/nickname'.""")
+            host = self._ask("Your hostname on the IRC frontend:")
+            if host:
+                permdb = self.config._permissions
+                permdb.load()
+                permdb.add_owner(host=host)
+                permdb.add_admin(host=host)
+        else:
+            frontend = {}
+
+        if self.data["components"]["irc_watcher"]:
+            print
+            watcher = self.data["irc"]["watcher"] = OrderedDict()
+            if self._wmf:
+                watcher["host"] = "irc.wikimedia.org"
+                watcher["port"] = 6667
+            else:
+                msg = "Hostname of the watcher's IRC server, without 'irc://':"
+                watcher["host"] = self._ask(msg)
+                watcher["port"] = self._ask("Watcher port:", 6667)
+            nick = self._ask("Watcher bot's nickname:", frontend.get("nick"))
+            ident = self._ask("Watcher bot's ident:", nick.lower())
+            watcher["nick"] = nick
+            watcher["ident"] = ident
+            question = "Watcher bot's real name (gecos):"
+            watcher["realname"] = self._ask(question, frontend.get("realname"))
+            watcher_ns = "Should the bot identify to NickServ?"
+            if not self._wmf and self._ask_bool(watcher_ns):
+                ns_user = self._ask("NickServ username:", watcher["nick"])
+                ns_pass = self._ask_pass("Nickserv password:")
+                watcher["nickservUsername"] = ns_user
+                watcher["nickservPassword"] = ns_pass
+            if self._wmf:
+                chan = "#{0}.{1}".format(self._lang, self._proj)
+                watcher["channels"] = [chan]
+            else:
+                chan_question = "Watcher channels to join by default:"
+                watcher["channels"] = self._ask_list(chan_question)
+            print
+            self._print("""I am now creating a blank 'rules.py' file, which
+                           will determine how the bot handles messages received
+                           from the IRC watcher. It contains a process()
+                           function that takes a Bot object (allowing you to
+                           start tasks) and an RC object (storing the message
+                           from the watcher). See the documentation for
+                           details.""")
+            with open(path.join(self.config.root_dir, "rules.py"), "w") as fp:
+                fp.write(RULES_TEMPLATE)
+            self._pause()
+
+        self.data["irc"]["version"] = "EarwigBot - $1 - Python/$2 https://github.com/earwig/earwigbot"
+
+    def _set_commands(self):
+        print
+        msg = """Would you like to disable the default IRC commands? You can
+                 fine-tune which commands are disabled later on."""
+        if (not self.data["components"]["irc_frontend"] or
+                self._ask_bool(msg, default=False)):
+            self.data["commands"]["disable"] = True
+        print
+        self._print("""I am now creating the 'commands/' directory, where you
+                       can place custom IRC commands and plugins. Creating your
+                       own commands is described in the documentation.""")
+        mkdir(path.join(self.config.root_dir, "commands"))
+        self._pause()
+
+    def _set_tasks(self):
+        print
+        self._print("""I am now creating the 'tasks/' directory, where you can
+                       place custom bot tasks and plugins. Creating your own
+                       tasks is described in the documentation.""")
+        mkdir(path.join(self.config.root_dir, "tasks"))
+        self._pause()
+
+    def _set_schedule(self):
+        print
+        self._print("""The final section of your config file, 'schedule', is a
+                       list of bot tasks to be started by the wiki scheduler.
+                       Each entry contains cron-like time quantifiers and a
+                       list of tasks. For example, the following starts the
+                       'foobot' task every hour on the half-hour:""")
+        print "\x1b[33mschedule:"
+        print "    - minute: 30"
+        print "      tasks:"
+        print "          - foobot\x1b[0m"
+        self._print("""The following starts the 'barbot' task with the keyword
+                       arguments 'action="baz"' every Monday at 05:00 UTC:""")
+        print "\x1b[33m    - week_day: 1"
+        print "      hour:     5"
+        print "      tasks:"
+        print '          - ["barbot", {"action": "baz"}]\x1b[0m'
+        self._print("""The full list of quantifiers is minute, hour, month_day,
+                       month, and week_day. See the documentation for more
+                       information.""")
+        self._pause()
+
+    def _save(self):
+        with open(self.config.path, "w") as stream:
+            yaml.dump(self.data, stream, OrderedDumper, indent=4,
+                      allow_unicode=True, default_flow_style=False)
+
+    def make_new(self):
+        """Make a new config file based on the user's input."""
+        try:
+            open(self.config.path, "w").close()
+            chmod(self.config.path, stat.S_IRUSR|stat.S_IWUSR)
+        except IOError:
+            print "I can't seem to write to the config file:"
+            raise
+        self._set_metadata()
+        self._set_components()
+        self._set_wiki()
+        components = self.data["components"]
+        if components["irc_frontend"] or components["irc_watcher"]:
+            self._set_irc()
+            self._set_commands()
+        self._set_tasks()
+        if components["wiki_scheduler"]:
+            self._set_schedule()
+        print
+        self._print("""I am now saving config.yml with your settings. YAML is a
+                       relatively straightforward format and you should be able
+                       to update these settings in the future when necessary.
+                       I will start the bot at your signal. Feel free to
+                       contact me at wikipedia.earwig@gmail.com if you have any
+                       questions.""")
+        self._save()
+        if not self._ask_bool("Start the bot now?"):
+            exit()

earwigbot/irc/watcher.py

@@ -94,7 +94,7 @@ class Watcher(IRCConnection):
         except ImportError:
             return
         try:
-            module = imp.load_module(name, f, path, desc)
+            module = imp.load_module("rules", f, path, desc)
         except Exception:
             return
         finally:

earwigbot/managers.py

@@ -149,11 +149,15 @@ class _ResourceManager(object):
             builtin_dir = path.join(path.dirname(__file__), name)
             plugins_dir = path.join(self.bot.config.root_dir, name)
             if getattr(self.bot.config, name).get("disable") is True:
-                log = "Skipping disabled builtins directory {0}"
+                log = "Skipping disabled builtins directory: {0}"
                 self.logger.debug(log.format(builtin_dir))
             else:
                 self._load_directory(builtin_dir)  # Built-in resources
-            self._load_directory(plugins_dir)  # Custom resources, aka plugins
+            if path.exists(plugins_dir) and path.isdir(plugins_dir):
+                self._load_directory(plugins_dir)  # Custom resources, plugins
+            else:
+                log = "Skipping nonexistent plugins directory: {0}"
+                self.logger.debug(log.format(plugins_dir))
 
         if self._resources:
             msg = "Loaded {0} {1}: {2}"

earwigbot/wiki/copyvios/__init__.py

@@ -52,7 +52,7 @@ class CopyvioMixIn(object):
 
     def __init__(self, site):
         self._search_config = site._search_config
-        self._exclusions_db = self._search_config["exclusions_db"]
+        self._exclusions_db = self._search_config.get("exclusions_db")
         self._opener = build_opener()
         self._opener.addheaders = site._opener.addheaders
 
@@ -137,7 +137,8 @@ class CopyvioMixIn(object):
         :py:exc:`~earwigbot.exceptions.SearchQueryError`, ...) on errors.
         """
         searcher = self._select_search_engine()
-        self._exclusions_db.sync(self.site.name)
+        if self._exclusions_db:
+            self._exclusions_db.sync(self.site.name)
         handled_urls = []
         best_confidence = 0
         best_match = None
@@ -163,8 +164,9 @@ class CopyvioMixIn(object):
             urls = [url for url in urls if url not in handled_urls]
             for url in urls:
                 handled_urls.append(url)
-                if self._exclusions_db.check(self.site.name, url):
-                    continue
+                if self._exclusions_db:
+                    if self._exclusions_db.check(self.site.name, url):
+                        continue
                 conf, chains = self._copyvio_compare_content(article_chain, url)
                 if conf > best_confidence:
                     best_confidence = conf

earwigbot/wiki/sitesdb.py

@@ -20,6 +20,7 @@
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 # SOFTWARE.
 
+from collections import OrderedDict
 from cookielib import LWPCookieJar, LoadError
 import errno
 from os import chmod, path
@@ -192,7 +193,7 @@ class SitesDB(object):
         maxlag = config.wiki.get("maxlag")
         wait_between_queries = config.wiki.get("waitTime", 3)
         logger = self._logger.getChild(name)
-        search_config = config.wiki.get("search", {}).copy()
+        search_config = config.wiki.get("search", OrderedDict()).copy()
 
         if user_agent:
             user_agent = user_agent.replace("$1", __version__)
@@ -204,7 +205,7 @@ class SitesDB(object):
             search_config["exclusions_db"] = self._exclusions_db
 
         if not sql:
-            sql = config.wiki.get("sql", {}).copy()
+            sql = config.wiki.get("sql", OrderedDict()).copy()
             for key, value in sql.iteritems():
                 if isinstance(value, basestring) and "$1" in value:
                     sql[key] = value.replace("$1", name)
@@ -386,7 +387,7 @@ class SitesDB(object):
         config = self.config
         login = (config.wiki.get("username"), config.wiki.get("password"))
         user_agent = config.wiki.get("userAgent")
-        use_https = config.wiki.get("useHTTPS", False)
+        use_https = config.wiki.get("useHTTPS", True)
         assert_edit = config.wiki.get("assert")
         maxlag = config.wiki.get("maxlag")
         wait_between_queries = config.wiki.get("waitTime", 3)