Browse Source

Merge branch 'feature/confscript' into develop (closes #1)

tags/v0.1^2
Ben Kurtovic 12 years ago
parent
commit
1fd3d01a50
10 changed files with 615 additions and 42 deletions
  1. +1
    -1
      earwigbot/bot.py
  2. +3
    -0
      earwigbot/commands/crypt.py
  3. +30
    -29
      earwigbot/config/__init__.py
  4. +4
    -2
      earwigbot/config/node.py
  5. +107
    -0
      earwigbot/config/ordered_yaml.py
  6. +453
    -0
      earwigbot/config/script.py
  7. +1
    -1
      earwigbot/irc/watcher.py
  8. +6
    -2
      earwigbot/managers.py
  9. +6
    -4
      earwigbot/wiki/copyvios/__init__.py
  10. +4
    -3
      earwigbot/wiki/sitesdb.py

+ 1
- 1
earwigbot/bot.py View File

@@ -60,7 +60,7 @@ class Bot(object):
"""

def __init__(self, root_dir, level=logging.INFO):
self.config = BotConfig(root_dir, level)
self.config = BotConfig(self, root_dir, level)
self.logger = logging.getLogger("earwigbot")
self.commands = CommandManager(self)
self.tasks = TaskManager(self)


+ 3
- 0
earwigbot/commands/crypt.py View File

@@ -69,6 +69,9 @@ class Crypt(Command):
cipher = Blowfish.new(hashlib.sha256(key).digest())
try:
if data.command == "encrypt":
if len(text) % 8:
pad = 8 - len(text) % 8
text = text.ljust(len(text) + pad, "\x00")
self.reply(data, cipher.encrypt(text).encode("hex"))
else:
self.reply(data, cipher.decrypt(text.decode("hex")))


+ 30
- 29
earwigbot/config/__init__.py View File

@@ -20,11 +20,13 @@
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.

from collections import OrderedDict
from getpass import getpass
from hashlib import sha256
import logging
import logging.handlers
from os import mkdir, path
import stat

try:
from Crypto.Cipher import Blowfish
@@ -43,7 +45,9 @@ except ImportError:

from earwigbot.config.formatter import BotFormatter
from earwigbot.config.node import ConfigNode
from earwigbot.config.ordered_yaml import OrderedLoader
from earwigbot.config.permissions import PermissionsDB
from earwigbot.config.script import ConfigScript
from earwigbot.exceptions import NoConfigError

__all__ = ["BotConfig"]
@@ -75,7 +79,8 @@ class BotConfig(object):
- :py:meth:`decrypt`: decrypts an object in the config tree
"""

def __init__(self, root_dir, level):
def __init__(self, bot, root_dir, level):
self._bot = bot
self._root_dir = root_dir
self._logging_level = level
self._config_path = path.join(self.root_dir, "config.yml")
@@ -112,12 +117,21 @@ class BotConfig(object):
"""Return a nice string representation of the BotConfig."""
return "<BotConfig at {0}>".format(self.root_dir)

def _handle_missing_config(self):
print "Config file missing or empty:", self._config_path
msg = "Would you like to create a config file now? [Y/n] "
choice = raw_input(msg)
if choice.lower().startswith("n"):
raise NoConfigError()
else:
ConfigScript(self).make_new()

def _load(self):
"""Load data from our JSON config file (config.yml) into self._data."""
filename = self._config_path
with open(filename, 'r') as fp:
try:
self._data = yaml.load(fp)
self._data = yaml.load(fp, OrderedLoader)
except yaml.YAMLError:
print "Error parsing config file {0}:".format(filename)
raise
@@ -137,7 +151,7 @@ class BotConfig(object):

if not path.isdir(log_dir):
if not path.exists(log_dir):
mkdir(log_dir, 0700)
mkdir(log_dir, stat.S_IWUSR|stat.S_IRUSR|stat.S_IXUSR)
else:
msg = "log_dir ({0}) exists but is not a directory!"
print msg.format(log_dir)
@@ -168,19 +182,10 @@ class BotConfig(object):
print "Error decrypting passwords:"
raise

def _make_new(self):
"""Make a new config file based on the user's input."""
#m = "Would you like to encrypt passwords stored in config.yml? [y/n] "
#encrypt = raw_input(m)
#if encrypt.lower().startswith("y"):
# is_encrypted = True
#else:
# is_encrypted = False
raise NotImplementedError()
# yaml.dumps() config.yml file (self._config_path)
# Create root_dir/, root_dir/commands/, root_dir/tasks/
# Give a reasonable message after config has been created regarding
# what to do next...
@property
def bot(self):
"""The config's Bot object."""
return self._bot

@property
def root_dir(self):
@@ -267,21 +272,17 @@ class BotConfig(object):
decrypted if they were decrypted earlier.
"""
if not path.exists(self._config_path):
print "Config file not found:", self._config_path
choice = raw_input("Would you like to create a config file now? [y/n] ")
if choice.lower().startswith("y"):
self._make_new()
else:
raise NoConfigError()

self._handle_missing_config()
self._load()
data = self._data
self.components._load(data.get("components", {}))
self.wiki._load(data.get("wiki", {}))
self.irc._load(data.get("irc", {}))
self.commands._load(data.get("commands", {}))
self.tasks._load(data.get("tasks", {}))
self.metadata._load(data.get("metadata", {}))
if not data:
self._handle_missing_config()
self.components._load(data.get("components", OrderedDict()))
self.wiki._load(data.get("wiki", OrderedDict()))
self.irc._load(data.get("irc", OrderedDict()))
self.commands._load(data.get("commands", OrderedDict()))
self.tasks._load(data.get("tasks", OrderedDict()))
self.metadata._load(data.get("metadata", OrderedDict()))

self._setup_logging()
if self.is_encrypted():


+ 4
- 2
earwigbot/config/node.py View File

@@ -20,11 +20,13 @@
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.

from collections import OrderedDict

__all__ = ["ConfigNode"]

class ConfigNode(object):
def __init__(self):
self._data = {}
self._data = OrderedDict()

def __repr__(self):
return self._data
@@ -99,4 +101,4 @@ class ConfigNode(object):
return self._data.itervalues()

def iteritems(self):
return self.__dict__.iteritems()
return self._data.iteritems()

+ 107
- 0
earwigbot/config/ordered_yaml.py View File

@@ -0,0 +1,107 @@
# -*- coding: utf-8 -*-
#
# Copyright (C) 2009-2012 Ben Kurtovic <ben.kurtovic@verizon.net>
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.

"""
Based on:
* https://gist.github.com/844388
* http://pyyaml.org/attachment/ticket/161/use_ordered_dict.py
with modifications.
"""

from collections import OrderedDict

try:
import yaml
except ImportError:
yaml = None

__all__ = ["OrderedLoader", "OrderedDumper"]

class OrderedLoader(yaml.Loader):
"""A YAML loader that loads mappings into ordered dictionaries."""

def __init__(self, *args, **kwargs):
super(OrderedLoader, self).__init__(*args, **kwargs)
constructor = type(self).construct_yaml_map
self.add_constructor(u"tag:yaml.org,2002:map", constructor)
self.add_constructor(u"tag:yaml.org,2002:omap", constructor)

def construct_yaml_map(self, node):
data = OrderedDict()
yield data
value = self.construct_mapping(node)
data.update(value)

def construct_mapping(self, node, deep=False):
if isinstance(node, yaml.MappingNode):
self.flatten_mapping(node)
else:
raise yaml.constructor.ConstructorError(None, None,
"expected a mapping node, but found {0}".format(node.id),
node.start_mark)

mapping = OrderedDict()
for key_node, value_node in node.value:
key = self.construct_object(key_node, deep=deep)
try:
hash(key)
except TypeError, exc:
raise yaml.constructor.ConstructorError(
"while constructing a mapping", node.start_mark,
"found unacceptable key ({0})".format(exc),
key_node.start_mark)
value = self.construct_object(value_node, deep=deep)
mapping[key] = value
return mapping


class OrderedDumper(yaml.SafeDumper):
"""A YAML dumper that dumps ordered dictionaries into mappings."""

def __init__(self, *args, **kwargs):
super(OrderedDumper, self).__init__(*args, **kwargs)
self.add_representer(OrderedDict, type(self).represent_dict)

def represent_mapping(self, tag, mapping, flow_style=None):
value = []
node = yaml.MappingNode(tag, value, flow_style=flow_style)
if self.alias_key is not None:
self.represented_objects[self.alias_key] = node
best_style = True
if hasattr(mapping, "items"):
mapping = list(mapping.items())
for item_key, item_value in mapping:
node_key = self.represent_data(item_key)
node_value = self.represent_data(item_value)
if not (isinstance(node_key, yaml.ScalarNode) and not
node_key.style):
best_style = False
if not (isinstance(node_value, yaml.ScalarNode) and not
node_value.style):
best_style = False
value.append((node_key, node_value))
if flow_style is None:
if self.default_flow_style is not None:
node.flow_style = self.default_flow_style
else:
node.flow_style = best_style
return node

+ 453
- 0
earwigbot/config/script.py View File

@@ -0,0 +1,453 @@
# -*- coding: utf-8 -*-
#
# Copyright (C) 2009-2012 Ben Kurtovic <ben.kurtovic@verizon.net>
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.

from collections import OrderedDict
from getpass import getpass
from hashlib import sha256
from os import chmod, mkdir, path
import re
import stat
import sys
from textwrap import fill, wrap

try:
from Crypto.Cipher import Blowfish
except ImportError:
Blowfish = None

try:
import bcrypt
except ImportError:
bcrypt = None

try:
import yaml
except ImportError:
yaml = None

from earwigbot import exceptions
from earwigbot.config.ordered_yaml import OrderedDumper

__all__ = ["ConfigScript"]

RULES_TEMPLATE = """# -*- coding: utf-8 -*-

def process(bot, rc):
\"\"\"Given a Bot() object and an RC() object, return a list of channels
to report this event to. Also, start any wiki bot tasks within this
function if necessary.\"\"\"
pass
"""

class ConfigScript(object):
"""A script to guide a user through the creation of a new config file."""
WIDTH = 79
PROMPT = "\x1b[32m> \x1b[0m"
BCRYPT_ROUNDS = 12

def __init__(self, config):
self.config = config
self.data = OrderedDict([
("metadata", OrderedDict()),
("components", OrderedDict()),
("wiki", OrderedDict()),
("irc", OrderedDict()),
("commands", OrderedDict()),
("tasks", OrderedDict()),
("schedule", [])
])

self._cipher = None
self._wmf = False
self._proj = None
self._lang = None

def _print(self, text):
print fill(re.sub("\s\s+", " ", text), self.WIDTH)

def _print_no_nl(self, text):
sys.stdout.write(fill(re.sub("\s\s+", " ", text), self.WIDTH))
sys.stdout.flush()

def _pause(self):
raw_input(self.PROMPT + "Press enter to continue: ")

def _ask(self, text, default=None):
text = self.PROMPT + text
if default:
text += " \x1b[33m[{0}]\x1b[0m".format(default)
lines = wrap(re.sub("\s\s+", " ", text), self.WIDTH)
if len(lines) > 1:
print "\n".join(lines[:-1])
return raw_input(lines[-1] + " ") or default

def _ask_bool(self, text, default=True):
text = self.PROMPT + text
if default:
text += " \x1b[33m[Y/n]\x1b[0m"
else:
text += " \x1b[33m[y/N]\x1b[0m"
lines = wrap(re.sub("\s\s+", " ", text), self.WIDTH)
if len(lines) > 1:
print "\n".join(lines[:-1])
while True:
answer = raw_input(lines[-1] + " ").lower()
if not answer:
return default
if answer.startswith("y"):
return True
if answer.startswith("n"):
return False

def _ask_pass(self, text, encrypt=True):
password = getpass(self.PROMPT + text + " ")
if encrypt:
return self._encrypt(password)
return password

def _encrypt(self, password):
if self._cipher:
mod = len(password) % 8
if mod:
password = password.ljust(len(password) + (8 - mod), "\x00")
return self._cipher.encrypt(password).encode("hex")
else:
return password

def _ask_list(self, text):
print fill(re.sub("\s\s+", " ", self.PROMPT + text), self.WIDTH)
print "[one item per line; blank line to end]:"
result = []
while True:
line = raw_input(self.PROMPT)
if line:
result.append(line)
else:
return result

def _set_metadata(self):
print
self.data["metadata"] = OrderedDict([("version", 1)])
self._print("""I can encrypt passwords stored in your config file in
addition to preventing other users on your system from
reading the file. Encryption is recommended is the bot
is to run on a public computer like the Toolserver, but
otherwise the need to enter a key everytime you start
the bot may be annoying.""")
if self._ask_bool("Encrypt stored passwords?"):
self.data["metadata"]["encryptPasswords"] = True
key = getpass(self.PROMPT + "Enter an encryption key: ")
msg = "Running {0} rounds of bcrypt...".format(self.BCRYPT_ROUNDS)
self._print_no_nl(msg)
signature = bcrypt.hashpw(key, bcrypt.gensalt(self.BCRYPT_ROUNDS))
self.data["metadata"]["signature"] = signature
self._cipher = Blowfish.new(sha256(key).digest())
print " done."
else:
self.data["metadata"]["encryptPasswords"] = False

print
self._print("""The bot can temporarily store its logs in the logs/
subdirectory. Error logs are kept for a month whereas
normal logs are kept for a week. If you disable this,
the bot will still print logs to stdout.""")
logging = self._ask_bool("Enable logging?")
self.data["metadata"]["enableLogging"] = logging

def _set_components(self):
print
self._print("""The bot contains three separate components that can run
independently of each other.""")
self._print("""- The IRC front-end runs on a normal IRC server, like
freenode, and expects users to interact with it through
commands.""")
self._print("""- The IRC watcher runs on a wiki recent-changes server,
like irc.wikimedia.org, and listens for edits. Users
cannot interact with this component. It can detect
specific events and report them to "feed" channels on
the front-end, or start bot tasks.""")
self._print("""- The wiki task scheduler runs wiki-editing bot tasks in
separate threads at user-defined times through a
cron-like interface. Tasks which are not scheduled can
be started by the IRC watcher manually through the IRC
front-end.""")
frontend = self._ask_bool("Enable the IRC front-end?")
watcher = self._ask_bool("Enable the IRC watcher?")
scheduler = self._ask_bool("Enable the wiki task scheduler?")
self.data["components"]["irc_frontend"] = frontend
self.data["components"]["irc_watcher"] = watcher
self.data["components"]["wiki_scheduler"] = scheduler

def _login(self, kwargs):
self.config.wiki._load(self.data["wiki"])
self._print_no_nl("Trying to connect to the site...")
try:
site = self.config.bot.wiki.add_site(**kwargs)
except exceptions.APIError as exc:
print " API error!"
print "\x1b[31m" + exc.message + "\x1b[0m"
question = "Would you like to re-enter the site information?"
if self._ask_bool(question):
return self._set_wiki()
question = "This will cancel the setup process. Are you sure?"
if self._ask_bool(question, default=False):
raise exceptions.NoConfigError()
return self._set_wiki()
except exceptions.LoginError as exc:
print " login error!"
print "\x1b[31m" + exc.message + "\x1b[0m"
question = "Would you like to re-enter your login information?"
if self._ask_bool(question):
self.data["wiki"]["username"] = self._ask("Bot username:")
password = self._ask_pass("Bot password:", encrypt=False)
self.data["wiki"]["password"] = password
return self._login(kwargs)
else:
password = self.data["wiki"]["password"]
question = "Would you like to re-enter the site information?"
if self._ask_bool(question):
return self._set_wiki()
print
self._print("""Moving on. You can modify the login information
stored in the bot's config in the future.""")
self.data["wiki"]["password"] = None # Clear so we don't login
self.config.wiki._load(self.data["wiki"])
self._print_no_nl("Trying to connect to the site...")
site = self.config.bot.wiki.add_site(**kwargs)
print " success."
self.data["wiki"]["password"] = password # Reset original value
else:
print " success."

# Remember to store the encrypted password:
password = self._encrypt(self.data["wiki"]["password"])
self.data["wiki"]["password"] = password
return site

def _set_wiki(self):
print
self._wmf = self._ask_bool("""Will this bot run on Wikimedia Foundation
wikis, like Wikipedia?""")
if self._wmf:
msg = "Site project (e.g. 'wikipedia', 'wiktionary', 'wikimedia'):"
self._proj = project = self._ask(msg, default="wikipedia").lower()
msg = "Site language code (e.g. 'en', 'fr', 'commons'):"
self._lang = lang = self._ask(msg, default="en").lower()
kwargs = {"project": project, "lang": lang}
else:
msg = "Site base URL, without the script path and trailing slash;"
msg += " can be protocol-insensitive (e.g. '//en.wikipedia.org'):"
url = self._ask(msg)
script = self._ask("Site script path:", default="/w")
kwargs = {"base_url": url, "script_path": script}

self.data["wiki"]["username"] = self._ask("Bot username:")
password = self._ask_pass("Bot password:", encrypt=False)
self.data["wiki"]["password"] = password
self.data["wiki"]["userAgent"] = "EarwigBot/$1 (Python/$2; https://github.com/earwig/earwigbot)"
self.data["wiki"]["summary"] = "([[WP:BOT|Bot]]): $2"
self.data["wiki"]["useHTTPS"] = True
self.data["wiki"]["assert"] = "user"
self.data["wiki"]["maxlag"] = 10
self.data["wiki"]["waitTime"] = 3
self.data["wiki"]["defaultSite"] = self._login(kwargs).name
self.data["wiki"]["sql"] = {}

if self._wmf:
msg = "Will this bot run from the Wikimedia Toolserver?"
toolserver = self._ask_bool(msg, default=False)
if toolserver:
args = [("host", "$1-p.rrdb.toolserver.org"), ("db", "$1_p")]
self.data["wiki"]["sql"] = OrderedDict(args)

self.data["wiki"]["shutoff"] = {}
msg = "Would you like to enable an automatic shutoff page for the bot?"
if self._ask_bool(msg):
print
self._print("""The page title can contain two wildcards: $1 will be
substituted with the bot's username, and $2 with the
current task number. This can be used to implement a
separate shutoff page for each task.""")
page = self._ask("Page title:", default="User:$1/Shutoff")
msg = "Page content to indicate the bot is *not* shut off:"
disabled = self._ask(msg, "run")
args = [("page", page), ("disabled", disabled)]
self.data["wiki"]["shutoff"] = OrderedDict(args)

self.data["wiki"]["search"] = {}

def _set_irc(self):
if self.data["components"]["irc_frontend"]:
print
frontend = self.data["irc"]["frontend"] = OrderedDict()
msg = "Hostname of the frontend's IRC server, without 'irc://':"
frontend["host"] = self._ask(msg, "irc.freenode.net")
frontend["port"] = self._ask("Frontend port:", 6667)
frontend["nick"] = self._ask("Frontend bot's nickname:")
frontend["ident"] = self._ask("Frontend bot's ident:",
frontend["nick"].lower())
question = "Frontend bot's real name (gecos):"
frontend["realname"] = self._ask(question)
if self._ask_bool("Should the bot identify to NickServ?"):
ns_user = self._ask("NickServ username:", frontend["nick"])
ns_pass = self._ask_pass("Nickserv password:")
frontend["nickservUsername"] = ns_user
frontend["nickservPassword"] = ns_pass
chan_question = "Frontend channels to join by default:"
frontend["channels"] = self._ask_list(chan_question)
print
self._print("""The bot keeps a database of its admins (users who
can use certain sensitive commands) and owners
(users who can quit the bot and modify its access
list), identified by nick, ident, and/or hostname.
Hostname is the most secure option since it cannot
be easily spoofed. If you have a cloak, this will
probably look like 'wikipedia/Username' or
'unaffiliated/nickname'.""")
host = self._ask("Your hostname on the IRC frontend:")
if host:
permdb = self.config._permissions
permdb.load()
permdb.add_owner(host=host)
permdb.add_admin(host=host)
else:
frontend = {}

if self.data["components"]["irc_watcher"]:
print
watcher = self.data["irc"]["watcher"] = OrderedDict()
if self._wmf:
watcher["host"] = "irc.wikimedia.org"
watcher["port"] = 6667
else:
msg = "Hostname of the watcher's IRC server, without 'irc://':"
watcher["host"] = self._ask(msg)
watcher["port"] = self._ask("Watcher port:", 6667)
nick = self._ask("Watcher bot's nickname:", frontend.get("nick"))
ident = self._ask("Watcher bot's ident:", nick.lower())
watcher["nick"] = nick
watcher["ident"] = ident
question = "Watcher bot's real name (gecos):"
watcher["realname"] = self._ask(question, frontend.get("realname"))
watcher_ns = "Should the bot identify to NickServ?"
if not self._wmf and self._ask_bool(watcher_ns):
ns_user = self._ask("NickServ username:", watcher["nick"])
ns_pass = self._ask_pass("Nickserv password:")
watcher["nickservUsername"] = ns_user
watcher["nickservPassword"] = ns_pass
if self._wmf:
chan = "#{0}.{1}".format(self._lang, self._proj)
watcher["channels"] = [chan]
else:
chan_question = "Watcher channels to join by default:"
watcher["channels"] = self._ask_list(chan_question)
print
self._print("""I am now creating a blank 'rules.py' file, which
will determine how the bot handles messages received
from the IRC watcher. It contains a process()
function that takes a Bot object (allowing you to
start tasks) and an RC object (storing the message
from the watcher). See the documentation for
details.""")
with open(path.join(self.config.root_dir, "rules.py"), "w") as fp:
fp.write(RULES_TEMPLATE)
self._pause()

self.data["irc"]["version"] = "EarwigBot - $1 - Python/$2 https://github.com/earwig/earwigbot"

def _set_commands(self):
print
msg = """Would you like to disable the default IRC commands? You can
fine-tune which commands are disabled later on."""
if (not self.data["components"]["irc_frontend"] or
self._ask_bool(msg, default=False)):
self.data["commands"]["disable"] = True
print
self._print("""I am now creating the 'commands/' directory, where you
can place custom IRC commands and plugins. Creating your
own commands is described in the documentation.""")
mkdir(path.join(self.config.root_dir, "commands"))
self._pause()

def _set_tasks(self):
print
self._print("""I am now creating the 'tasks/' directory, where you can
place custom bot tasks and plugins. Creating your own
tasks is described in the documentation.""")
mkdir(path.join(self.config.root_dir, "tasks"))
self._pause()

def _set_schedule(self):
print
self._print("""The final section of your config file, 'schedule', is a
list of bot tasks to be started by the wiki scheduler.
Each entry contains cron-like time quantifiers and a
list of tasks. For example, the following starts the
'foobot' task every hour on the half-hour:""")
print "\x1b[33mschedule:"
print " - minute: 30"
print " tasks:"
print " - foobot\x1b[0m"
self._print("""The following starts the 'barbot' task with the keyword
arguments 'action="baz"' every Monday at 05:00 UTC:""")
print "\x1b[33m - week_day: 1"
print " hour: 5"
print " tasks:"
print ' - ["barbot", {"action": "baz"}]\x1b[0m'
self._print("""The full list of quantifiers is minute, hour, month_day,
month, and week_day. See the documentation for more
information.""")
self._pause()

def _save(self):
with open(self.config.path, "w") as stream:
yaml.dump(self.data, stream, OrderedDumper, indent=4,
allow_unicode=True, default_flow_style=False)

def make_new(self):
"""Make a new config file based on the user's input."""
try:
open(self.config.path, "w").close()
chmod(self.config.path, stat.S_IRUSR|stat.S_IWUSR)
except IOError:
print "I can't seem to write to the config file:"
raise
self._set_metadata()
self._set_components()
self._set_wiki()
components = self.data["components"]
if components["irc_frontend"] or components["irc_watcher"]:
self._set_irc()
self._set_commands()
self._set_tasks()
if components["wiki_scheduler"]:
self._set_schedule()
print
self._print("""I am now saving config.yml with your settings. YAML is a
relatively straightforward format and you should be able
to update these settings in the future when necessary.
I will start the bot at your signal. Feel free to
contact me at wikipedia.earwig@gmail.com if you have any
questions.""")
self._save()
if not self._ask_bool("Start the bot now?"):
exit()

+ 1
- 1
earwigbot/irc/watcher.py View File

@@ -94,7 +94,7 @@ class Watcher(IRCConnection):
except ImportError:
return
try:
module = imp.load_module(name, f, path, desc)
module = imp.load_module("rules", f, path, desc)
except Exception:
return
finally:


+ 6
- 2
earwigbot/managers.py View File

@@ -149,11 +149,15 @@ class _ResourceManager(object):
builtin_dir = path.join(path.dirname(__file__), name)
plugins_dir = path.join(self.bot.config.root_dir, name)
if getattr(self.bot.config, name).get("disable") is True:
log = "Skipping disabled builtins directory {0}"
log = "Skipping disabled builtins directory: {0}"
self.logger.debug(log.format(builtin_dir))
else:
self._load_directory(builtin_dir) # Built-in resources
self._load_directory(plugins_dir) # Custom resources, aka plugins
if path.exists(plugins_dir) and path.isdir(plugins_dir):
self._load_directory(plugins_dir) # Custom resources, plugins
else:
log = "Skipping nonexistent plugins directory: {0}"
self.logger.debug(log.format(plugins_dir))

if self._resources:
msg = "Loaded {0} {1}: {2}"


+ 6
- 4
earwigbot/wiki/copyvios/__init__.py View File

@@ -52,7 +52,7 @@ class CopyvioMixIn(object):

def __init__(self, site):
self._search_config = site._search_config
self._exclusions_db = self._search_config["exclusions_db"]
self._exclusions_db = self._search_config.get("exclusions_db")
self._opener = build_opener()
self._opener.addheaders = site._opener.addheaders

@@ -137,7 +137,8 @@ class CopyvioMixIn(object):
:py:exc:`~earwigbot.exceptions.SearchQueryError`, ...) on errors.
"""
searcher = self._select_search_engine()
self._exclusions_db.sync(self.site.name)
if self._exclusions_db:
self._exclusions_db.sync(self.site.name)
handled_urls = []
best_confidence = 0
best_match = None
@@ -163,8 +164,9 @@ class CopyvioMixIn(object):
urls = [url for url in urls if url not in handled_urls]
for url in urls:
handled_urls.append(url)
if self._exclusions_db.check(self.site.name, url):
continue
if self._exclusions_db:
if self._exclusions_db.check(self.site.name, url):
continue
conf, chains = self._copyvio_compare_content(article_chain, url)
if conf > best_confidence:
best_confidence = conf


+ 4
- 3
earwigbot/wiki/sitesdb.py View File

@@ -20,6 +20,7 @@
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.

from collections import OrderedDict
from cookielib import LWPCookieJar, LoadError
import errno
from os import chmod, path
@@ -192,7 +193,7 @@ class SitesDB(object):
maxlag = config.wiki.get("maxlag")
wait_between_queries = config.wiki.get("waitTime", 3)
logger = self._logger.getChild(name)
search_config = config.wiki.get("search", {}).copy()
search_config = config.wiki.get("search", OrderedDict()).copy()

if user_agent:
user_agent = user_agent.replace("$1", __version__)
@@ -204,7 +205,7 @@ class SitesDB(object):
search_config["exclusions_db"] = self._exclusions_db

if not sql:
sql = config.wiki.get("sql", {}).copy()
sql = config.wiki.get("sql", OrderedDict()).copy()
for key, value in sql.iteritems():
if isinstance(value, basestring) and "$1" in value:
sql[key] = value.replace("$1", name)
@@ -386,7 +387,7 @@ class SitesDB(object):
config = self.config
login = (config.wiki.get("username"), config.wiki.get("password"))
user_agent = config.wiki.get("userAgent")
use_https = config.wiki.get("useHTTPS", False)
use_https = config.wiki.get("useHTTPS", True)
assert_edit = config.wiki.get("assert")
maxlag = config.wiki.get("maxlag")
wait_between_queries = config.wiki.get("waitTime", 3)


Loading…
Cancel
Save