Просмотр исходного кода

Moving rest of earwigbot.classes to earwigbot.tasks

tags/v0.1^2
Ben Kurtovic 12 лет назад
Родитель
Сommit
424a954af6
18 измененных файлов: 212 добавлений и 261 удалений
  1. +1
    -2
      earwigbot/__init__.py
  2. +0
    -23
      earwigbot/classes/__init__.py
  3. +0
    -117
      earwigbot/classes/base_task.py
  4. +1
    -1
      earwigbot/commands/__init__.py
  5. +2
    -2
      earwigbot/commands/afc_report.py
  6. +5
    -5
      earwigbot/commands/threads.py
  7. +5
    -8
      earwigbot/main.py
  8. +3
    -3
      earwigbot/rules.py
  9. +186
    -91
      earwigbot/tasks/__init__.py
  10. +1
    -1
      earwigbot/tasks/afc_catdelink.py
  11. +1
    -1
      earwigbot/tasks/afc_copyvios.py
  12. +1
    -1
      earwigbot/tasks/afc_dailycats.py
  13. +1
    -1
      earwigbot/tasks/afc_history.py
  14. +1
    -1
      earwigbot/tasks/afc_statistics.py
  15. +1
    -1
      earwigbot/tasks/afc_undated.py
  16. +1
    -1
      earwigbot/tasks/blptag.py
  17. +1
    -1
      earwigbot/tasks/feed_dailycats.py
  18. +1
    -1
      earwigbot/tasks/wrongmime.py

+ 1
- 2
earwigbot/__init__.py Просмотреть файл

@@ -32,6 +32,5 @@ __version__ = "0.1.dev"
__email__ = "ben.kurtovic@verizon.net"

from earwigbot import (
blowfish, config, classes, commands, config, irc, main, rules, runner,
tasks, tests, wiki
blowfish, commands, config, irc, main, rules, runner, tasks, tests, wiki
)

+ 0
- 23
earwigbot/classes/__init__.py Просмотреть файл

@@ -1,23 +0,0 @@
# -*- coding: utf-8 -*-
#
# Copyright (C) 2009-2012 by Ben Kurtovic <ben.kurtovic@verizon.net>
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.

from earwigbot.classes.base_task import *

+ 0
- 117
earwigbot/classes/base_task.py Просмотреть файл

@@ -1,117 +0,0 @@
# -*- coding: utf-8 -*-
#
# Copyright (C) 2009-2012 by Ben Kurtovic <ben.kurtovic@verizon.net>
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.

import logging

from earwigbot.config import config
from earwigbot import wiki

__all__ = ["BaseTask"]

class BaseTask(object):
"""A base class for bot tasks that edit Wikipedia."""
name = None
number = 0

def __init__(self):
"""Constructor for new tasks.

This is called once immediately after the task class is loaded by
the task manager (in tasks._load_task()).
"""
pass

def _setup_logger(self):
"""Set up a basic module-level logger."""
logger_name = ".".join(("earwigbot", "tasks", self.name))
self.logger = logging.getLogger(logger_name)
self.logger.setLevel(logging.DEBUG)

def run(self, **kwargs):
"""Main entry point to run a given task.

This is called directly by tasks.start() and is the main way to make a
task do stuff. kwargs will be any keyword arguments passed to start()
which are entirely optional.

The same task instance is preserved between runs, so you can
theoretically store data in self (e.g.
start('mytask', action='store', data='foo')) and then use it later
(e.g. start('mytask', action='save')).
"""
pass

def make_summary(self, comment):
"""Makes an edit summary by filling in variables in a config value.

config.wiki["summary"] is used, where $2 is replaced by the main
summary body, given as a method arg, and $1 is replaced by the task
number.

If the config value is not found, we just return the arg as-is.
"""
try:
summary = config.wiki["summary"]
except KeyError:
return comment
return summary.replace("$1", str(self.number)).replace("$2", comment)

def shutoff_enabled(self, site=None):
"""Returns whether on-wiki shutoff is enabled for this task.

We check a certain page for certain content. This is determined by
our config file: config.wiki["shutoff"]["page"] is used as the title,
with $1 replaced by our username and $2 replaced by the task number,
and config.wiki["shutoff"]["disabled"] is used as the content.

If the page has that content or the page does not exist, then shutoff
is "disabled", meaning the bot is supposed to run normally, and we
return False. If the page's content is something other than what we
expect, shutoff is enabled, and we return True.

If a site is not provided, we'll try to use self.site if it's set.
Otherwise, we'll use our default site.
"""
if not site:
try:
site = self.site
except AttributeError:
site = wiki.get_site()

try:
cfg = config.wiki["shutoff"]
except KeyError:
return False
title = cfg.get("page", "User:$1/Shutoff/Task $2")
username = site.get_user().name()
title = title.replace("$1", username).replace("$2", str(self.number))
page = site.get_page(title)

try:
content = page.get()
except wiki.PageNotFoundError:
return False
if content == cfg.get("disabled", "run"):
return False

self.logger.warn("Emergency task shutoff has been enabled!")
return True

+ 1
- 1
earwigbot/commands/__init__.py Просмотреть файл

@@ -26,7 +26,7 @@ EarwigBot's IRC Command Manager
This package provides the IRC "commands" used by the bot's front-end component.
This module contains the BaseCommand class (import with
`from earwigbot.commands import BaseCommand`) and an internal _CommandManager
class. This can be accessed through the singleton `command_manager`.
class. This can be accessed through the `command_manager` singleton.
"""

import logging


+ 2
- 2
earwigbot/commands/afc_report.py Просмотреть файл

@@ -22,9 +22,9 @@

import re

from earwigbot import tasks
from earwigbot import wiki
from earwigbot.commands import BaseCommand
from earwigbot.tasks import task_manager

class Command(BaseCommand):
"""Get information about an AFC submission by name."""
@@ -36,7 +36,7 @@ class Command(BaseCommand):
self.data = data

try:
self.statistics = tasks.get("afc_statistics")
self.statistics = task_manager.get("afc_statistics")
except KeyError:
e = "Cannot run command: requires afc_statistics task."
self.logger.error(e)


+ 5
- 5
earwigbot/commands/threads.py Просмотреть файл

@@ -23,10 +23,10 @@
import threading
import re

from earwigbot import tasks
from earwigbot.commands import BaseCommand
from earwigbot.config import config
from earwigbot.irc import KwargParseException
from earwigbot.tasks import task_manager

class Command(BaseCommand):
"""Manage wiki tasks from IRC, and check on thread status."""
@@ -106,7 +106,7 @@ class Command(BaseCommand):
def do_listall(self):
"""With !tasks listall or !tasks all, list all loaded tasks, and report
whether they are currently running or idle."""
all_tasks = tasks.get_all().keys()
all_tasks = task_manager.get_all().keys()
threads = threading.enumerate()
tasklist = []

@@ -147,14 +147,14 @@ class Command(BaseCommand):
self.connection.reply(data, msg)
return

if task_name not in tasks.get_all().keys():
if task_name not in task_manager.get_all().keys():
# This task does not exist or hasn't been loaded:
msg = "task could not be found; either bot/tasks/{0}.py doesn't exist, or it wasn't loaded correctly."
msg = "task could not be found; either tasks/{0}.py doesn't exist, or it wasn't loaded correctly."
self.connection.reply(data, msg.format(task_name))
return

data.kwargs["fromIRC"] = True
tasks.start(task_name, **data.kwargs)
task_manager.start(task_name, **data.kwargs)
msg = "task \x0302{0}\x0301 started.".format(task_name)
self.connection.reply(data, msg)



+ 5
- 8
earwigbot/main.py Просмотреть файл

@@ -49,9 +49,9 @@ import logging
import threading
import time

from earwigbot import tasks
from earwigbot.irc import Frontend, Watcher
from earwigbot.config import config
from earwigbot.irc import Frontend, Watcher
from earwigbot.tasks import task_manager

logger = logging.getLogger("earwigbot")

@@ -72,10 +72,7 @@ def wiki_scheduler():
primary thread if the IRC frontend is not enabled."""
while 1:
time_start = time.time()
now = time.gmtime(time_start)

tasks.schedule(now)

task_manager.schedule()
time_end = time.time()
time_diff = time_start - time_end
if time_diff < 60: # Sleep until the next minute
@@ -90,7 +87,7 @@ def irc_frontend():

if config.components.get("wiki_schedule"):
logger.info("Starting wiki scheduler")
tasks.load()
task_manager.load()
t_scheduler = threading.Thread(target=wiki_scheduler)
t_scheduler.name = "wiki-scheduler"
t_scheduler.daemon = True
@@ -119,7 +116,7 @@ def main():
# Run the scheduler on the main thread, but also run the IRC watcher on
# another thread iff it is enabled:
logger.info("Starting wiki scheduler")
tasks.load()
task_manager.load()
if "irc_watcher" in enabled:
logger.info("Starting IRC watcher")
t_watcher = threading.Thread(target=irc_watcher)


+ 3
- 3
earwigbot/rules.py Просмотреть файл

@@ -29,7 +29,7 @@ recieves an event from IRC.

import re

from earwigbot import tasks
from earwigbot.tasks import task_manager

afc_prefix = "wikipedia( talk)?:(wikiproject )?articles for creation"

@@ -56,7 +56,7 @@ def process(rc):
chans.update(("##earwigbot", "#wikipedia-en-afc-feed"))

if r_page.search(page_name):
#tasks.start("afc_copyvios", page=rc.page)
#task_manager.start("afc_copyvios", page=rc.page)
chans.add("#wikipedia-en-afc-feed")

elif r_ffu.match(page_name):
@@ -76,7 +76,7 @@ def process(rc):

elif rc.flags == "restore" and r_restore.match(comment):
p = r_restored_page.findall(rc.comment)[0]
#tasks.start("afc_copyvios", page=p)
#task_manager.start("afc_copyvios", page=p)
chans.add("#wikipedia-en-afc-feed")

elif rc.flags == "protect" and r_protect.match(comment):


+ 186
- 91
earwigbot/tasks/__init__.py Просмотреть файл

@@ -23,8 +23,10 @@
"""
EarwigBot's Wiki Task Manager

This package provides the wiki bot "tasks" EarwigBot runs. Here in __init__,
you can find some functions used to load and run these tasks.
This package provides the wiki bot "tasks" EarwigBot runs. This module contains
the BaseTask class (import with `from earwigbot.tasks import BaseTask`) and an
internal _TaskManager class. This can be accessed through the `task_manager`
singleton.
"""

import logging
@@ -33,106 +35,199 @@ import sys
import threading
import time

from earwigbot.classes import BaseTask
from earwigbot import wiki
from earwigbot.config import config

__all__ = ["load", "schedule", "start", "get", "get_all"]

# Base directory when searching for tasks:
base_dir = os.path.dirname(os.path.abspath(__file__))

# Store loaded tasks as a dict where the key is the task name and the value is
# an instance of the task class:
_tasks = {}

# Logger for this module:
logger = logging.getLogger("earwigbot.commands")

def _load_task(filename):
"""Try to load a specific task from a module, identified by file name."""
global _tasks

# Strip .py from the end of the filename and join with our package name:
name = ".".join(("tasks", filename[:-3]))
try:
__import__(name)
except:
logger.exception("Couldn't load file {0}:".format(filename))
return

task = sys.modules[name].Task()
task._setup_logger()
if not isinstance(task, BaseTask):
return

_tasks[task.name] = task
logger.debug("Added task {0}".format(task.name))

def _wrapper(task, **kwargs):
"""Wrapper for task classes: run the task and catch any errors."""
try:
task.run(**kwargs)
except:
error = "Task '{0}' raised an exception and had to stop"
logger.exception(error.format(task.name))
else:
logger.info("Task '{0}' finished without error".format(task.name))

def load():
"""Load all valid tasks from bot/tasks/, into the _tasks variable."""
files = os.listdir(base_dir)
files.sort()

for filename in files:
if filename.startswith("_") or not filename.endswith(".py"):
continue
__all__ = ["BaseTask", "task_manager"]

class BaseTask(object):
"""A base class for bot tasks that edit Wikipedia."""
name = None
number = 0

def __init__(self):
"""Constructor for new tasks.

This is called once immediately after the task class is loaded by
the task manager (in tasks._load_task()).
"""
pass

def _setup_logger(self):
"""Set up a basic module-level logger."""
logger_name = ".".join(("earwigbot", "tasks", self.name))
self.logger = logging.getLogger(logger_name)
self.logger.setLevel(logging.DEBUG)

def run(self, **kwargs):
"""Main entry point to run a given task.

This is called directly by tasks.start() and is the main way to make a
task do stuff. kwargs will be any keyword arguments passed to start()
which are entirely optional.

The same task instance is preserved between runs, so you can
theoretically store data in self (e.g.
start('mytask', action='store', data='foo')) and then use it later
(e.g. start('mytask', action='save')).
"""
pass

def make_summary(self, comment):
"""Makes an edit summary by filling in variables in a config value.

config.wiki["summary"] is used, where $2 is replaced by the main
summary body, given as a method arg, and $1 is replaced by the task
number.

If the config value is not found, we just return the arg as-is.
"""
try:
_load_task(filename)
except AttributeError:
pass # The file is doesn't contain a task, so just move on
summary = config.wiki["summary"]
except KeyError:
return comment
return summary.replace("$1", str(self.number)).replace("$2", comment)

def shutoff_enabled(self, site=None):
"""Returns whether on-wiki shutoff is enabled for this task.

We check a certain page for certain content. This is determined by
our config file: config.wiki["shutoff"]["page"] is used as the title,
with $1 replaced by our username and $2 replaced by the task number,
and config.wiki["shutoff"]["disabled"] is used as the content.

If the page has that content or the page does not exist, then shutoff
is "disabled", meaning the bot is supposed to run normally, and we
return False. If the page's content is something other than what we
expect, shutoff is enabled, and we return True.

If a site is not provided, we'll try to use self.site if it's set.
Otherwise, we'll use our default site.
"""
if not site:
try:
site = self.site
except AttributeError:
site = wiki.get_site()

try:
cfg = config.wiki["shutoff"]
except KeyError:
return False
title = cfg.get("page", "User:$1/Shutoff/Task $2")
username = site.get_user().name()
title = title.replace("$1", username).replace("$2", str(self.number))
page = site.get_page(title)

try:
content = page.get()
except wiki.PageNotFoundError:
return False
if content == cfg.get("disabled", "run"):
return False

self.logger.warn("Emergency task shutoff has been enabled!")
return True


class _TaskManager(object):
def __init__(self):
self.logger = logging.getLogger("earwigbot.commands")
self._base_dir = os.path.dirname(os.path.abspath(__file__))
self._tasks = {}

def _load_task(self, filename):
"""Load a specific task from a module, identified by file name."""
# Strip .py from the filename's end and join with our package name:
name = ".".join(("tasks", filename[:-3]))
try:
__import__(name)
except:
self.logger.exception("Couldn't load file {0}:".format(filename))
return

logger.info("Found {0} tasks: {1}".format(len(_tasks), ', '.join(_tasks.keys())))
try:
task = sys.modules[name].Task()
except AttributeError:
return # No task in this module
if not isinstance(task, BaseTask):
return
task._setup_logger()

def schedule(now=time.gmtime()):
"""Start all tasks that are supposed to be run at a given time."""
# Get list of tasks to run this turn:
tasks = config.schedule(now.tm_min, now.tm_hour, now.tm_mday, now.tm_mon,
now.tm_wday)
self._tasks[task.name] = task
self.logger.debug("Added task {0}".format(task.name))

for task in tasks:
if isinstance(task, list): # they've specified kwargs
start(task[0], **task[1]) # so pass those to start_task
else: # otherwise, just pass task_name
start(task)
def _wrapper(self, task, **kwargs):
"""Wrapper for task classes: run the task and catch any errors."""
try:
task.run(**kwargs)
except:
msg = "Task '{0}' raised an exception and had to stop"
self.logger.exception(msg.format(task.name))
else:
msg = "Task '{0}' finished without error"
self.logger.info(msg.format(task.name))

def load(self):
"""Load all valid tasks from tasks/ into self._tasks."""
files = os.listdir(self._base_dir)
files.sort()

for filename in files:
if filename.startswith("_") or not filename.endswith(".py"):
continue
self._load_task(filename)

msg = "Found {0} tasks: {1}"
tasks = ', '.join(self._tasks.keys())
self.logger.info(msg.format(len(self._tasks), tasks))

def schedule(self, now=None):
"""Start all tasks that are supposed to be run at a given time."""
if not now:
now = time.gmtime()
# Get list of tasks to run this turn:
tasks = config.schedule(now.tm_min, now.tm_hour, now.tm_mday,
now.tm_mon, now.tm_wday)

for task in tasks:
if isinstance(task, list): # They've specified kwargs,
self.start(task[0], **task[1]) # so pass those to start_task
else: # Otherwise, just pass task_name
self.start(task)

def start(self, task_name, **kwargs):
"""Start a given task in a new thread. Pass args to the task's run()
function."""
msg = "Starting task '{0}' in a new thread"
self.logger.info(msg.format(task_name))

def start(task_name, **kwargs):
"""Start a given task in a new thread. Pass args to the task's run()
function."""
logger.info("Starting task '{0}' in a new thread".format(task_name))
try:
task = self._tasks[task_name]
except KeyError:
e = "Couldn't find task '{0}': bot/tasks/{0}.py does not exist"
self.logger.error(e.format(task_name))
return

try:
task = _tasks[task_name]
except KeyError:
error = "Couldn't find task '{0}': bot/tasks/{0}.py does not exist"
logger.error(error.format(task_name))
return
func = lambda: self._wrapper(task, **kwargs)
task_thread = threading.Thread(target=func)
start_time = time.strftime("%b %d %H:%M:%S")
task_thread.name = "{0} ({1})".format(task_name, start_time)

task_thread = threading.Thread(target=lambda: _wrapper(task, **kwargs))
start_time = time.strftime("%b %d %H:%M:%S")
task_thread.name = "{0} ({1})".format(task_name, start_time)
# Stop bot task threads automagically if the main bot stops:
task_thread.daemon = True

# Stop bot task threads automagically if the main bot stops:
task_thread.daemon = True
task_thread.start()

task_thread.start()
def get(self, task_name):
"""Return the class instance associated with a certain task name.

def get(task_name):
"""Return the class instance associated with a certain task name.
Will raise KeyError if the task is not found.
"""
return self._tasks[task_name]

Will raise KeyError if the task is not found.
"""
return _tasks[task_name]
def get_all(self):
"""Return our dict of all loaded tasks."""
return self._tasks

def get_all():
"""Return our dict of all loaded tasks."""
return _tasks
task_manager = _TaskManager()

+ 1
- 1
earwigbot/tasks/afc_catdelink.py Просмотреть файл

@@ -20,7 +20,7 @@
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.

from earwigbot.classes import BaseTask
from earwigbot.tasks import BaseTask

class Task(BaseTask):
"""A task to delink mainspace categories in declined [[WP:AFC]]


+ 1
- 1
earwigbot/tasks/afc_copyvios.py Просмотреть файл

@@ -27,8 +27,8 @@ from threading import Lock
import oursql

from earwigbot import wiki
from earwigbot.classes import BaseTask
from earwigbot.config import config
from earwigbot.tasks import BaseTask

class Task(BaseTask):
"""A task to check newly-edited [[WP:AFC]] submissions for copyright


+ 1
- 1
earwigbot/tasks/afc_dailycats.py Просмотреть файл

@@ -20,7 +20,7 @@
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.

from earwigbot.classes import BaseTask
from earwigbot.tasks import BaseTask

class Task(BaseTask):
""" A task to create daily categories for [[WP:AFC]]."""


+ 1
- 1
earwigbot/tasks/afc_history.py Просмотреть файл

@@ -32,8 +32,8 @@ from numpy import arange
import oursql

from earwigbot import wiki
from earwigbot.classes import BaseTask
from earwigbot.config import config
from earwigbot.tasks import BaseTask

# Valid submission statuses:
STATUS_NONE = 0


+ 1
- 1
earwigbot/tasks/afc_statistics.py Просмотреть файл

@@ -30,8 +30,8 @@ from time import sleep
import oursql

from earwigbot import wiki
from earwigbot.classes import BaseTask
from earwigbot.config import config
from earwigbot.tasks import BaseTask

# Chart status number constants:
CHART_NONE = 0


+ 1
- 1
earwigbot/tasks/afc_undated.py Просмотреть файл

@@ -20,7 +20,7 @@
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.

from earwigbot.classes import BaseTask
from earwigbot.tasks import BaseTask

class Task(BaseTask):
"""A task to clear [[Category:Undated AfC submissions]]."""


+ 1
- 1
earwigbot/tasks/blptag.py Просмотреть файл

@@ -20,7 +20,7 @@
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.

from earwigbot.classes import BaseTask
from earwigbot.tasks import BaseTask

class Task(BaseTask):
"""A task to add |blp=yes to {{WPB}} or {{WPBS}} when it is used along with


+ 1
- 1
earwigbot/tasks/feed_dailycats.py Просмотреть файл

@@ -20,7 +20,7 @@
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.

from earwigbot.classes import BaseTask
from earwigbot.tasks import BaseTask

class Task(BaseTask):
"""A task to create daily categories for [[WP:FEED]]."""


+ 1
- 1
earwigbot/tasks/wrongmime.py Просмотреть файл

@@ -20,7 +20,7 @@
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.

from earwigbot.classes import BaseTask
from earwigbot.tasks import BaseTask

class Task(BaseTask):
"""A task to tag files whose extensions do not agree with their MIME


Загрузка…
Отмена
Сохранить