From 8c8b497d63f34938354162de78d9faaf1d8c2a2a Mon Sep 17 00:00:00 2001 From: Ben Kurtovic Date: Fri, 27 Jul 2012 21:33:29 -0400 Subject: [PATCH 01/22] Starting work --- earwigbot/tasks/drn_clerkbot.py | 54 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) create mode 100644 earwigbot/tasks/drn_clerkbot.py diff --git a/earwigbot/tasks/drn_clerkbot.py b/earwigbot/tasks/drn_clerkbot.py new file mode 100644 index 0000000..871016f --- /dev/null +++ b/earwigbot/tasks/drn_clerkbot.py @@ -0,0 +1,54 @@ +# -*- coding: utf-8 -*- +# +# Copyright (C) 2009-2012 Ben Kurtovic +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +import oursql + +from earwigbot.tasks import Task + +class DRNClerkBot(Task): + """A task to clerk for [[WP:DRN]].""" + name = "drn_clerkbot" + number = 19 + + def setup(self): + """Hook called immediately after the task is loaded.""" + cfg = self.config.tasks.get(self.name, {}) + self.title = cfg.get("page", "Wikipedia:Dispute resolution noticeboard") + + # Templates used in chart generation: + templates = cfg.get("templates", {}) + self.tl_status = templates.get("status", "DR case status") + self.tl_notify_party = templates.get("notifyParty", "DRN-notice") + self.tl_notify_stale = templates.get("notifyStale", "DRN stale notice") + self.tl_archive_top = templates.get("archiveTop", "DRN archive top") + self.tl_archive_bottom = templates.get("archiveBottom", + "DRN archive bottom") + + # Connection data for our SQL database: + kwargs = cfg.get("sql", {}) + kwargs["read_default_file"] = expanduser("~/.my.cnf") + self.conn_data = kwargs + self.db_access_lock = Lock() + + def run(self, **kwargs): + """Entry point for a task event.""" + page = self.bot.wiki.get_site().get_page(self.title) From e2dcc9d50b83d3f63c68975a7fbcb5f533dc92b1 Mon Sep 17 00:00:00 2001 From: Ben Kurtovic Date: Fri, 27 Jul 2012 23:33:25 -0400 Subject: [PATCH 02/22] Add status parsing, plus start schema for the SQL table. --- earwigbot/tasks/drn_clerkbot.py | 64 ++++++++++++++++++++++++++++++++- earwigbot/tasks/schema/drn_clerkbot.sql | 22 ++++++++++++ 2 files changed, 85 insertions(+), 1 deletion(-) create mode 100644 earwigbot/tasks/schema/drn_clerkbot.sql diff --git a/earwigbot/tasks/drn_clerkbot.py b/earwigbot/tasks/drn_clerkbot.py index 871016f..453fee7 100644 --- a/earwigbot/tasks/drn_clerkbot.py +++ b/earwigbot/tasks/drn_clerkbot.py @@ -20,6 +20,9 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. +from os import expanduser +import re + import oursql from earwigbot.tasks import Task @@ -29,6 +32,17 @@ class DRNClerkBot(Task): name = "drn_clerkbot" number = 19 + # Case status: + STATUS_UNKNOWN = 0 + STATUS_NEW = 1 + STATUS_OPEN = 2 + STATUS_STALE = 3 + STATUS_NEEDASSIST = 4 + STATUS_REVIEW = 5 + STATUS_RESOLVED = 6 + STATUS_CLOSED = 7 + STATUS_ARCHIVE = 8 + def setup(self): """Hook called immediately after the task is loaded.""" cfg = self.config.tasks.get(self.name, {}) @@ -51,4 +65,52 @@ class DRNClerkBot(Task): def run(self, **kwargs): """Entry point for a task event.""" - page = self.bot.wiki.get_site().get_page(self.title) + with self.db_access_lock: + page = self.bot.wiki.get_site().get_page(self.title) + text = page.get() + current = read_page(text) + + def read_page(self, text): + split = re.split("(^==\s*[^=]+?\s*==$)", text, flags=re.M|re.U) + cases = [] + case = None + for item in split: + if item.startswith("=="): + if case: + cases.append(case) + case = _Case() + case.title = item[2:-2].strip() + else: + templ = re.escape(self.tl_status) + if case and re.match("\s*\{\{" + templ, item, re.U): + case.body = case.old_body = item + case.status = self.read_status(body) + if case: + cases.append(case) + return cases + + def read_status(self, body): + aliases = { + self.STATUS_NEW: ("",), + self.STATUS_OPEN: ("open", "active", "inprogress"), + self.STATUS_STALE: ("stale",), + self.STATUS_NEEDASSIST: ("needassist", "relist", "relisted"), + self.STATUS_REVIEW: ("review",), + self.STATUS_RESOLVED: ("resolved", "resolve"), + self.STATUS_CLOSED: ("closed", "close"), + } + templ = re.escape(self.tl_status) + status = re.search("\{\{" + templ + "\|?(.*?)\}\}", body, re.S|re.U) + if not status: + return self.STATUS_UNKNOWN + for option, names in aliases.iteritems(): + if status.group(1).lower() in names: + return option + return self.STATUS_UNKNOWN + + +class _Case(object): + def __init__(self): + self.title = None + self.body = None + self.status = None diff --git a/earwigbot/tasks/schema/drn_clerkbot.sql b/earwigbot/tasks/schema/drn_clerkbot.sql new file mode 100644 index 0000000..ab6d06e --- /dev/null +++ b/earwigbot/tasks/schema/drn_clerkbot.sql @@ -0,0 +1,22 @@ +-- MySQL dump 10.13 Distrib 5.5.12, for solaris10 (i386) +-- +-- Host: sql Database: u_earwig_drn_clerkbot +-- ------------------------------------------------------ +-- Server version 5.1.59 + +CREATE DATABASE `u_earwig_drn_clerkbot` + DEFAULT CHARACTER SET utf8 + DEFAULT COLLATE utf8_unicode_ci; + +-- +-- Table structure for table `case` +-- + +DROP TABLE IF EXISTS `case`; +CREATE TABLE `case` ( + `case_id` int(10) unsigned NOT NULL, + `case_name` varchar(512) COLLATE utf8_unicode_ci DEFAULT NULL, + PRIMARY KEY (`case_id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci; + +-- Dump completed on 2012-07-27 00:00:00 From 93228d5b0848cb8df1cab0bbafb7d07bd2944490 Mon Sep 17 00:00:00 2001 From: Ben Kurtovic Date: Fri, 27 Jul 2012 23:45:39 -0400 Subject: [PATCH 03/22] Read cases from the database too. --- earwigbot/tasks/drn_clerkbot.py | 28 ++++++++++++++++++++++------ earwigbot/tasks/schema/drn_clerkbot.sql | 3 ++- 2 files changed, 24 insertions(+), 7 deletions(-) diff --git a/earwigbot/tasks/drn_clerkbot.py b/earwigbot/tasks/drn_clerkbot.py index 453fee7..6ae7229 100644 --- a/earwigbot/tasks/drn_clerkbot.py +++ b/earwigbot/tasks/drn_clerkbot.py @@ -66,11 +66,24 @@ class DRNClerkBot(Task): def run(self, **kwargs): """Entry point for a task event.""" with self.db_access_lock: + conn = oursql.connect(**self.conn_data) + cases = read_database(conn) page = self.bot.wiki.get_site().get_page(self.title) text = page.get() - current = read_page(text) + current = read_page(cases, text) - def read_page(self, text): + def read_database(self, conn): + """Return a list of _Cases from the database.""" + cases = [] + query = "SELECT case_id, case_title, case_status FROM case" + with conn.cursor() as cursor: + cursor.execute(query) + for id_, name, status in cursor: + cases.append(_Case(id_, title, status)) + return cases + + def read_page(self, cases, text): + """Read the noticeboard content and update the list of _Cases.""" split = re.split("(^==\s*[^=]+?\s*==$)", text, flags=re.M|re.U) cases = [] case = None @@ -87,9 +100,9 @@ class DRNClerkBot(Task): case.status = self.read_status(body) if case: cases.append(case) - return cases def read_status(self, body): + """Parse the current status from a case body.""" aliases = { self.STATUS_NEW: ("",), self.STATUS_OPEN: ("open", "active", "inprogress"), @@ -110,7 +123,10 @@ class DRNClerkBot(Task): class _Case(object): - def __init__(self): - self.title = None + """A simple object representing a dispute resolution case.""" + def __init__(self, id_, title, status): + self.id = id_ + self.title = title + self.status = status + self.body = None - self.status = None diff --git a/earwigbot/tasks/schema/drn_clerkbot.sql b/earwigbot/tasks/schema/drn_clerkbot.sql index ab6d06e..450aac9 100644 --- a/earwigbot/tasks/schema/drn_clerkbot.sql +++ b/earwigbot/tasks/schema/drn_clerkbot.sql @@ -15,7 +15,8 @@ CREATE DATABASE `u_earwig_drn_clerkbot` DROP TABLE IF EXISTS `case`; CREATE TABLE `case` ( `case_id` int(10) unsigned NOT NULL, - `case_name` varchar(512) COLLATE utf8_unicode_ci DEFAULT NULL, + `case_title` varchar(512) COLLATE utf8_unicode_ci DEFAULT NULL, + `case_status` int(2) unsigned NOT NULL, PRIMARY KEY (`case_id`) ) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci; From 015eb44a89297dd77b84624f02b487fa8a527bc7 Mon Sep 17 00:00:00 2001 From: Ben Kurtovic Date: Sat, 28 Jul 2012 00:40:07 -0400 Subject: [PATCH 04/22] A bunch of updates. --- earwigbot/tasks/drn_clerkbot.py | 94 ++++++++++++++++++++++++++++++++--------- 1 file changed, 75 insertions(+), 19 deletions(-) diff --git a/earwigbot/tasks/drn_clerkbot.py b/earwigbot/tasks/drn_clerkbot.py index 6ae7229..c9ebc43 100644 --- a/earwigbot/tasks/drn_clerkbot.py +++ b/earwigbot/tasks/drn_clerkbot.py @@ -22,6 +22,8 @@ from os import expanduser import re +from threading import RLock +from time import sleep, time import oursql @@ -46,9 +48,13 @@ class DRNClerkBot(Task): def setup(self): """Hook called immediately after the task is loaded.""" cfg = self.config.tasks.get(self.name, {}) + + # Set some wiki-related attributes: self.title = cfg.get("page", "Wikipedia:Dispute resolution noticeboard") + default_summary = "Updating $3 cases for the [[WP:DRN|dispute resolution noticeboard]]." + self.summary = self.make_summary(cfg.get("summary", default_summary)) - # Templates used in chart generation: + # Templates used: templates = cfg.get("templates", {}) self.tl_status = templates.get("status", "DR case status") self.tl_notify_party = templates.get("notifyParty", "DRN-notice") @@ -61,16 +67,47 @@ class DRNClerkBot(Task): kwargs = cfg.get("sql", {}) kwargs["read_default_file"] = expanduser("~/.my.cnf") self.conn_data = kwargs - self.db_access_lock = Lock() + self.db_access_lock = RLock() def run(self, **kwargs): """Entry point for a task event.""" + if not self.db_access_lock.acquire(False): # Non-blocking + self.logger.info("A job is already ongoing; aborting") + return + with self.db_access_lock: + self.logger.info(u"Starting update to [[{0}]]".format(self.title)) + start = time() conn = oursql.connect(**self.conn_data) cases = read_database(conn) page = self.bot.wiki.get_site().get_page(self.title) text = page.get() - current = read_page(cases, text) + read_page(conn, cases, text) + + # Work! + # Send messages! + + self.save(page, cases) + + def save(self, page, cases): + newtext = text = page.get() + counter = 0 + for case in cases: + if case.old != case.body: + newtext = newtext.replace(case.old, case.body) + counter += 1 + + worktime = time() - start + if worktime < 60: + sleep(60 - worktime) + page.reload() + if page.get() != text: + log = "Someone has edited the page while we were working; restarting" + self.logger.warn(log) + return self.run() + summary = self.summary.replace("$3", str(counter)) + page.edit(text, summary, minor=False, bot=True) + self.logger.info(u"Saved page [[{0}]]".format(page.title)) def read_database(self, conn): """Return a list of _Cases from the database.""" @@ -82,24 +119,42 @@ class DRNClerkBot(Task): cases.append(_Case(id_, title, status)) return cases - def read_page(self, cases, text): + def read_page(self, conn, cases, text): """Read the noticeboard content and update the list of _Cases.""" + tl_status_esc = re.escape(self.tl_status) split = re.split("(^==\s*[^=]+?\s*==$)", text, flags=re.M|re.U) - cases = [] - case = None - for item in split: - if item.startswith("=="): - if case: - cases.append(case) - case = _Case() - case.title = item[2:-2].strip() - else: - templ = re.escape(self.tl_status) - if case and re.match("\s*\{\{" + templ, item, re.U): - case.body = case.old_body = item - case.status = self.read_status(body) - if case: - cases.append(case) + for i in xrange(len(split)): + if i + 1 == len(split): + break + if not split[i].startswith("=="): + continue + title = split[i][2:-2].strip() + body = old = split[i + 1] + if not re.search("\s*\{\{" + tl_status_esc, body, re.U): + continue + status = self.read_status(body) + re_id = "" + try: + id_ = re.search(re_id, body).group(1) + case = [case for case in cases if case.id == id_][0] + except (AttributeError, IndexError): + id_ = self.select_next_id(conn) + re_id2 = "(\{\{" + tl_status_esc + "(.*?)\}\})()?" + repl = ur"\1 " + body = re.sub(re_id2, repl.format(id_), body) + case = _Case(id_, title, status) + cases.append(case) + case.body, case.old = body, old + + def select_next_id(self, conn): + """Return the next incremental ID for a case.""" + query = "SELECT MAX(case_id) FROM case" + with conn.cursor() as cursor: + cursor.execute(query) + current = cursor.fetchone()[0] + if current: + return current + 1 + return 1 def read_status(self, body): """Parse the current status from a case body.""" @@ -130,3 +185,4 @@ class _Case(object): self.status = status self.body = None + self.old = None From c589d16ede2afebd224152bc336e8fec656e2b03 Mon Sep 17 00:00:00 2001 From: Ben Kurtovic Date: Sat, 28 Jul 2012 01:10:11 -0400 Subject: [PATCH 05/22] Support for sending notices. --- earwigbot/tasks/drn_clerkbot.py | 65 ++++++++++++++++++++++++++++++++++------- 1 file changed, 55 insertions(+), 10 deletions(-) diff --git a/earwigbot/tasks/drn_clerkbot.py b/earwigbot/tasks/drn_clerkbot.py index c9ebc43..ae457e1 100644 --- a/earwigbot/tasks/drn_clerkbot.py +++ b/earwigbot/tasks/drn_clerkbot.py @@ -27,6 +27,7 @@ from time import sleep, time import oursql +from earwigbot import exceptions from earwigbot.tasks import Task class DRNClerkBot(Task): @@ -50,7 +51,8 @@ class DRNClerkBot(Task): cfg = self.config.tasks.get(self.name, {}) # Set some wiki-related attributes: - self.title = cfg.get("page", "Wikipedia:Dispute resolution noticeboard") + self.title = cfg.get("title", "Wikipedia:Dispute resolution noticeboard") + self.talk = cfg.get("talk", "Wikipedia talk:Dispute resolution noticeboard") default_summary = "Updating $3 cases for the [[WP:DRN|dispute resolution noticeboard]]." self.summary = self.make_summary(cfg.get("summary", default_summary)) @@ -79,23 +81,26 @@ class DRNClerkBot(Task): self.logger.info(u"Starting update to [[{0}]]".format(self.title)) start = time() conn = oursql.connect(**self.conn_data) - cases = read_database(conn) - page = self.bot.wiki.get_site().get_page(self.title) + cases = self.read_database(conn) + site = self.bot.wiki.get_site() + page = site.get_page(self.title) text = page.get() - read_page(conn, cases, text) - - # Work! - # Send messages! - + self.read_page(conn, cases, text) + noticies = self.clerk(cases) self.save(page, cases) + self.send_notices(site, notices) def save(self, page, cases): + """Save any changes to the noticeboard.""" newtext = text = page.get() counter = 0 for case in cases: if case.old != case.body: newtext = newtext.replace(case.old, case.body) counter += 1 + if newtext == text: + self.logger.info(u"Nothing to edit on [[{0}]]".format(page.title)) + return worktime = time() - start if worktime < 60: @@ -106,9 +111,36 @@ class DRNClerkBot(Task): self.logger.warn(log) return self.run() summary = self.summary.replace("$3", str(counter)) - page.edit(text, summary, minor=False, bot=True) + page.edit(text, summary, minor=True, bot=True) self.logger.info(u"Saved page [[{0}]]".format(page.title)) + def send_notices(self, site, notices): + """Send out any templated notices to users or pages.""" + if not notices: + self.logger.info("No notices to send; finishing") + return + for notice in notices: + target, template = notice.target, notice.template + log = u"Notifying [[{0}]] with {1}".format(target, template) + self.logger.info(log) + page = site.get_page(target) + try: + text = page.get() + except exceptions.PageNotFoundError: + text = "" + if notice.too_late in text: + log = u"Skipping [[{0}]]; was already notified".format(target) + self.logger.info(log) + text += ("\n" if text else "") + template + try: + page.edit(text, summary, minor=False, bot=True) + except exceptions.EditError as error: + name, msg = type(error).name, error.message + log = u"Couldn't leave notice on {0} because of {1}: {2}" + self.logger.error(log.format(page.title, name, msg)) + + self.logger.info("Done sending notices") + def read_database(self, conn): """Return a list of _Cases from the database.""" cases = [] @@ -176,9 +208,14 @@ class DRNClerkBot(Task): return option return self.STATUS_UNKNOWN + def clerk(self): + """Actually go through cases and modify those to be updated.""" + notices = [] + return notices + class _Case(object): - """A simple object representing a dispute resolution case.""" + """A object representing a dispute resolution case.""" def __init__(self, id_, title, status): self.id = id_ self.title = title @@ -186,3 +223,11 @@ class _Case(object): self.body = None self.old = None + + +class _Notice(object): + """An object representing a notice to be sent to a user or a page.""" + def __init__(self, target, template, too_late): + self.target = target + self.template = template + self.too_late = too_late From ef4c7a187a6eaec150e7c9f4ab73e0fa812df3a3 Mon Sep 17 00:00:00 2001 From: Ben Kurtovic Date: Sat, 28 Jul 2012 01:14:42 -0400 Subject: [PATCH 06/22] update_case_title() --- earwigbot/tasks/drn_clerkbot.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/earwigbot/tasks/drn_clerkbot.py b/earwigbot/tasks/drn_clerkbot.py index ae457e1..954aa6a 100644 --- a/earwigbot/tasks/drn_clerkbot.py +++ b/earwigbot/tasks/drn_clerkbot.py @@ -176,6 +176,10 @@ class DRNClerkBot(Task): body = re.sub(re_id2, repl.format(id_), body) case = _Case(id_, title, status) cases.append(case) + else: + if case.title != title: + self.update_case_title(conn, id_, title) + case.title = title case.body, case.old = body, old def select_next_id(self, conn): @@ -208,6 +212,12 @@ class DRNClerkBot(Task): return option return self.STATUS_UNKNOWN + def update_case_title(self, conn, id_, title): + """Update a case title in the database.""" + query = "UPDATE case SET case_title = ? WHERE case_id = ?" + with conn.cursor() as cursor: + cursor.execute(query, (title, id_)) + def clerk(self): """Actually go through cases and modify those to be updated.""" notices = [] From d2b82b60d99453f2cb1c750f005704e7d14add20 Mon Sep 17 00:00:00 2001 From: Ben Kurtovic Date: Sat, 28 Jul 2012 01:59:40 -0400 Subject: [PATCH 07/22] Implementing more code for dealing with the volunteer list. --- earwigbot/tasks/drn_clerkbot.py | 164 ++++++++++++++++++++------------ earwigbot/tasks/schema/drn_clerkbot.sql | 10 ++ 2 files changed, 113 insertions(+), 61 deletions(-) diff --git a/earwigbot/tasks/drn_clerkbot.py b/earwigbot/tasks/drn_clerkbot.py index 954aa6a..5d0ecbb 100644 --- a/earwigbot/tasks/drn_clerkbot.py +++ b/earwigbot/tasks/drn_clerkbot.py @@ -53,6 +53,7 @@ class DRNClerkBot(Task): # Set some wiki-related attributes: self.title = cfg.get("title", "Wikipedia:Dispute resolution noticeboard") self.talk = cfg.get("talk", "Wikipedia talk:Dispute resolution noticeboard") + self.volunteer_title = cfg.get("volunteers", "Wikipedia:Dispute resolution noticeboard/Volunteering") default_summary = "Updating $3 cases for the [[WP:DRN|dispute resolution noticeboard]]." self.summary = self.make_summary(cfg.get("summary", default_summary)) @@ -76,70 +77,47 @@ class DRNClerkBot(Task): if not self.db_access_lock.acquire(False): # Non-blocking self.logger.info("A job is already ongoing; aborting") return - - with self.db_access_lock: - self.logger.info(u"Starting update to [[{0}]]".format(self.title)) + action = kwargs.get("action", "all") + try: start = time() conn = oursql.connect(**self.conn_data) - cases = self.read_database(conn) site = self.bot.wiki.get_site() - page = site.get_page(self.title) - text = page.get() - self.read_page(conn, cases, text) - noticies = self.clerk(cases) - self.save(page, cases) - self.send_notices(site, notices) - - def save(self, page, cases): - """Save any changes to the noticeboard.""" - newtext = text = page.get() - counter = 0 - for case in cases: - if case.old != case.body: - newtext = newtext.replace(case.old, case.body) - counter += 1 - if newtext == text: - self.logger.info(u"Nothing to edit on [[{0}]]".format(page.title)) - return - - worktime = time() - start - if worktime < 60: - sleep(60 - worktime) - page.reload() - if page.get() != text: - log = "Someone has edited the page while we were working; restarting" - self.logger.warn(log) - return self.run() - summary = self.summary.replace("$3", str(counter)) - page.edit(text, summary, minor=True, bot=True) - self.logger.info(u"Saved page [[{0}]]".format(page.title)) - - def send_notices(self, site, notices): - """Send out any templated notices to users or pages.""" - if not notices: - self.logger.info("No notices to send; finishing") - return - for notice in notices: - target, template = notice.target, notice.template - log = u"Notifying [[{0}]] with {1}".format(target, template) - self.logger.info(log) - page = site.get_page(target) - try: - text = page.get() - except exceptions.PageNotFoundError: - text = "" - if notice.too_late in text: - log = u"Skipping [[{0}]]; was already notified".format(target) + if action in ["all", "update_volunteers"]: + self.update_volunteers(conn, site) + if action in ["all", "clerk"]: + log = u"Starting update to [[{0}]]".format(self.title) self.logger.info(log) - text += ("\n" if text else "") + template - try: - page.edit(text, summary, minor=False, bot=True) - except exceptions.EditError as error: - name, msg = type(error).name, error.message - log = u"Couldn't leave notice on {0} because of {1}: {2}" - self.logger.error(log.format(page.title, name, msg)) + cases = self.read_database(conn) + page = site.get_page(self.title) + text = page.get() + self.read_page(conn, cases, text) + noticies = self.clerk(conn, cases) + self.save(page, cases, kwargs) + self.send_notices(site, notices) + finally: + self.db_access_lock.release() - self.logger.info("Done sending notices") + def update_volunteers(self, conn, site): + """Updates and stores the list of dispute resolution volunteers.""" + log = u"Updating volunteer list from [[{0}]]" + self.logger.info(log.format(self.volunteer_title)) + page = site.get_page(self.volunteer_title) + try: + text = page.get() + except exceptions.PageNotFoundError: + text = "" + marker = "" + if marker not in text: + log = u"The marker ({0}) wasn't found in the volunteer list at [[{1}]]!" + self.logger.error(log.format(marker, page.title)) + text = text.split(marker)[1] + users = set() + for line in text.splitlines(): + user = re.search("\# \{\{User\|(.*?)\}\}", line) + if user: + users.add(user.group(1)) + + # SYNCHRONIZE USERS WITH DATABASE def read_database(self, conn): """Return a list of _Cases from the database.""" @@ -148,7 +126,8 @@ class DRNClerkBot(Task): with conn.cursor() as cursor: cursor.execute(query) for id_, name, status in cursor: - cases.append(_Case(id_, title, status)) + case = _Case(id_, title, status) + cases.append(case) return cases def read_page(self, conn, cases, text): @@ -181,6 +160,7 @@ class DRNClerkBot(Task): self.update_case_title(conn, id_, title) case.title = title case.body, case.old = body, old + case.apparent_status = status def select_next_id(self, conn): """Return the next incremental ID for a case.""" @@ -218,11 +198,72 @@ class DRNClerkBot(Task): with conn.cursor() as cursor: cursor.execute(query, (title, id_)) - def clerk(self): + def clerk(self, conn, cases): """Actually go through cases and modify those to be updated.""" + query = "SELECT volunteer_username FROM volunteer" + with conn.cursor() as cursor: + cursor.execute(query) + volunteers = [name for (name,) in cursor.fetchall()] notices = [] + for case in cases: + notices += self.clerk_case(conn, case, volunteers) return notices + def clerk_case(self, conn, case, volunteers): + """Clerk a particular case and return a list of any notices to send.""" + case # TODO + + def save(self, page, cases, kwargs): + """Save any changes to the noticeboard.""" + newtext = text = page.get() + counter = 0 + for case in cases: + if case.old != case.body: + newtext = newtext.replace(case.old, case.body) + counter += 1 + if newtext == text: + self.logger.info(u"Nothing to edit on [[{0}]]".format(page.title)) + return + + worktime = time() - start + if worktime < 60: + sleep(60 - worktime) + page.reload() + if page.get() != text: + log = "Someone has edited the page while we were working; restarting" + self.logger.warn(log) + return self.run(**kwargs) + summary = self.summary.replace("$3", str(counter)) + page.edit(text, summary, minor=True, bot=True) + self.logger.info(u"Saved page [[{0}]]".format(page.title)) + + def send_notices(self, site, notices): + """Send out any templated notices to users or pages.""" + if not notices: + self.logger.info("No notices to send; finishing") + return + for notice in notices: + target, template = notice.target, notice.template + log = u"Notifying [[{0}]] with {1}".format(target, template) + self.logger.info(log) + page = site.get_page(target) + try: + text = page.get() + except exceptions.PageNotFoundError: + text = "" + if notice.too_late in text: + log = u"Skipping [[{0}]]; was already notified".format(target) + self.logger.info(log) + text += ("\n" if text else "") + template + try: + page.edit(text, summary, minor=False, bot=True) + except exceptions.EditError as error: + name, msg = type(error).name, error.message + log = u"Couldn't leave notice on {0} because of {1}: {2}" + self.logger.error(log.format(page.title, name, msg)) + + self.logger.info("Done sending notices") + class _Case(object): """A object representing a dispute resolution case.""" @@ -233,6 +274,7 @@ class _Case(object): self.body = None self.old = None + self.apparent_status = None class _Notice(object): diff --git a/earwigbot/tasks/schema/drn_clerkbot.sql b/earwigbot/tasks/schema/drn_clerkbot.sql index 450aac9..bb6b227 100644 --- a/earwigbot/tasks/schema/drn_clerkbot.sql +++ b/earwigbot/tasks/schema/drn_clerkbot.sql @@ -20,4 +20,14 @@ CREATE TABLE `case` ( PRIMARY KEY (`case_id`) ) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci; +-- +-- Table structure for table `volunteer` +-- + +DROP TABLE IF EXISTS `volunteer`; +CREATE TABLE `volunteer` ( + `volunteer_username` varchar(512) COLLATE utf8_unicode_ci DEFAULT NULL, + PRIMARY KEY (`volunteer_username`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci; + -- Dump completed on 2012-07-27 00:00:00 From 2703d0eed96bfe0c8bd93d7e522717db70f98023 Mon Sep 17 00:00:00 2001 From: Ben Kurtovic Date: Sat, 28 Jul 2012 02:22:58 -0400 Subject: [PATCH 08/22] Implement database synchronization with volunteer list. --- earwigbot/tasks/drn_clerkbot.py | 27 +++++++++++++++++++++------ 1 file changed, 21 insertions(+), 6 deletions(-) diff --git a/earwigbot/tasks/drn_clerkbot.py b/earwigbot/tasks/drn_clerkbot.py index 5d0ecbb..2af7674 100644 --- a/earwigbot/tasks/drn_clerkbot.py +++ b/earwigbot/tasks/drn_clerkbot.py @@ -111,13 +111,27 @@ class DRNClerkBot(Task): log = u"The marker ({0}) wasn't found in the volunteer list at [[{1}]]!" self.logger.error(log.format(marker, page.title)) text = text.split(marker)[1] - users = set() + additions = set() for line in text.splitlines(): user = re.search("\# \{\{User\|(.*?)\}\}", line) if user: - users.add(user.group(1)) + additions.add((user.group(1))) - # SYNCHRONIZE USERS WITH DATABASE + removals = set() + query1 = "SELECT volunteer_username FROM volunteer" + query2 = "DELETE FROM volunteer WHERE volunteer_username = ?" + query3 = "INSERT INTO volunteer VALUES (?)" + with conn.cursor() as cursor: + cursor.execute(query1) + for row in cursor: + if row in additions: + additions.remove(row) + else: + removals.add(row) + if removals: + cursor.executemany(query2, removals) + if additions: + cursor.executemany(query3, additions) def read_database(self, conn): """Return a list of _Cases from the database.""" @@ -144,14 +158,15 @@ class DRNClerkBot(Task): if not re.search("\s*\{\{" + tl_status_esc, body, re.U): continue status = self.read_status(body) - re_id = "" + re_id = "" try: id_ = re.search(re_id, body).group(1) case = [case for case in cases if case.id == id_][0] except (AttributeError, IndexError): id_ = self.select_next_id(conn) - re_id2 = "(\{\{" + tl_status_esc + "(.*?)\}\})()?" - repl = ur"\1 " + re_id2 = "(\{\{" + tl_status_esc + re_id2 += "(.*?)\}\})()?" + repl = ur"\1 " body = re.sub(re_id2, repl.format(id_), body) case = _Case(id_, title, status) cases.append(case) From ca6f3868021bf708195060afba357ff3cade5a59 Mon Sep 17 00:00:00 2001 From: Ben Kurtovic Date: Sat, 28 Jul 2012 02:45:52 -0400 Subject: [PATCH 09/22] TODO list for clerk_case(). --- earwigbot/tasks/drn_clerkbot.py | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/earwigbot/tasks/drn_clerkbot.py b/earwigbot/tasks/drn_clerkbot.py index 2af7674..d101158 100644 --- a/earwigbot/tasks/drn_clerkbot.py +++ b/earwigbot/tasks/drn_clerkbot.py @@ -226,7 +226,27 @@ class DRNClerkBot(Task): def clerk_case(self, conn, case, volunteers): """Clerk a particular case and return a list of any notices to send.""" - case # TODO + if case.apparent_status == self.STATUS_NEW: + # NOTIFY ALL PARTIES + # SET STATUS TO OPEN IF VOLUNTEER EDITS + elif case.apparent_status == self.STATUS_OPEN: + # OPEN FOR OVER FOUR DAYS: SET STATUS TO REVIEW + # SEND MESSAGE TO WT:DRN + # ELSE OVER 10K TEXT SINCE LAST VOLUNTEER EDIT: SET STATUS TO NEEDASSIST + # SEND MESSAGE TO WT:DRN + # ELSE NO EDITS IN ONE DAY: SET STATUS TO STALE + # SEND MESSAGE TO WT:DRN + elif case.apparent_status == self.STATUS_NEEDASSIST: + # OPEN FOR OVER FOUR DAYS: SET STATUS TO REVIEW + # ELSE VOLUNTEER EDIT SINCE STATUS SET: SET STATUS OPEN + elif case.apparent_status == self.STATUS_STALE: + # OPEN FOR OVER FOUR DAYS: SET STATUS TO REVIEW + # ELSE EDITS SINCE STATUS SET: SET STATUS OPEN + elif case.apparent_status == self.STATUS_REVIEW: + # OPEN FOR OVER SEVEN DAYS: SEND MESSAGE TO ZHANG + elif case.apparent_status in [self.STATUS_RESOLVED, self.STATUS_CLOSED]: + # ONE DAY SINCE STATUS SET: SET STATUS ARCHIVED + # ADD ARCHIVE TEMPLATE, REMOVE NOARCHIVE def save(self, page, cases, kwargs): """Save any changes to the noticeboard.""" From 5d99c7e1a77a724c9e9010188ce2dcbb5cd2e315 Mon Sep 17 00:00:00 2001 From: Ben Kurtovic Date: Sat, 28 Jul 2012 03:58:19 -0400 Subject: [PATCH 10/22] More of a framework for the clerking function. --- earwigbot/tasks/drn_clerkbot.py | 131 +++++++++++++++++++++++++------- earwigbot/tasks/schema/drn_clerkbot.sql | 6 +- 2 files changed, 107 insertions(+), 30 deletions(-) diff --git a/earwigbot/tasks/drn_clerkbot.py b/earwigbot/tasks/drn_clerkbot.py index d101158..32541de 100644 --- a/earwigbot/tasks/drn_clerkbot.py +++ b/earwigbot/tasks/drn_clerkbot.py @@ -136,11 +136,10 @@ class DRNClerkBot(Task): def read_database(self, conn): """Return a list of _Cases from the database.""" cases = [] - query = "SELECT case_id, case_title, case_status FROM case" with conn.cursor() as cursor: - cursor.execute(query) - for id_, name, status in cursor: - case = _Case(id_, title, status) + cursor.execute("SELECT * FROM case") + for row in cursor: + case = _Case(*row) cases.append(case) return cases @@ -168,14 +167,14 @@ class DRNClerkBot(Task): re_id2 += "(.*?)\}\})()?" repl = ur"\1 " body = re.sub(re_id2, repl.format(id_), body) - case = _Case(id_, title, status) + case = _Case(id_, title, status, time()) cases.append(case) else: + case.status = status if case.title != title: self.update_case_title(conn, id_, title) case.title = title case.body, case.old = body, old - case.apparent_status = status def select_next_id(self, conn): """Return the next incremental ID for a case.""" @@ -226,27 +225,96 @@ class DRNClerkBot(Task): def clerk_case(self, conn, case, volunteers): """Clerk a particular case and return a list of any notices to send.""" - if case.apparent_status == self.STATUS_NEW: - # NOTIFY ALL PARTIES - # SET STATUS TO OPEN IF VOLUNTEER EDITS - elif case.apparent_status == self.STATUS_OPEN: - # OPEN FOR OVER FOUR DAYS: SET STATUS TO REVIEW - # SEND MESSAGE TO WT:DRN - # ELSE OVER 10K TEXT SINCE LAST VOLUNTEER EDIT: SET STATUS TO NEEDASSIST - # SEND MESSAGE TO WT:DRN - # ELSE NO EDITS IN ONE DAY: SET STATUS TO STALE - # SEND MESSAGE TO WT:DRN - elif case.apparent_status == self.STATUS_NEEDASSIST: - # OPEN FOR OVER FOUR DAYS: SET STATUS TO REVIEW - # ELSE VOLUNTEER EDIT SINCE STATUS SET: SET STATUS OPEN - elif case.apparent_status == self.STATUS_STALE: - # OPEN FOR OVER FOUR DAYS: SET STATUS TO REVIEW - # ELSE EDITS SINCE STATUS SET: SET STATUS OPEN - elif case.apparent_status == self.STATUS_REVIEW: - # OPEN FOR OVER SEVEN DAYS: SEND MESSAGE TO ZHANG - elif case.apparent_status in [self.STATUS_RESOLVED, self.STATUS_CLOSED]: - # ONE DAY SINCE STATUS SET: SET STATUS ARCHIVED - # ADD ARCHIVE TEMPLATE, REMOVE NOARCHIVE + notices = [] + if case.status == self.STATUS_NEW: + notices = self.clerk_new_case(conn, case, volunteers) + elif case.status == self.STATUS_OPEN: + notices = self.clerk_open_case(conn, case, volunteers) + elif case.status == self.STATUS_NEEDASSIST: + notices = self.clerk_needassist_case(conn, case, volunteers) + elif case.status == self.STATUS_STALE: + notices = self.clerk_stale_case(conn, case, volunteers) + elif case.status == self.STATUS_REVIEW: + notices = self.clerk_review_case(conn, case, volunteers) + elif case.status in [self.STATUS_RESOLVED, self.STATUS_CLOSED]: + self.clerk_closed_case(conn) + else: + LOG NOT SURE HOW TO DEAL WITH CASE + return notices + + # STORE UPDATES IN DATABASE + # APPLY STATUS UPDATES TO CASE BODY + return notices + + def clerk_new_case(self, conn, case, volunteers): + notices = self.notify_parties(conn, case) + signatures = self.read_signatures(case.body) + if any([editor in volunteers for (editor, time) in signatures]): + if case.last_action != self.STATUS_OPEN: + case.status = self.STATUS_OPEN + return notices + + def clerk_open_case(self, conn, case, volunteers): + if time() - case.file_time > 60 * 60 * 24 * 4: + if case.last_action != self.STATUS_REVIEW: + case.status = self.STATUS_REVIEW + return SEND_MESSAGE_TO_WT:DRN + + if len(case.body) - SIZE_WHEN_LAST_VOLUNTEER_EDIT > 15000: + if case.last_action != self.STATUS_NEEDASSIST: + case.status = self.STATUS_NEEDASSIST + return SEND_MESSAGE_TO_WT:DRN + + if time() - LAST_EDIT > 60 * 60 * 24 * 2: + if case.last_action != self.STATUS_STALE: + case.status = self.STATUS_STALE + return SEND_MESSAGE_TO_WT:DRN + return [] + + def clerk_needassist_case(self, conn, case, volunteers): + if time() - case.file_time > 60 * 60 * 24 * 4: + if case.last_action != self.STATUS_REVIEW: + case.status = self.STATUS_REVIEW + return SEND_MESSAGE_TO_WT:DRN + + signatures = self.read_signatures(case.body) + newsigs = signatures - SIGNATURES_FROM_DATABASE + if any([editor in volunteers for (editor, time) in newsigs]): + if case.last_action != self.STATUS_OPEN: + case.status = self.STATUS_OPEN + return [] + + def clerk_stale_case(self, conn, case, volunteers): + if time() - case.file_time > 60 * 60 * 24 * 4: + if case.last_action != self.STATUS_REVIEW: + case.status = self.STATUS_REVIEW + return SEND_MESSAGE_TO_WT:DRN + + signatures = self.read_signatures(case.body) + if signatures - SIGNATURES_FROM_DATABASE: + if case.last_action != self.STATUS_OPEN: + case.status = self.STATUS_OPEN + return [] + + def clerk_review_case(self, conn, case, volunteers): + if time() - case.file_time > 60 * 60 * 24 * 7: + if not case.very_old_notified: + return SEND_MESSAGE_TO_ZHANG + return [] + + def clerk_closed_case(self, case): + if time() - TIME_STATUS_SET AND LAST_EDIT > 60 * 60 * 24: + case.status = self.STATUS_ARCHIVE + ADD_ARCHIVE_TEMPLATE + REMOVE_NOARCHIVE + + def read_signatures(self, text): + raise NotImplementedError() + + def notify_parties(self, conn, case): + if case.parties_notified: + return + raise NotImplementedError() def save(self, page, cases, kwargs): """Save any changes to the noticeboard.""" @@ -302,14 +370,19 @@ class DRNClerkBot(Task): class _Case(object): """A object representing a dispute resolution case.""" - def __init__(self, id_, title, status): + def __init__(self, id_, title, status, last_action, file_time, + parties_notified, very_old_notified): self.id = id_ self.title = title self.status = status + self.last_action = last_action + self.file_time = file_time + self.parties_notified = parties_notified + self.very_old_notified = very_old_notified + self.original_status = status self.body = None self.old = None - self.apparent_status = None class _Notice(object): diff --git a/earwigbot/tasks/schema/drn_clerkbot.sql b/earwigbot/tasks/schema/drn_clerkbot.sql index bb6b227..81739dc 100644 --- a/earwigbot/tasks/schema/drn_clerkbot.sql +++ b/earwigbot/tasks/schema/drn_clerkbot.sql @@ -16,7 +16,11 @@ DROP TABLE IF EXISTS `case`; CREATE TABLE `case` ( `case_id` int(10) unsigned NOT NULL, `case_title` varchar(512) COLLATE utf8_unicode_ci DEFAULT NULL, - `case_status` int(2) unsigned NOT NULL, + `case_status` int(2) unsigned DEFAULT NULL, + `case_last_action` int(2) unsigned DEFAULT NULL, + `case_file_time` int(10) unsigned DEFAULT NULL, + `case_parties_notified` tinyint(1) unsigned DEFAULT NULL, + `case_very_old_notified` tinyint(1) unsigned DEFAULT NULL, PRIMARY KEY (`case_id`) ) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci; From de5fc95c838f5d77ea8ba80c2565fc8990d93795 Mon Sep 17 00:00:00 2001 From: Ben Kurtovic Date: Sat, 28 Jul 2012 04:04:35 -0400 Subject: [PATCH 11/22] Cleanup function signatures a bit; some more work. --- earwigbot/tasks/drn_clerkbot.py | 52 ++++++++++++++++++++++------------------- 1 file changed, 28 insertions(+), 24 deletions(-) diff --git a/earwigbot/tasks/drn_clerkbot.py b/earwigbot/tasks/drn_clerkbot.py index 32541de..2e74e0a 100644 --- a/earwigbot/tasks/drn_clerkbot.py +++ b/earwigbot/tasks/drn_clerkbot.py @@ -227,15 +227,15 @@ class DRNClerkBot(Task): """Clerk a particular case and return a list of any notices to send.""" notices = [] if case.status == self.STATUS_NEW: - notices = self.clerk_new_case(conn, case, volunteers) + notices = self.clerk_new_case(case, volunteers) elif case.status == self.STATUS_OPEN: - notices = self.clerk_open_case(conn, case, volunteers) + notices = self.clerk_open_case(case) elif case.status == self.STATUS_NEEDASSIST: - notices = self.clerk_needassist_case(conn, case, volunteers) + notices = self.clerk_needassist_case(case, volunteers) elif case.status == self.STATUS_STALE: - notices = self.clerk_stale_case(conn, case, volunteers) + notices = self.clerk_stale_case(case) elif case.status == self.STATUS_REVIEW: - notices = self.clerk_review_case(conn, case, volunteers) + notices = self.clerk_review_case(case) elif case.status in [self.STATUS_RESOLVED, self.STATUS_CLOSED]: self.clerk_closed_case(conn) else: @@ -246,19 +246,24 @@ class DRNClerkBot(Task): # APPLY STATUS UPDATES TO CASE BODY return notices - def clerk_new_case(self, conn, case, volunteers): - notices = self.notify_parties(conn, case) + def check_for_review(self, case): + if time() - case.file_time > 60 * 60 * 24 * 4: + if case.last_action != self.STATUS_REVIEW: + case.status = self.STATUS_REVIEW + return SEND_MESSAGE_TO_WT:DRN + + def clerk_new_case(self, case, volunteers): + notices = self.notify_parties(case) signatures = self.read_signatures(case.body) if any([editor in volunteers for (editor, time) in signatures]): if case.last_action != self.STATUS_OPEN: case.status = self.STATUS_OPEN return notices - def clerk_open_case(self, conn, case, volunteers): - if time() - case.file_time > 60 * 60 * 24 * 4: - if case.last_action != self.STATUS_REVIEW: - case.status = self.STATUS_REVIEW - return SEND_MESSAGE_TO_WT:DRN + def clerk_open_case(self, case): + flagged = self.check_for_review(case): + if flagged: + return flagged if len(case.body) - SIZE_WHEN_LAST_VOLUNTEER_EDIT > 15000: if case.last_action != self.STATUS_NEEDASSIST: @@ -271,11 +276,10 @@ class DRNClerkBot(Task): return SEND_MESSAGE_TO_WT:DRN return [] - def clerk_needassist_case(self, conn, case, volunteers): - if time() - case.file_time > 60 * 60 * 24 * 4: - if case.last_action != self.STATUS_REVIEW: - case.status = self.STATUS_REVIEW - return SEND_MESSAGE_TO_WT:DRN + def clerk_needassist_case(self, case, volunteers): + flagged = self.check_for_review(case): + if flagged: + return flagged signatures = self.read_signatures(case.body) newsigs = signatures - SIGNATURES_FROM_DATABASE @@ -284,11 +288,10 @@ class DRNClerkBot(Task): case.status = self.STATUS_OPEN return [] - def clerk_stale_case(self, conn, case, volunteers): - if time() - case.file_time > 60 * 60 * 24 * 4: - if case.last_action != self.STATUS_REVIEW: - case.status = self.STATUS_REVIEW - return SEND_MESSAGE_TO_WT:DRN + def clerk_stale_case(self, case): + flagged = self.check_for_review(case): + if flagged: + return flagged signatures = self.read_signatures(case.body) if signatures - SIGNATURES_FROM_DATABASE: @@ -296,7 +299,7 @@ class DRNClerkBot(Task): case.status = self.STATUS_OPEN return [] - def clerk_review_case(self, conn, case, volunteers): + def clerk_review_case(self, case): if time() - case.file_time > 60 * 60 * 24 * 7: if not case.very_old_notified: return SEND_MESSAGE_TO_ZHANG @@ -311,10 +314,11 @@ class DRNClerkBot(Task): def read_signatures(self, text): raise NotImplementedError() - def notify_parties(self, conn, case): + def notify_parties(self, case): if case.parties_notified: return raise NotImplementedError() + case.parties_notified = True def save(self, page, cases, kwargs): """Save any changes to the noticeboard.""" From cdee43eb0602b196d05ab5d02b0b883eec104424 Mon Sep 17 00:00:00 2001 From: Ben Kurtovic Date: Sat, 28 Jul 2012 04:18:12 -0400 Subject: [PATCH 12/22] Implement some more clerking cases. --- earwigbot/tasks/drn_clerkbot.py | 83 ++++++++++++++++++--------------- earwigbot/tasks/schema/drn_clerkbot.sql | 1 + 2 files changed, 46 insertions(+), 38 deletions(-) diff --git a/earwigbot/tasks/drn_clerkbot.py b/earwigbot/tasks/drn_clerkbot.py index 2e74e0a..8a77662 100644 --- a/earwigbot/tasks/drn_clerkbot.py +++ b/earwigbot/tasks/drn_clerkbot.py @@ -46,6 +46,16 @@ class DRNClerkBot(Task): STATUS_CLOSED = 7 STATUS_ARCHIVE = 8 + ALIASES = { + STATUS_NEW: ("",), + STATUS_OPEN: ("open", "active", "inprogress"), + STATUS_STALE: ("stale",), + STATUS_NEEDASSIST: ("needassist", "relist", "relisted"), + STATUS_REVIEW: ("review",), + STATUS_RESOLVED: ("resolved", "resolve"), + STATUS_CLOSED: ("closed", "close"), + } + def setup(self): """Hook called immediately after the task is loaded.""" cfg = self.config.tasks.get(self.name, {}) @@ -167,7 +177,8 @@ class DRNClerkBot(Task): re_id2 += "(.*?)\}\})()?" repl = ur"\1 " body = re.sub(re_id2, repl.format(id_), body) - case = _Case(id_, title, status, time()) + case = _Case(id_, title, status, self.STATUS_UNKNOWN, time(), + time(), False, False) cases.append(case) else: case.status = status @@ -188,20 +199,11 @@ class DRNClerkBot(Task): def read_status(self, body): """Parse the current status from a case body.""" - aliases = { - self.STATUS_NEW: ("",), - self.STATUS_OPEN: ("open", "active", "inprogress"), - self.STATUS_STALE: ("stale",), - self.STATUS_NEEDASSIST: ("needassist", "relist", "relisted"), - self.STATUS_REVIEW: ("review",), - self.STATUS_RESOLVED: ("resolved", "resolve"), - self.STATUS_CLOSED: ("closed", "close"), - } templ = re.escape(self.tl_status) status = re.search("\{\{" + templ + "\|?(.*?)\}\}", body, re.S|re.U) if not status: return self.STATUS_UNKNOWN - for option, names in aliases.iteritems(): + for option, names in self.ALIASES.iteritems(): if status.group(1).lower() in names: return option return self.STATUS_UNKNOWN @@ -226,35 +228,30 @@ class DRNClerkBot(Task): def clerk_case(self, conn, case, volunteers): """Clerk a particular case and return a list of any notices to send.""" notices = [] + signatures = self.read_signatures(case.body) if case.status == self.STATUS_NEW: - notices = self.clerk_new_case(case, volunteers) + notices = self.clerk_new_case(case, volunteers, signatures) elif case.status == self.STATUS_OPEN: notices = self.clerk_open_case(case) elif case.status == self.STATUS_NEEDASSIST: - notices = self.clerk_needassist_case(case, volunteers) + notices = self.clerk_needassist_case(case, volunteers, signatures) elif case.status == self.STATUS_STALE: - notices = self.clerk_stale_case(case) + notices = self.clerk_stale_case(case, signatures) elif case.status == self.STATUS_REVIEW: notices = self.clerk_review_case(case) elif case.status in [self.STATUS_RESOLVED, self.STATUS_CLOSED]: self.clerk_closed_case(conn) else: - LOG NOT SURE HOW TO DEAL WITH CASE + log = u"Unsure of how to deal with case {0} (title: {1})" + self.logger.error(log.format(case.id, case.title)) return notices - # STORE UPDATES IN DATABASE - # APPLY STATUS UPDATES TO CASE BODY + STORE UPDATES IN DATABASE + APPLY STATUS UPDATES TO CASE BODY return notices - def check_for_review(self, case): - if time() - case.file_time > 60 * 60 * 24 * 4: - if case.last_action != self.STATUS_REVIEW: - case.status = self.STATUS_REVIEW - return SEND_MESSAGE_TO_WT:DRN - - def clerk_new_case(self, case, volunteers): + def clerk_new_case(self, case, volunteers, signatures): notices = self.notify_parties(case) - signatures = self.read_signatures(case.body) if any([editor in volunteers for (editor, time) in signatures]): if case.last_action != self.STATUS_OPEN: case.status = self.STATUS_OPEN @@ -268,32 +265,30 @@ class DRNClerkBot(Task): if len(case.body) - SIZE_WHEN_LAST_VOLUNTEER_EDIT > 15000: if case.last_action != self.STATUS_NEEDASSIST: case.status = self.STATUS_NEEDASSIST - return SEND_MESSAGE_TO_WT:DRN + return self.build_talk_notice(self.STATUS_NEEDASSIST) - if time() - LAST_EDIT > 60 * 60 * 24 * 2: + if time() - case.modify_time > 60 * 60 * 24 * 2: if case.last_action != self.STATUS_STALE: case.status = self.STATUS_STALE - return SEND_MESSAGE_TO_WT:DRN + return self.build_talk_notice(self.STATUS_STALE) return [] - def clerk_needassist_case(self, case, volunteers): + def clerk_needassist_case(self, case, volunteers, signatures): flagged = self.check_for_review(case): if flagged: return flagged - signatures = self.read_signatures(case.body) newsigs = signatures - SIGNATURES_FROM_DATABASE if any([editor in volunteers for (editor, time) in newsigs]): if case.last_action != self.STATUS_OPEN: case.status = self.STATUS_OPEN return [] - def clerk_stale_case(self, case): + def clerk_stale_case(self, case, signatures): flagged = self.check_for_review(case): if flagged: return flagged - signatures = self.read_signatures(case.body) if signatures - SIGNATURES_FROM_DATABASE: if case.last_action != self.STATUS_OPEN: case.status = self.STATUS_OPEN @@ -306,18 +301,29 @@ class DRNClerkBot(Task): return [] def clerk_closed_case(self, case): - if time() - TIME_STATUS_SET AND LAST_EDIT > 60 * 60 * 24: + if time() - TIME_STATUS_SET > 60 * 60 * 24 and time() - case.modify_time > 60 * 60 * 24: case.status = self.STATUS_ARCHIVE ADD_ARCHIVE_TEMPLATE REMOVE_NOARCHIVE + def check_for_review(self, case): + if time() - case.file_time > 60 * 60 * 24 * 4: + if case.last_action != self.STATUS_REVIEW: + case.status = self.STATUS_REVIEW + return self.build_talk_notice(self.STATUS_REVIEW) + def read_signatures(self, text): - raise NotImplementedError() + raise NotImplementedError() # TODO + + def build_talk_notice(self, status): + param = self.ALIASES[status][0] + template = "{{subst:" + self.tl_notify_stale + "|" + param + "}} ~~~~" + return _Notice(self.talk, template) def notify_parties(self, case): if case.parties_notified: return - raise NotImplementedError() + raise NotImplementedError() # TODO case.parties_notified = True def save(self, page, cases, kwargs): @@ -358,7 +364,7 @@ class DRNClerkBot(Task): text = page.get() except exceptions.PageNotFoundError: text = "" - if notice.too_late in text: + if notice.too_late and notice.too_late in text: log = u"Skipping [[{0}]]; was already notified".format(target) self.logger.info(log) text += ("\n" if text else "") + template @@ -374,13 +380,14 @@ class DRNClerkBot(Task): class _Case(object): """A object representing a dispute resolution case.""" - def __init__(self, id_, title, status, last_action, file_time, + def __init__(self, id_, title, status, last_action, file_time, modify_time, parties_notified, very_old_notified): self.id = id_ self.title = title self.status = status self.last_action = last_action self.file_time = file_time + self.modify_time = modify_time self.parties_notified = parties_notified self.very_old_notified = very_old_notified @@ -391,7 +398,7 @@ class _Case(object): class _Notice(object): """An object representing a notice to be sent to a user or a page.""" - def __init__(self, target, template, too_late): + def __init__(self, target, template, too_late=None): self.target = target self.template = template self.too_late = too_late diff --git a/earwigbot/tasks/schema/drn_clerkbot.sql b/earwigbot/tasks/schema/drn_clerkbot.sql index 81739dc..ecf0920 100644 --- a/earwigbot/tasks/schema/drn_clerkbot.sql +++ b/earwigbot/tasks/schema/drn_clerkbot.sql @@ -19,6 +19,7 @@ CREATE TABLE `case` ( `case_status` int(2) unsigned DEFAULT NULL, `case_last_action` int(2) unsigned DEFAULT NULL, `case_file_time` int(10) unsigned DEFAULT NULL, + `case_modify_time` int(10) unsigned DEFAULT NULL, `case_parties_notified` tinyint(1) unsigned DEFAULT NULL, `case_very_old_notified` tinyint(1) unsigned DEFAULT NULL, PRIMARY KEY (`case_id`) From ec0a95d66ac6cacfbfe7cb1b05550fe938e7bbeb Mon Sep 17 00:00:00 2001 From: Ben Kurtovic Date: Sat, 28 Jul 2012 04:32:13 -0400 Subject: [PATCH 13/22] Implementing more things. --- earwigbot/tasks/drn_clerkbot.py | 32 ++++++++++++++++++++------------ earwigbot/tasks/schema/drn_clerkbot.sql | 2 +- 2 files changed, 21 insertions(+), 13 deletions(-) diff --git a/earwigbot/tasks/drn_clerkbot.py b/earwigbot/tasks/drn_clerkbot.py index 8a77662..99ccd21 100644 --- a/earwigbot/tasks/drn_clerkbot.py +++ b/earwigbot/tasks/drn_clerkbot.py @@ -167,7 +167,7 @@ class DRNClerkBot(Task): if not re.search("\s*\{\{" + tl_status_esc, body, re.U): continue status = self.read_status(body) - re_id = "" + re_id = "" try: id_ = re.search(re_id, body).group(1) case = [case for case in cases if case.id == id_][0] @@ -178,7 +178,7 @@ class DRNClerkBot(Task): repl = ur"\1 " body = re.sub(re_id2, repl.format(id_), body) case = _Case(id_, title, status, self.STATUS_UNKNOWN, time(), - time(), False, False) + 0, False, False) cases.append(case) else: case.status = status @@ -232,7 +232,7 @@ class DRNClerkBot(Task): if case.status == self.STATUS_NEW: notices = self.clerk_new_case(case, volunteers, signatures) elif case.status == self.STATUS_OPEN: - notices = self.clerk_open_case(case) + notices = self.clerk_open_case(case, signatures) elif case.status == self.STATUS_NEEDASSIST: notices = self.clerk_needassist_case(case, volunteers, signatures) elif case.status == self.STATUS_STALE: @@ -240,7 +240,7 @@ class DRNClerkBot(Task): elif case.status == self.STATUS_REVIEW: notices = self.clerk_review_case(case) elif case.status in [self.STATUS_RESOLVED, self.STATUS_CLOSED]: - self.clerk_closed_case(conn) + self.clerk_closed_case(conn, signatures) else: log = u"Unsure of how to deal with case {0} (title: {1})" self.logger.error(log.format(case.id, case.title)) @@ -252,12 +252,12 @@ class DRNClerkBot(Task): def clerk_new_case(self, case, volunteers, signatures): notices = self.notify_parties(case) - if any([editor in volunteers for (editor, time) in signatures]): + if any([editor in volunteers for (editor, timestamp) in signatures]): if case.last_action != self.STATUS_OPEN: case.status = self.STATUS_OPEN return notices - def clerk_open_case(self, case): + def clerk_open_case(self, case, signatures): flagged = self.check_for_review(case): if flagged: return flagged @@ -267,7 +267,8 @@ class DRNClerkBot(Task): case.status = self.STATUS_NEEDASSIST return self.build_talk_notice(self.STATUS_NEEDASSIST) - if time() - case.modify_time > 60 * 60 * 24 * 2: + timestamps = [timestamp for (editor, timestamp) in signatures] + if time() - max(timestamps) > 60 * 60 * 24 * 2: if case.last_action != self.STATUS_STALE: case.status = self.STATUS_STALE return self.build_talk_notice(self.STATUS_STALE) @@ -279,7 +280,7 @@ class DRNClerkBot(Task): return flagged newsigs = signatures - SIGNATURES_FROM_DATABASE - if any([editor in volunteers for (editor, time) in newsigs]): + if any([editor in volunteers for (editor, timestamp) in newsigs]): if case.last_action != self.STATUS_OPEN: case.status = self.STATUS_OPEN return [] @@ -297,11 +298,17 @@ class DRNClerkBot(Task): def clerk_review_case(self, case): if time() - case.file_time > 60 * 60 * 24 * 7: if not case.very_old_notified: + case.very_old_notified = True return SEND_MESSAGE_TO_ZHANG return [] - def clerk_closed_case(self, case): - if time() - TIME_STATUS_SET > 60 * 60 * 24 and time() - case.modify_time > 60 * 60 * 24: + def clerk_closed_case(self, case, signatures): + if not case.close_time: + case.close_time = time() + timestamps = [timestamp for (editor, timestamp) in signatures] + closed_long_ago = time() - case.close_time > 60 * 60 * 24 + modified_long_ago = time() - max(timestamps) > 60 * 60 * 24 + if closed_long_ago and modified_long_ago: case.status = self.STATUS_ARCHIVE ADD_ARCHIVE_TEMPLATE REMOVE_NOARCHIVE @@ -314,6 +321,7 @@ class DRNClerkBot(Task): def read_signatures(self, text): raise NotImplementedError() # TODO + return [(username, timestamp_datetime)...] def build_talk_notice(self, status): param = self.ALIASES[status][0] @@ -380,14 +388,14 @@ class DRNClerkBot(Task): class _Case(object): """A object representing a dispute resolution case.""" - def __init__(self, id_, title, status, last_action, file_time, modify_time, + def __init__(self, id_, title, status, last_action, file_time, close_time, parties_notified, very_old_notified): self.id = id_ self.title = title self.status = status self.last_action = last_action self.file_time = file_time - self.modify_time = modify_time + self.close_time = close_time self.parties_notified = parties_notified self.very_old_notified = very_old_notified diff --git a/earwigbot/tasks/schema/drn_clerkbot.sql b/earwigbot/tasks/schema/drn_clerkbot.sql index ecf0920..cc11363 100644 --- a/earwigbot/tasks/schema/drn_clerkbot.sql +++ b/earwigbot/tasks/schema/drn_clerkbot.sql @@ -19,7 +19,7 @@ CREATE TABLE `case` ( `case_status` int(2) unsigned DEFAULT NULL, `case_last_action` int(2) unsigned DEFAULT NULL, `case_file_time` int(10) unsigned DEFAULT NULL, - `case_modify_time` int(10) unsigned DEFAULT NULL, + `case_close_time` int(10) unsigned DEFAULT NULL, `case_parties_notified` tinyint(1) unsigned DEFAULT NULL, `case_very_old_notified` tinyint(1) unsigned DEFAULT NULL, PRIMARY KEY (`case_id`) From d2b4379ebff6937d600ed6920ce638b43c351a7c Mon Sep 17 00:00:00 2001 From: Ben Kurtovic Date: Sat, 28 Jul 2012 04:43:33 -0400 Subject: [PATCH 14/22] Implementing some more; adding TODO notes in margins. Will finish in morning. --- earwigbot/tasks/drn_clerkbot.py | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/earwigbot/tasks/drn_clerkbot.py b/earwigbot/tasks/drn_clerkbot.py index 99ccd21..7e4a920 100644 --- a/earwigbot/tasks/drn_clerkbot.py +++ b/earwigbot/tasks/drn_clerkbot.py @@ -20,6 +20,7 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. +from datetime import datetime from os import expanduser import re from threading import RLock @@ -246,8 +247,8 @@ class DRNClerkBot(Task): self.logger.error(log.format(case.id, case.title)) return notices - STORE UPDATES IN DATABASE - APPLY STATUS UPDATES TO CASE BODY + STORE UPDATES IN DATABASE # TODO + APPLY STATUS UPDATES TO CASE BODY # TODO return notices def clerk_new_case(self, case, volunteers, signatures): @@ -262,7 +263,7 @@ class DRNClerkBot(Task): if flagged: return flagged - if len(case.body) - SIZE_WHEN_LAST_VOLUNTEER_EDIT > 15000: + if len(case.body) - SIZE_WHEN_LAST_VOLUNTEER_EDIT > 15000: # TODO if case.last_action != self.STATUS_NEEDASSIST: case.status = self.STATUS_NEEDASSIST return self.build_talk_notice(self.STATUS_NEEDASSIST) @@ -279,7 +280,7 @@ class DRNClerkBot(Task): if flagged: return flagged - newsigs = signatures - SIGNATURES_FROM_DATABASE + newsigs = signatures - SIGNATURES_FROM_DATABASE # TODO if any([editor in volunteers for (editor, timestamp) in newsigs]): if case.last_action != self.STATUS_OPEN: case.status = self.STATUS_OPEN @@ -290,7 +291,7 @@ class DRNClerkBot(Task): if flagged: return flagged - if signatures - SIGNATURES_FROM_DATABASE: + if signatures - SIGNATURES_FROM_DATABASE: # TODO if case.last_action != self.STATUS_OPEN: case.status = self.STATUS_OPEN return [] @@ -299,7 +300,7 @@ class DRNClerkBot(Task): if time() - case.file_time > 60 * 60 * 24 * 7: if not case.very_old_notified: case.very_old_notified = True - return SEND_MESSAGE_TO_ZHANG + return SEND_MESSAGE_TO_ZHANG # TODO return [] def clerk_closed_case(self, case, signatures): @@ -310,8 +311,10 @@ class DRNClerkBot(Task): modified_long_ago = time() - max(timestamps) > 60 * 60 * 24 if closed_long_ago and modified_long_ago: case.status = self.STATUS_ARCHIVE - ADD_ARCHIVE_TEMPLATE - REMOVE_NOARCHIVE + case.body = "{{" + self.tl_archive_top + "}}\n" + case.body + case.body += "\n{{" + self.tl_archive_bottom + "}}" + reg = "()?" + case.body = re.sub(reg, "", case.body) def check_for_review(self, case): if time() - case.file_time > 60 * 60 * 24 * 4: @@ -320,7 +323,7 @@ class DRNClerkBot(Task): return self.build_talk_notice(self.STATUS_REVIEW) def read_signatures(self, text): - raise NotImplementedError() # TODO + raise NotImplementedError() # TODO return [(username, timestamp_datetime)...] def build_talk_notice(self, status): @@ -331,7 +334,7 @@ class DRNClerkBot(Task): def notify_parties(self, case): if case.parties_notified: return - raise NotImplementedError() # TODO + raise NotImplementedError() # TODO case.parties_notified = True def save(self, page, cases, kwargs): From eb729fd9333aadf1d6cbbd0923acb4c4def6d9ef Mon Sep 17 00:00:00 2001 From: Ben Kurtovic Date: Sat, 28 Jul 2012 13:54:59 -0400 Subject: [PATCH 15/22] Implement storing updates in database and applying updates to case. --- earwigbot/tasks/drn_clerkbot.py | 48 +++++++++++++++++++++++++++++---- earwigbot/tasks/schema/drn_clerkbot.sql | 12 +++++++++ 2 files changed, 55 insertions(+), 5 deletions(-) diff --git a/earwigbot/tasks/drn_clerkbot.py b/earwigbot/tasks/drn_clerkbot.py index 7e4a920..c8f72c7 100644 --- a/earwigbot/tasks/drn_clerkbot.py +++ b/earwigbot/tasks/drn_clerkbot.py @@ -150,7 +150,7 @@ class DRNClerkBot(Task): with conn.cursor() as cursor: cursor.execute("SELECT * FROM case") for row in cursor: - case = _Case(*row) + case = _Case(*row, new=False) cases.append(case) return cases @@ -179,7 +179,7 @@ class DRNClerkBot(Task): repl = ur"\1 " body = re.sub(re_id2, repl.format(id_), body) case = _Case(id_, title, status, self.STATUS_UNKNOWN, time(), - 0, False, False) + 0, False, False, new=True) cases.append(case) else: case.status = status @@ -247,8 +247,45 @@ class DRNClerkBot(Task): self.logger.error(log.format(case.id, case.title)) return notices - STORE UPDATES IN DATABASE # TODO - APPLY STATUS UPDATES TO CASE BODY # TODO + if case.status != case.original_status: + case.last_action = case.status + new = self.ALIASES[case.status][0] + tl_status_esc = re.escape(self.tl_status) + search = "\{\{" + tl_status_esc + "(\|?.*?)\}\}" + repl = "{{" + self.tl_status + "|" + new + "}}" + case.body = re.sub(search, repl, case.body) + + if case.new: + with conn.cursor() as cursor: + query = "INSERT INTO case VALUES (?, ?, ?, ?, ?, ?, ?, ?)" + cursor.execute(query, (case.id, case.title, case.status, + case.last_action, case.file_time, + case.close_time, case.parties_notified, + case.very_old_notified)) + return notices + + with conn.cursor(oursql.DictCursor) as cursor: + query = "SELECT * FROM case WHERE case_id = ?" + cursor.execute(query, (case.id,)) + stored = cursor.fetchone() + with conn.cursor() as cursor: + changes, args = [], [] + fields_to_check = [ + ("case_status", case.status), + ("case_last_action", case.last_action), + ("case_close_time", case.close_time), + ("case_parties_notified", case.parties_notified), + ("case_very_old_notified", case.very_old_notified) + ] + for column, data in fields_to_check: + if data != stored[column]: + changes.append(column + " = ?") + args.append(data) + if changes: + changes = ", ".join(changes) + args.append(case.id) + query = "UPDATE case SET {0} WHERE case_id = ?".format(changes) + cursor.execute(query, args) return notices def clerk_new_case(self, case, volunteers, signatures): @@ -392,7 +429,7 @@ class DRNClerkBot(Task): class _Case(object): """A object representing a dispute resolution case.""" def __init__(self, id_, title, status, last_action, file_time, close_time, - parties_notified, very_old_notified): + parties_notified, very_old_notified, new): self.id = id_ self.title = title self.status = status @@ -401,6 +438,7 @@ class _Case(object): self.close_time = close_time self.parties_notified = parties_notified self.very_old_notified = very_old_notified + self.new = new self.original_status = status self.body = None diff --git a/earwigbot/tasks/schema/drn_clerkbot.sql b/earwigbot/tasks/schema/drn_clerkbot.sql index cc11363..25cded6 100644 --- a/earwigbot/tasks/schema/drn_clerkbot.sql +++ b/earwigbot/tasks/schema/drn_clerkbot.sql @@ -26,6 +26,18 @@ CREATE TABLE `case` ( ) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci; -- +-- Table structure for table `signature` +-- + +DROP TABLE IF EXISTS `signature`; +CREATE TABLE `signature` ( + `signature_id` int(10) unsigned NOT NULL, + `signature_case` int(10) unsigned NOT NULL, + `signature_timestamp` timestamp NOT NULL DEFAULT '0000-00-00 00:00:00', + PRIMARY KEY (`signature_id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci; + +-- -- Table structure for table `volunteer` -- From ab1fe9926ab4009459960a84010fe9cfff373847 Mon Sep 17 00:00:00 2001 From: Ben Kurtovic Date: Sat, 28 Jul 2012 13:59:29 -0400 Subject: [PATCH 16/22] Split up that one huge function into three. --- earwigbot/tasks/drn_clerkbot.py | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/earwigbot/tasks/drn_clerkbot.py b/earwigbot/tasks/drn_clerkbot.py index c8f72c7..13f9b4c 100644 --- a/earwigbot/tasks/drn_clerkbot.py +++ b/earwigbot/tasks/drn_clerkbot.py @@ -246,7 +246,10 @@ class DRNClerkBot(Task): log = u"Unsure of how to deal with case {0} (title: {1})" self.logger.error(log.format(case.id, case.title)) return notices + self.save_case_updates(conn, case) + return notices + def save_case_updates(self, conn, case): if case.status != case.original_status: case.last_action = case.status new = self.ALIASES[case.status][0] @@ -256,14 +259,19 @@ class DRNClerkBot(Task): case.body = re.sub(search, repl, case.body) if case.new: - with conn.cursor() as cursor: - query = "INSERT INTO case VALUES (?, ?, ?, ?, ?, ?, ?, ?)" - cursor.execute(query, (case.id, case.title, case.status, - case.last_action, case.file_time, - case.close_time, case.parties_notified, - case.very_old_notified)) - return notices + self.save_new_case(conn, case) + else: + self.save_existing_case(conn, case) + def save_new_case(self, conn, case): + args = (case.id, case.title, case.status, case.last_action, + case.file_time, case.close_time, case.parties_notified, + case.very_old_notified) + with conn.cursor() as cursor: + query = "INSERT INTO case VALUES (?, ?, ?, ?, ?, ?, ?, ?)" + cursor.execute(query, args) + + def save_existing_case(self, conn, case): with conn.cursor(oursql.DictCursor) as cursor: query = "SELECT * FROM case WHERE case_id = ?" cursor.execute(query, (case.id,)) @@ -286,7 +294,6 @@ class DRNClerkBot(Task): args.append(case.id) query = "UPDATE case SET {0} WHERE case_id = ?".format(changes) cursor.execute(query, args) - return notices def clerk_new_case(self, case, volunteers, signatures): notices = self.notify_parties(case) From 64b9b77cf40fe08ec4664b8be18586203afc53b5 Mon Sep 17 00:00:00 2001 From: Ben Kurtovic Date: Sat, 28 Jul 2012 14:18:08 -0400 Subject: [PATCH 17/22] Implement storing signatures in the database. --- earwigbot/tasks/drn_clerkbot.py | 136 +++++++++++++++++++------------- earwigbot/tasks/schema/drn_clerkbot.sql | 1 + 2 files changed, 80 insertions(+), 57 deletions(-) diff --git a/earwigbot/tasks/drn_clerkbot.py b/earwigbot/tasks/drn_clerkbot.py index 13f9b4c..7f3e2c2 100644 --- a/earwigbot/tasks/drn_clerkbot.py +++ b/earwigbot/tasks/drn_clerkbot.py @@ -173,7 +173,7 @@ class DRNClerkBot(Task): id_ = re.search(re_id, body).group(1) case = [case for case in cases if case.id == id_][0] except (AttributeError, IndexError): - id_ = self.select_next_id(conn) + id_ = self.select_next_id(conn, "case_id", "case") re_id2 = "(\{\{" + tl_status_esc re_id2 += "(.*?)\}\})()?" repl = ur"\1 " @@ -188,11 +188,11 @@ class DRNClerkBot(Task): case.title = title case.body, case.old = body, old - def select_next_id(self, conn): + def select_next_id(self, conn, column, table): """Return the next incremental ID for a case.""" - query = "SELECT MAX(case_id) FROM case" + query = "SELECT MAX(?) FROM ?" with conn.cursor() as cursor: - cursor.execute(query) + cursor.execute(query, (column, table)) current = cursor.fetchone()[0] if current: return current + 1 @@ -230,14 +230,16 @@ class DRNClerkBot(Task): """Clerk a particular case and return a list of any notices to send.""" notices = [] signatures = self.read_signatures(case.body) + storedsigs = self.get_signatures_from_db(conn, case) if case.status == self.STATUS_NEW: notices = self.clerk_new_case(case, volunteers, signatures) elif case.status == self.STATUS_OPEN: notices = self.clerk_open_case(case, signatures) elif case.status == self.STATUS_NEEDASSIST: - notices = self.clerk_needassist_case(case, volunteers, signatures) + notices = self.clerk_needassist_case(case, volunteers, signatures, + storedsigs) elif case.status == self.STATUS_STALE: - notices = self.clerk_stale_case(case, signatures) + notices = self.clerk_stale_case(case, signatures, storedsigs) elif case.status == self.STATUS_REVIEW: notices = self.clerk_review_case(case) elif case.status in [self.STATUS_RESOLVED, self.STATUS_CLOSED]: @@ -246,55 +248,9 @@ class DRNClerkBot(Task): log = u"Unsure of how to deal with case {0} (title: {1})" self.logger.error(log.format(case.id, case.title)) return notices - self.save_case_updates(conn, case) + self.save_case_updates(conn, case, signatures, storedsigs) return notices - def save_case_updates(self, conn, case): - if case.status != case.original_status: - case.last_action = case.status - new = self.ALIASES[case.status][0] - tl_status_esc = re.escape(self.tl_status) - search = "\{\{" + tl_status_esc + "(\|?.*?)\}\}" - repl = "{{" + self.tl_status + "|" + new + "}}" - case.body = re.sub(search, repl, case.body) - - if case.new: - self.save_new_case(conn, case) - else: - self.save_existing_case(conn, case) - - def save_new_case(self, conn, case): - args = (case.id, case.title, case.status, case.last_action, - case.file_time, case.close_time, case.parties_notified, - case.very_old_notified) - with conn.cursor() as cursor: - query = "INSERT INTO case VALUES (?, ?, ?, ?, ?, ?, ?, ?)" - cursor.execute(query, args) - - def save_existing_case(self, conn, case): - with conn.cursor(oursql.DictCursor) as cursor: - query = "SELECT * FROM case WHERE case_id = ?" - cursor.execute(query, (case.id,)) - stored = cursor.fetchone() - with conn.cursor() as cursor: - changes, args = [], [] - fields_to_check = [ - ("case_status", case.status), - ("case_last_action", case.last_action), - ("case_close_time", case.close_time), - ("case_parties_notified", case.parties_notified), - ("case_very_old_notified", case.very_old_notified) - ] - for column, data in fields_to_check: - if data != stored[column]: - changes.append(column + " = ?") - args.append(data) - if changes: - changes = ", ".join(changes) - args.append(case.id) - query = "UPDATE case SET {0} WHERE case_id = ?".format(changes) - cursor.execute(query, args) - def clerk_new_case(self, case, volunteers, signatures): notices = self.notify_parties(case) if any([editor in volunteers for (editor, timestamp) in signatures]): @@ -319,23 +275,23 @@ class DRNClerkBot(Task): return self.build_talk_notice(self.STATUS_STALE) return [] - def clerk_needassist_case(self, case, volunteers, signatures): + def clerk_needassist_case(self, case, volunteers, signatures, storedsigs): flagged = self.check_for_review(case): if flagged: return flagged - newsigs = signatures - SIGNATURES_FROM_DATABASE # TODO + newsigs = set(signatures) - set(storedsigs) if any([editor in volunteers for (editor, timestamp) in newsigs]): if case.last_action != self.STATUS_OPEN: case.status = self.STATUS_OPEN return [] - def clerk_stale_case(self, case, signatures): + def clerk_stale_case(self, case, signatures, storedsigs): flagged = self.check_for_review(case): if flagged: return flagged - if signatures - SIGNATURES_FROM_DATABASE: # TODO + if set(signatures) - set(storedsigs) if case.last_action != self.STATUS_OPEN: case.status = self.STATUS_OPEN return [] @@ -370,6 +326,9 @@ class DRNClerkBot(Task): raise NotImplementedError() # TODO return [(username, timestamp_datetime)...] + def get_signatures_from_db(self, conn, case): + raise NotImplementedError() # TODO + def build_talk_notice(self, status): param = self.ALIASES[status][0] template = "{{subst:" + self.tl_notify_stale + "|" + param + "}} ~~~~" @@ -381,6 +340,69 @@ class DRNClerkBot(Task): raise NotImplementedError() # TODO case.parties_notified = True + def save_case_updates(self, conn, case, signatures, storedsigs): + if case.status != case.original_status: + case.last_action = case.status + new = self.ALIASES[case.status][0] + tl_status_esc = re.escape(self.tl_status) + search = "\{\{" + tl_status_esc + "(\|?.*?)\}\}" + repl = "{{" + self.tl_status + "|" + new + "}}" + case.body = re.sub(search, repl, case.body) + + if case.new: + self.save_new_case(conn, case) + else: + self.save_existing_case(conn, case) + + with conn.cursor() as cursor: + query1 = "DELETE FROM signature WHERE signature_case = ? AND signature_username = ? AND signature_timestamp = ?" + query2 = "INSERT INTO signature VALUES (?, ?, ?, ?)" + removals = set(storedsigs) - set(signatures) + additions = set(signatures) - set(storedsigs) + if removals: + args = [(case.id, name, stamp) for (name, stamp) in removals] + cursor.execute(query1, args) + if additions: + nextid = self.select_next_id(conn, "signature_id", "signature") + args = [] + for name, stamp in additions: + args.append((nextid, case.id, name, stamp)) + nextid += 1 + cursor.execute(query2, args) + + def save_new_case(self, conn, case): + args = (case.id, case.title, case.status, case.last_action, + case.file_time, case.close_time, case.parties_notified, + case.very_old_notified) + with conn.cursor() as cursor: + query = "INSERT INTO case VALUES (?, ?, ?, ?, ?, ?, ?, ?)" + cursor.execute(query, args) + + def save_existing_case(self, conn, case): + with conn.cursor(oursql.DictCursor) as cursor: + query = "SELECT * FROM case WHERE case_id = ?" + cursor.execute(query, (case.id,)) + stored = cursor.fetchone() + + with conn.cursor() as cursor: + changes, args = [], [] + fields_to_check = [ + ("case_status", case.status), + ("case_last_action", case.last_action), + ("case_close_time", case.close_time), + ("case_parties_notified", case.parties_notified), + ("case_very_old_notified", case.very_old_notified) + ] + for column, data in fields_to_check: + if data != stored[column]: + changes.append(column + " = ?") + args.append(data) + if changes: + changes = ", ".join(changes) + args.append(case.id) + query = "UPDATE case SET {0} WHERE case_id = ?".format(changes) + cursor.execute(query, args) + def save(self, page, cases, kwargs): """Save any changes to the noticeboard.""" newtext = text = page.get() diff --git a/earwigbot/tasks/schema/drn_clerkbot.sql b/earwigbot/tasks/schema/drn_clerkbot.sql index 25cded6..f39e1e4 100644 --- a/earwigbot/tasks/schema/drn_clerkbot.sql +++ b/earwigbot/tasks/schema/drn_clerkbot.sql @@ -33,6 +33,7 @@ DROP TABLE IF EXISTS `signature`; CREATE TABLE `signature` ( `signature_id` int(10) unsigned NOT NULL, `signature_case` int(10) unsigned NOT NULL, + `signature_username` varchar(512) COLLATE utf8_unicode_ci DEFAULT NULL, `signature_timestamp` timestamp NOT NULL DEFAULT '0000-00-00 00:00:00', PRIMARY KEY (`signature_id`) ) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci; From 8713607df39d3e9b0432c4bfadc6c327fdb87d14 Mon Sep 17 00:00:00 2001 From: Ben Kurtovic Date: Sat, 28 Jul 2012 14:23:53 -0400 Subject: [PATCH 18/22] Implement get_signatures_from_db(). --- earwigbot/tasks/drn_clerkbot.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/earwigbot/tasks/drn_clerkbot.py b/earwigbot/tasks/drn_clerkbot.py index 7f3e2c2..a268742 100644 --- a/earwigbot/tasks/drn_clerkbot.py +++ b/earwigbot/tasks/drn_clerkbot.py @@ -327,7 +327,10 @@ class DRNClerkBot(Task): return [(username, timestamp_datetime)...] def get_signatures_from_db(self, conn, case): - raise NotImplementedError() # TODO + query = "SELECT signature_username, signature_timestamp FROM signature WHERE signature_case = ?" + with conn.cursor() as cursor: + cursor.execute(query, (case.id,)) + return cursor.fetchall() def build_talk_notice(self, status): param = self.ALIASES[status][0] From d086f03c2ca0036cce4190a55bdf1bbd48dc4264 Mon Sep 17 00:00:00 2001 From: Ben Kurtovic Date: Sat, 28 Jul 2012 14:43:33 -0400 Subject: [PATCH 19/22] Implement last_volunteer_size. --- earwigbot/tasks/drn_clerkbot.py | 31 +++++++++++++++++++------------ earwigbot/tasks/schema/drn_clerkbot.sql | 1 + 2 files changed, 20 insertions(+), 12 deletions(-) diff --git a/earwigbot/tasks/drn_clerkbot.py b/earwigbot/tasks/drn_clerkbot.py index a268742..565e720 100644 --- a/earwigbot/tasks/drn_clerkbot.py +++ b/earwigbot/tasks/drn_clerkbot.py @@ -150,7 +150,7 @@ class DRNClerkBot(Task): with conn.cursor() as cursor: cursor.execute("SELECT * FROM case") for row in cursor: - case = _Case(*row, new=False) + case = _Case(*row) cases.append(case) return cases @@ -179,7 +179,7 @@ class DRNClerkBot(Task): repl = ur"\1 " body = re.sub(re_id2, repl.format(id_), body) case = _Case(id_, title, status, self.STATUS_UNKNOWN, time(), - 0, False, False, new=True) + 0, False, False, 0, new=True) cases.append(case) else: case.status = status @@ -231,15 +231,18 @@ class DRNClerkBot(Task): notices = [] signatures = self.read_signatures(case.body) storedsigs = self.get_signatures_from_db(conn, case) + newsigs = set(signatures) - set(storedsigs) + if any([editor in volunteers for (editor, timestamp) in newsigs]): + case.last_volunteer_size = len(case.body) + if case.status == self.STATUS_NEW: notices = self.clerk_new_case(case, volunteers, signatures) elif case.status == self.STATUS_OPEN: notices = self.clerk_open_case(case, signatures) elif case.status == self.STATUS_NEEDASSIST: - notices = self.clerk_needassist_case(case, volunteers, signatures, - storedsigs) + notices = self.clerk_needassist_case(case, volunteers, newsigs) elif case.status == self.STATUS_STALE: - notices = self.clerk_stale_case(case, signatures, storedsigs) + notices = self.clerk_stale_case(case, newsigs) elif case.status == self.STATUS_REVIEW: notices = self.clerk_review_case(case) elif case.status in [self.STATUS_RESOLVED, self.STATUS_CLOSED]: @@ -263,7 +266,7 @@ class DRNClerkBot(Task): if flagged: return flagged - if len(case.body) - SIZE_WHEN_LAST_VOLUNTEER_EDIT > 15000: # TODO + if len(case.body) - case.last_volunteer_size > 15000: if case.last_action != self.STATUS_NEEDASSIST: case.status = self.STATUS_NEEDASSIST return self.build_talk_notice(self.STATUS_NEEDASSIST) @@ -275,23 +278,22 @@ class DRNClerkBot(Task): return self.build_talk_notice(self.STATUS_STALE) return [] - def clerk_needassist_case(self, case, volunteers, signatures, storedsigs): + def clerk_needassist_case(self, case, volunteers, newsigs): flagged = self.check_for_review(case): if flagged: return flagged - newsigs = set(signatures) - set(storedsigs) if any([editor in volunteers for (editor, timestamp) in newsigs]): if case.last_action != self.STATUS_OPEN: case.status = self.STATUS_OPEN return [] - def clerk_stale_case(self, case, signatures, storedsigs): + def clerk_stale_case(self, case, newsigs): flagged = self.check_for_review(case): if flagged: return flagged - if set(signatures) - set(storedsigs) + if newsigs: if case.last_action != self.STATUS_OPEN: case.status = self.STATUS_OPEN return [] @@ -340,6 +342,8 @@ class DRNClerkBot(Task): def notify_parties(self, case): if case.parties_notified: return + template = "{{subst:" + self.tl_notify_party + template += "|thread=" + case.title + "}} ~~~~" raise NotImplementedError() # TODO case.parties_notified = True @@ -394,7 +398,8 @@ class DRNClerkBot(Task): ("case_last_action", case.last_action), ("case_close_time", case.close_time), ("case_parties_notified", case.parties_notified), - ("case_very_old_notified", case.very_old_notified) + ("case_very_old_notified", case.very_old_notified), + ("case_last_volunteer_size", case.last_volunteer_size) ] for column, data in fields_to_check: if data != stored[column]: @@ -461,7 +466,8 @@ class DRNClerkBot(Task): class _Case(object): """A object representing a dispute resolution case.""" def __init__(self, id_, title, status, last_action, file_time, close_time, - parties_notified, very_old_notified, new): + parties_notified, very_old_notified, last_volunteer_size, + new=False): self.id = id_ self.title = title self.status = status @@ -470,6 +476,7 @@ class _Case(object): self.close_time = close_time self.parties_notified = parties_notified self.very_old_notified = very_old_notified + self.last_volunteer_size = last_volunteer_size self.new = new self.original_status = status diff --git a/earwigbot/tasks/schema/drn_clerkbot.sql b/earwigbot/tasks/schema/drn_clerkbot.sql index f39e1e4..23ce572 100644 --- a/earwigbot/tasks/schema/drn_clerkbot.sql +++ b/earwigbot/tasks/schema/drn_clerkbot.sql @@ -22,6 +22,7 @@ CREATE TABLE `case` ( `case_close_time` int(10) unsigned DEFAULT NULL, `case_parties_notified` tinyint(1) unsigned DEFAULT NULL, `case_very_old_notified` tinyint(1) unsigned DEFAULT NULL, + `case_last_volunteer_size` int(9) unsigned DEFAULT NULL, PRIMARY KEY (`case_id`) ) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci; From ff93105c3cd2d6aa5378d153831b1b9c378b7bd9 Mon Sep 17 00:00:00 2001 From: Ben Kurtovic Date: Sat, 28 Jul 2012 14:53:23 -0400 Subject: [PATCH 20/22] Finish implementing clerk_review_case(). --- earwigbot/tasks/drn_clerkbot.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/earwigbot/tasks/drn_clerkbot.py b/earwigbot/tasks/drn_clerkbot.py index 565e720..74359bc 100644 --- a/earwigbot/tasks/drn_clerkbot.py +++ b/earwigbot/tasks/drn_clerkbot.py @@ -65,6 +65,7 @@ class DRNClerkBot(Task): self.title = cfg.get("title", "Wikipedia:Dispute resolution noticeboard") self.talk = cfg.get("talk", "Wikipedia talk:Dispute resolution noticeboard") self.volunteer_title = cfg.get("volunteers", "Wikipedia:Dispute resolution noticeboard/Volunteering") + self.very_old_title = cfg.get("veryOldTitle", "User talk:Szhang (WMF)") default_summary = "Updating $3 cases for the [[WP:DRN|dispute resolution noticeboard]]." self.summary = self.make_summary(cfg.get("summary", default_summary)) @@ -301,8 +302,10 @@ class DRNClerkBot(Task): def clerk_review_case(self, case): if time() - case.file_time > 60 * 60 * 24 * 7: if not case.very_old_notified: + template = "{{subst:" + self.tl_notify_stale + "|zhang}} ~~~~" + notice = _Notice(self.very_old_title, template) case.very_old_notified = True - return SEND_MESSAGE_TO_ZHANG # TODO + return [notice] return [] def clerk_closed_case(self, case, signatures): From 01d72171e3113b1f8de359944c9f837314495776 Mon Sep 17 00:00:00 2001 From: Ben Kurtovic Date: Sat, 28 Jul 2012 15:26:01 -0400 Subject: [PATCH 21/22] Implement notify_parties(). --- earwigbot/tasks/drn_clerkbot.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/earwigbot/tasks/drn_clerkbot.py b/earwigbot/tasks/drn_clerkbot.py index 74359bc..088c1dd 100644 --- a/earwigbot/tasks/drn_clerkbot.py +++ b/earwigbot/tasks/drn_clerkbot.py @@ -344,11 +344,23 @@ class DRNClerkBot(Task): def notify_parties(self, case): if case.parties_notified: - return + return [] + + notices = [] template = "{{subst:" + self.tl_notify_party template += "|thread=" + case.title + "}} ~~~~" - raise NotImplementedError() # TODO + too_late = "" + + re_parties = "'''Users involved'''(.*?)" + text = re.search(re_parties, case, re.S|re.U) + for line in text.group(1).splitlines(): + user = re.search("\# \{\{User\|(.*?)\}\}", line) + if user: + party = user.group(1).strip() + notice = _Notice("User talk:" + party, template, too_late) + case.parties_notified = True + return notices def save_case_updates(self, conn, case, signatures, storedsigs): if case.status != case.original_status: From 205ae4b7a63dc3a01ef264729e6b9bf707491ca2 Mon Sep 17 00:00:00 2001 From: Ben Kurtovic Date: Sat, 28 Jul 2012 15:46:41 -0400 Subject: [PATCH 22/22] Implement read_signatures(). --- earwigbot/tasks/drn_clerkbot.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/earwigbot/tasks/drn_clerkbot.py b/earwigbot/tasks/drn_clerkbot.py index 088c1dd..94771a1 100644 --- a/earwigbot/tasks/drn_clerkbot.py +++ b/earwigbot/tasks/drn_clerkbot.py @@ -328,8 +328,14 @@ class DRNClerkBot(Task): return self.build_talk_notice(self.STATUS_REVIEW) def read_signatures(self, text): - raise NotImplementedError() # TODO - return [(username, timestamp_datetime)...] + regex = r"\[\[(?:User(?:\stalk)?\:|Special\:Contributions\/)(.*?)(?:\||\]\]).{,256}?(\d{2}:\d{2},\s\d{2}\s\w+\s\d{4}\s\(UTC\))" + matches = re.findall(regex, text, re.U) + signatures = [] + for userlink, stamp in matches: + username = userlink.split("/", 1)[0].replace("_", " ") + timestamp = datetime.strptime("%H:%M, %d %B %Y (UTC)", stamp) + signatures.append((username, timestamp)) + return signatures def get_signatures_from_db(self, conn, case): query = "SELECT signature_username, signature_timestamp FROM signature WHERE signature_case = ?"