Additional IRC commands and bot tasks for EarwigBot https://en.wikipedia.org/wiki/User:EarwigBot
Não pode escolher mais do que 25 tópicos Os tópicos devem começar com uma letra ou um número, podem incluir traços ('-') e podem ter até 35 caracteres.

799 linhas
35 KiB

  1. # -*- coding: utf-8 -*-
  2. #
  3. # Copyright (C) 2009-2013 Ben Kurtovic <ben.kurtovic@verizon.net>
  4. #
  5. # Permission is hereby granted, free of charge, to any person obtaining a copy
  6. # of this software and associated documentation files (the "Software"), to deal
  7. # in the Software without restriction, including without limitation the rights
  8. # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
  9. # copies of the Software, and to permit persons to whom the Software is
  10. # furnished to do so, subject to the following conditions:
  11. #
  12. # The above copyright notice and this permission notice shall be included in
  13. # all copies or substantial portions of the Software.
  14. #
  15. # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16. # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17. # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  18. # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19. # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
  20. # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  21. # SOFTWARE.
  22. from datetime import datetime
  23. from os.path import expanduser
  24. import re
  25. from threading import RLock
  26. from time import mktime, sleep, time
  27. import mwparserfromhell
  28. import oursql
  29. from earwigbot import exceptions
  30. from earwigbot.tasks import Task
  31. from earwigbot.wiki import constants
  32. class DRNClerkBot(Task):
  33. """A task to clerk for [[WP:DRN]]."""
  34. name = "drn_clerkbot"
  35. number = 19
  36. # Case status:
  37. STATUS_UNKNOWN = 0
  38. STATUS_NEW = 1
  39. STATUS_OPEN = 2
  40. STATUS_STALE = 3
  41. STATUS_NEEDASSIST = 4
  42. STATUS_RESOLVED = 6
  43. STATUS_CLOSED = 7
  44. STATUS_FAILED = 8
  45. ALIASES = {
  46. STATUS_NEW: ("",),
  47. STATUS_OPEN: ("open", "active", "inprogress"),
  48. STATUS_STALE: ("stale",),
  49. STATUS_NEEDASSIST: ("needassist", "review", "relist", "relisted"),
  50. STATUS_RESOLVED: ("resolved", "resolve"),
  51. STATUS_CLOSED: ("closed", "close"),
  52. STATUS_FAILED: ("failed", "fail"),
  53. }
  54. def setup(self):
  55. """Hook called immediately after the task is loaded."""
  56. cfg = self.config.tasks.get(self.name, {})
  57. # Set some wiki-related attributes:
  58. self.title = cfg.get("title",
  59. "Wikipedia:Dispute resolution noticeboard")
  60. self.chart_title = cfg.get("chartTitle", "Template:DRN case status")
  61. self.volunteer_title = cfg.get("volunteers",
  62. "Wikipedia:Dispute resolution noticeboard/Volunteering")
  63. self.very_old_title = cfg.get("veryOldTitle", "User talk:Szhang (WMF)")
  64. self.notify_stale_cases = cf.get("notifyStaleCases", False)
  65. clerk_summary = "Updating case."
  66. notify_summary = "Notifying user regarding [[WP:DRN|dispute resolution noticeboard]] case."
  67. chart_summary = "Updating statistics for the [[WP:DRN|dispute resolution noticeboard]]."
  68. self.clerk_summary = self.make_summary(cfg.get("clerkSummary", clerk_summary))
  69. self.notify_summary = self.make_summary(cfg.get("notifySummary", notify_summary))
  70. self.chart_summary = self.make_summary(cfg.get("chartSummary", chart_summary))
  71. # Templates used:
  72. templates = cfg.get("templates", {})
  73. self.tl_status = templates.get("status", "DR case status")
  74. self.tl_notify_party = templates.get("notifyParty", "DRN-notice")
  75. self.tl_notify_stale = templates.get("notifyStale", "DRN stale notice")
  76. self.tl_archive_top = templates.get("archiveTop", "DRN archive top")
  77. self.tl_archive_bottom = templates.get("archiveBottom",
  78. "DRN archive bottom")
  79. self.tl_chart_header = templates.get("chartHeader",
  80. "DRN case status/header")
  81. self.tl_chart_row = templates.get("chartRow", "DRN case status/row")
  82. self.tl_chart_footer = templates.get("chartFooter",
  83. "DRN case status/footer")
  84. # Connection data for our SQL database:
  85. kwargs = cfg.get("sql", {})
  86. kwargs["read_default_file"] = expanduser("~/.my.cnf")
  87. self.conn_data = kwargs
  88. self.db_access_lock = RLock()
  89. # Minimum size a MySQL TIMESTAMP field can hold:
  90. self.min_ts = datetime(1970, 1, 1, 0, 0, 1)
  91. def run(self, **kwargs):
  92. """Entry point for a task event."""
  93. if not self.db_access_lock.acquire(False): # Non-blocking
  94. self.logger.info("A job is already ongoing; aborting")
  95. return
  96. action = kwargs.get("action", "all")
  97. try:
  98. start = time()
  99. conn = oursql.connect(**self.conn_data)
  100. site = self.bot.wiki.get_site()
  101. if action in ["all", "update_volunteers"]:
  102. self.update_volunteers(conn, site)
  103. if action in ["all", "clerk"]:
  104. volunteers = self.get_volunteers(conn)
  105. log = u"Reading cases from [[{0}]]".format(self.title)
  106. self.logger.info(log)
  107. page = site.get_page(self.title)
  108. cases = self.read_database(conn)
  109. self.read_cases(conn, site, cases, page)
  110. if self.shutoff_enabled():
  111. return
  112. notices = []
  113. for case in cases:
  114. notices += self.clerk(conn, volunteers, case)
  115. if not self.save(case, kwargs, start):
  116. return
  117. self.send_notices(site, notices)
  118. if action in ["all", "update_chart"]:
  119. if self.shutoff_enabled():
  120. return
  121. self.update_chart(conn, site)
  122. if action in ["all", "purge"]:
  123. self.purge_old_data(conn)
  124. finally:
  125. self.db_access_lock.release()
  126. def update_volunteers(self, conn, site):
  127. """Updates and stores the list of dispute resolution volunteers."""
  128. log = u"Updating volunteer list from [[{0}]]"
  129. self.logger.info(log.format(self.volunteer_title))
  130. page = site.get_page(self.volunteer_title)
  131. try:
  132. text = page.get()
  133. except exceptions.PageNotFoundError:
  134. text = ""
  135. marker = "<!-- please don't remove this comment (used by EarwigBot) -->"
  136. if marker not in text:
  137. log = u"The marker ({0}) wasn't found in the volunteer list at [[{1}]]!"
  138. self.logger.error(log.format(marker, page.title))
  139. return
  140. text = text.split(marker)[1]
  141. additions = set()
  142. for line in text.splitlines():
  143. user = re.search("\# \{\{User\|(.+?)\}\}", line)
  144. if user:
  145. uname = user.group(1).replace("_", " ").strip()
  146. additions.add((uname[0].upper() + uname[1:],))
  147. removals = set()
  148. query1 = "SELECT volunteer_username FROM volunteers"
  149. query2 = "DELETE FROM volunteers WHERE volunteer_username = ?"
  150. query3 = "INSERT INTO volunteers (volunteer_username) VALUES (?)"
  151. with conn.cursor() as cursor:
  152. cursor.execute(query1)
  153. for row in cursor:
  154. if row in additions:
  155. additions.remove(row)
  156. else:
  157. removals.add(row)
  158. if removals:
  159. cursor.executemany(query2, removals)
  160. if additions:
  161. cursor.executemany(query3, additions)
  162. def get_volunteers(self, conn):
  163. """Return a list of all DRN volunteers."""
  164. query = "SELECT volunteer_username FROM volunteers"
  165. with conn.cursor() as cursor:
  166. cursor.execute(query)
  167. volunteers = [name for (name,) in cursor.fetchall()]
  168. return volunteers
  169. def read_database(self, conn):
  170. """Return a list of _Cases from the database."""
  171. cases = []
  172. query = "SELECT * FROM cases"
  173. with conn.cursor() as cursor:
  174. cursor.execute(query)
  175. for row in cursor:
  176. case = _Case(*row)
  177. cases.append(case)
  178. log = "Read {0} cases from the database"
  179. self.logger.debug(log.format(len(cases)))
  180. return cases
  181. def read_cases(self, conn, site, cases, page):
  182. """Read the noticeboard content and update the list of _Cases."""
  183. text = page.get()
  184. code = mwparserfromhell.parse(text)
  185. nextid = self.select_next_id(conn)
  186. tl_status_esc = re.escape(self.tl_status)
  187. for template in code.filter_templates(recursive=True):
  188. title = template.name.lower().strip()
  189. if not title.startswith(self.title + "/"):
  190. continue
  191. if title == self.title + "/Header":
  192. continue
  193. subpage = site.get_page(title)
  194. body = old = subpage.get()
  195. casename = subpage.title.split("/", 1)[1]
  196. if not re.search(r"\s*\{\{" + tl_status_esc, body, re.U):
  197. continue
  198. status = self.read_status(body)
  199. re_id = r"<!-- Bot Case ID \(please don't modify\): (.*?) -->"
  200. try:
  201. id_ = int(re.search(re_id, body).group(1))
  202. case = [case for case in cases if case.id == id_][0]
  203. except (AttributeError, IndexError, ValueError):
  204. id_ = nextid
  205. nextid += 1
  206. re_id2 = "(\{\{" + tl_status_esc
  207. re_id2 += r"(.*?)\}\})(<!-- Bot Case ID \(please don't modify\): .*? -->)?"
  208. repl = ur"\1 <!-- Bot Case ID (please don't modify): {0} -->"
  209. body = re.sub(re_id2, repl.format(id_), body)
  210. re_f = r"\{\{drn filing editor\|(.*?)\|"
  211. re_f += r"(\d{2}:\d{2},\s\d{1,2}\s\w+\s\d{4}\s\(UTC\))\}\}"
  212. match = re.search(re_f, body, re.U)
  213. if match:
  214. f_user = match.group(1).split("/", 1)[0].replace("_", " ")
  215. f_user = f_user[0].upper() + f_user[1:]
  216. strp = "%H:%M, %d %B %Y (UTC)"
  217. f_time = datetime.strptime(match.group(2), strp)
  218. else:
  219. f_user, f_time = None, datetime.utcnow()
  220. case = _Case(id_, casename, status, self.STATUS_UNKNOWN,
  221. f_user, f_time, f_user, f_time, "", self.min_ts,
  222. self.min_ts, False, False, False, len(body),
  223. subpage, new=True)
  224. cases.append(case)
  225. log = u"Added new case {0} ('{1}', status={2}, by {3})"
  226. self.logger.debug(log.format(id_, casename, status, f_user))
  227. else:
  228. case.status = status
  229. log = u"Read active case {0} ('{1}')".format(id_, casename)
  230. self.logger.debug(log)
  231. if case.title != casename:
  232. self.update_case_title(conn, id_, casename)
  233. case.title = casename
  234. case.body, case.old = body, old
  235. for case in cases[:]:
  236. if case.body is None:
  237. if case.original_status == self.STATUS_UNKNOWN:
  238. cases.remove(case) # Ignore archived case
  239. else:
  240. case.status = self.STATUS_UNKNOWN
  241. log = u"Dropped case {0} because it is no longer on the page ('{1}')"
  242. self.logger.debug(log.format(case.id, case.title))
  243. self.logger.debug("Done reading cases from the noticeboard page")
  244. def select_next_id(self, conn):
  245. """Return the next incremental ID for a case."""
  246. query = "SELECT MAX(case_id) FROM cases"
  247. with conn.cursor() as cursor:
  248. cursor.execute(query)
  249. current = cursor.fetchone()[0]
  250. if current:
  251. return int(current) + 1
  252. return 1
  253. def read_status(self, body):
  254. """Parse the current status from a case body."""
  255. templ = re.escape(self.tl_status)
  256. status = re.search("\{\{" + templ + "\|?(.*?)\}\}", body, re.S|re.U)
  257. if not status:
  258. return self.STATUS_NEW
  259. for option, names in self.ALIASES.iteritems():
  260. if status.group(1).lower() in names:
  261. return option
  262. return self.STATUS_NEW
  263. def update_case_title(self, conn, id_, title):
  264. """Update a case title in the database."""
  265. query = "UPDATE cases SET case_title = ? WHERE case_id = ?"
  266. with conn.cursor() as cursor:
  267. cursor.execute(query, (title, id_))
  268. log = u"Updated title of case {0} to '{1}'".format(id_, title)
  269. self.logger.debug(log)
  270. def clerk(self, conn, volunteers, case):
  271. """Actually go through a case and modify it if it is to be updated.
  272. Return a list of any notices to send.
  273. """
  274. notices = []
  275. log = u"Clerking case {0} ('{1}')".format(case.id, case.title)
  276. self.logger.debug(log)
  277. if case.status == self.STATUS_UNKNOWN:
  278. self.save_existing_case(conn, case)
  279. else:
  280. notices += self.clerk_case(conn, case, volunteers)
  281. self.logger.debug("Done clerking case")
  282. return notices
  283. def clerk_case(self, conn, case, volunteers):
  284. """Clerk a particular case and return a list of any notices to send."""
  285. notices = []
  286. signatures = self.read_signatures(case.body)
  287. storedsigs = self.get_signatures_from_db(conn, case)
  288. newsigs = set(signatures) - set(storedsigs)
  289. if any([editor in volunteers for (editor, timestamp) in newsigs]):
  290. case.last_volunteer_size = len(case.body)
  291. if case.status == self.STATUS_NEW:
  292. notices = self.clerk_new_case(case, volunteers, signatures)
  293. elif case.status == self.STATUS_OPEN:
  294. notices = self.clerk_open_case(case, signatures)
  295. elif case.status == self.STATUS_NEEDASSIST:
  296. notices = self.clerk_needassist_case(case, volunteers, newsigs)
  297. elif case.status == self.STATUS_STALE:
  298. notices = self.clerk_stale_case(case, newsigs)
  299. elif case.status in [self.STATUS_RESOLVED, self.STATUS_CLOSED,
  300. self.STATUS_FAILED]:
  301. self.clerk_closed_case(case, signatures)
  302. self.save_case_updates(conn, case, volunteers, signatures, storedsigs)
  303. return notices
  304. def clerk_new_case(self, case, volunteers, signatures):
  305. """Clerk a case in the "brand new" state.
  306. The case will be set to "open" if a volunteer edits it, or "needassist"
  307. if it increases by over 15,000 bytes or goes by without any volunteer
  308. edits for two days.
  309. """
  310. notices = self.notify_parties(case)
  311. if any([editor in volunteers for (editor, timestamp) in signatures]):
  312. self.update_status(case, self.STATUS_OPEN)
  313. else:
  314. age = (datetime.utcnow() - case.file_time).total_seconds()
  315. if age > 60 * 60 * 24 * 2:
  316. self.update_status(case, self.STATUS_NEEDASSIST)
  317. elif len(case.body) - case.last_volunteer_size > 15000:
  318. self.update_status(case, self.STATUS_NEEDASSIST)
  319. return notices
  320. def clerk_open_case(self, case, signatures):
  321. """Clerk an open case (has been edited by a reviewer).
  322. The case will be set to "needassist" if 15,000 bytes have been added
  323. since a volunteer last edited or if it has been open for over seven
  324. days, or "stale" if no edits at all have occurred in two days.
  325. """
  326. if self.check_for_needassist(case):
  327. return []
  328. if len(case.body) - case.last_volunteer_size > 15000:
  329. self.update_status(case, self.STATUS_NEEDASSIST)
  330. timestamps = [timestamp for (editor, timestamp) in signatures]
  331. if timestamps:
  332. age = (datetime.utcnow() - max(timestamps)).total_seconds()
  333. if age > 60 * 60 * 24 * 2:
  334. self.update_status(case, self.STATUS_STALE)
  335. return []
  336. def clerk_needassist_case(self, case, volunteers, newsigs):
  337. """Clerk a "needassist" case (no volunteers in 15kb or >= 7 days old).
  338. The case will be set to "open" if a volunteer edits and the case is
  339. less than a week old. A message will be set to the "very old notifiee",
  340. which is generally [[User talk:Szhang (WMF)]], if the case has been
  341. open for more than ten days.
  342. """
  343. age = (datetime.utcnow() - case.file_time).total_seconds()
  344. if age <= 60 * 60 * 24 * 7:
  345. if any([editor in volunteers for (editor, timestamp) in newsigs]):
  346. self.update_status(case, self.STATUS_OPEN)
  347. elif age > 60 * 60 * 24 * 10:
  348. if not case.very_old_notified and self.notify_stale_cases:
  349. tmpl = self.tl_notify_stale
  350. title = case.title.replace("|", "&#124;")
  351. template = "{{subst:" + tmpl + "|" + title + "}}"
  352. miss = "<!-- Template:DRN stale notice | {0} -->".format(title)
  353. too_late = lambda text: miss in text
  354. notice = _Notice(self.very_old_title, template, too_late)
  355. case.very_old_notified = True
  356. msg = u" {0}: will notify [[{1}]] with '{2}'"
  357. log = msg.format(case.id, self.very_old_title, template)
  358. self.logger.debug(log)
  359. return [notice]
  360. return []
  361. def clerk_stale_case(self, case, newsigs):
  362. """Clerk a stale case (no edits in two days).
  363. The case will be set to "open" if anyone edits, or "needassist" if it
  364. has been open for over seven days.
  365. """
  366. if self.check_for_needassist(case):
  367. return []
  368. if newsigs:
  369. self.update_status(case, self.STATUS_OPEN)
  370. return []
  371. def clerk_closed_case(self, case, signatures):
  372. """Clerk a closed or resolved case.
  373. The case will be archived if it has been closed/resolved for more than
  374. one day and no edits have been made in the meantime. "Archiving" is
  375. the process of adding {{DRN archive top}}, {{DRN archive bottom}}, and
  376. removing the [[User:DoNotArchiveUntil]] comment.
  377. """
  378. if case.close_time == self.min_ts:
  379. case.close_time = datetime.utcnow()
  380. if case.archived:
  381. return
  382. timestamps = [timestamp for (editor, timestamp) in signatures]
  383. closed_age = (datetime.utcnow() - case.close_time).total_seconds()
  384. if timestamps:
  385. modify_age = (datetime.utcnow() - max(timestamps)).total_seconds()
  386. else:
  387. modify_age = 0
  388. if closed_age > 60 * 60 * 24 and modify_age > 60 * 60 * 24:
  389. arch_top = self.tl_archive_top
  390. arch_bottom = self.tl_archive_bottom
  391. reg = r"<!-- \[\[User:DoNotArchiveUntil\]\] .*? -->(<!-- .*? -->)?"
  392. if re.search(r"\{\{\s*" + arch_top, case.body):
  393. if re.search(reg, case.body):
  394. case.body = re.sub(reg, "", case.body)
  395. else:
  396. if re.search(reg, case.body):
  397. case.body = re.sub(r"\{\{" + arch_top + r"\}\}", "", case.body)
  398. case.body = re.sub(reg, "{{" + arch_top + "}}", case.body)
  399. if not re.search(arch_bottom + r"\s*\}\}\s*\Z", case.body):
  400. case.body += "\n{{" + arch_bottom + "}}"
  401. case.archived = True
  402. self.logger.debug(u" {0}: archived case".format(case.id))
  403. def check_for_needassist(self, case):
  404. """Check whether a case is old enough to be set to "needassist"."""
  405. age = (datetime.utcnow() - case.file_time).total_seconds()
  406. if age > 60 * 60 * 24 * 7:
  407. self.update_status(case, self.STATUS_NEEDASSIST)
  408. return True
  409. return False
  410. def update_status(self, case, new):
  411. """Safely update the status of a case, so we don't edit war."""
  412. old_n = self.ALIASES[case.status][0].upper()
  413. new_n = self.ALIASES[new][0].upper()
  414. old_n = "NEW" if not old_n else old_n
  415. new_n = "NEW" if not new_n else new_n
  416. if case.last_action != new:
  417. case.status = new
  418. log = u" {0}: {1} -> {2}"
  419. self.logger.debug(log.format(case.id, old_n, new_n))
  420. return
  421. log = u"Avoiding {0} {1} -> {2} because we already did this ('{3}')"
  422. self.logger.info(log.format(case.id, old_n, new_n, case.title))
  423. def read_signatures(self, text):
  424. """Return a list of all parseable signatures in the body of a case.
  425. Signatures are returned as tuples of (editor, timestamp as datetime).
  426. """
  427. regex = r"\[\[(?:User(?:\stalk)?\:|Special\:Contributions\/)"
  428. regex += r"([^\n\[\]|]{,256}?)(?:\||\]\])"
  429. regex += r"(?!.*?(?:User(?:\stalk)?\:|Special\:Contributions\/).*?)"
  430. regex += r".{,256}?(\d{2}:\d{2},\s\d{1,2}\s\w+\s\d{4}\s\(UTC\))"
  431. matches = re.findall(regex, text, re.U|re.I)
  432. signatures = []
  433. for userlink, stamp in matches:
  434. username = userlink.split("/", 1)[0].replace("_", " ").strip()
  435. username = username[0].upper() + username[1:]
  436. if username == "DoNotArchiveUntil":
  437. continue
  438. stamp = stamp.strip()
  439. timestamp = datetime.strptime(stamp, "%H:%M, %d %B %Y (UTC)")
  440. signatures.append((username, timestamp))
  441. return signatures
  442. def get_signatures_from_db(self, conn, case):
  443. """Return a list of signatures in a case from the database.
  444. The return type is the same as read_signatures().
  445. """
  446. query = "SELECT signature_username, signature_timestamp FROM signatures WHERE signature_case = ?"
  447. with conn.cursor() as cursor:
  448. cursor.execute(query, (case.id,))
  449. return cursor.fetchall()
  450. def notify_parties(self, case):
  451. """Schedule notices to be sent to all parties of a case."""
  452. if case.parties_notified:
  453. return []
  454. def too_late(text):
  455. code = mwparserfromhell.parse(text)
  456. for link in code.filter_links(recursive=True):
  457. title = link.title.strip_code().lower().strip()
  458. title = re.sub(r"^wp:", "wikipedia:", title.replace("_", " "))
  459. if title == case.page.title.lower():
  460. return True
  461. return False
  462. notices = []
  463. template = "{{subst:" + self.tl_notify_party
  464. template += "|thread=" + case.title + "}} ~~~~"
  465. re_parties = r"<span.*?>'''Users involved'''</span>(.*?)<span.*?>"
  466. text = re.search(re_parties, case.body, re.S|re.U)
  467. for line in text.group(1).splitlines():
  468. user = re.search("[:*#]{,5} \{\{User\|(.*?)\}\}", line)
  469. if user:
  470. party = user.group(1).replace("_", " ").strip()
  471. if party:
  472. party = party[0].upper() + party[1:]
  473. if party == case.file_user:
  474. continue
  475. notice = _Notice("User talk:" + party, template, too_late)
  476. notices.append(notice)
  477. case.parties_notified = True
  478. log = u" {0}: will try to notify {1} parties with '{2}'"
  479. self.logger.debug(log.format(case.id, len(notices), template))
  480. return notices
  481. def save_case_updates(self, conn, case, volunteers, sigs, storedsigs):
  482. """Save any updates made to a case and signatures in the database."""
  483. if case.status != case.original_status:
  484. case.last_action = case.status
  485. new = self.ALIASES[case.status][0]
  486. tl_status_esc = re.escape(self.tl_status)
  487. search = "\{\{" + tl_status_esc + "(\|?.*?)\}\}"
  488. repl = "{{" + self.tl_status + "|" + new + "}}"
  489. case.body = re.sub(search, repl, case.body)
  490. if sigs:
  491. newest_ts = max([stamp for (user, stamp) in sigs])
  492. newest_user = [usr for (usr, stamp) in sigs if stamp == newest_ts][0]
  493. case.modify_time = newest_ts
  494. case.modify_user = newest_user
  495. if any([usr in volunteers for (usr, stamp) in sigs]):
  496. newest_vts = max([stamp for (usr, stamp) in sigs if usr in volunteers])
  497. newest_vuser = [usr for (usr, stamp) in sigs if stamp == newest_vts][0]
  498. case.volunteer_time = newest_vts
  499. case.volunteer_user = newest_vuser
  500. if case.new:
  501. self.save_new_case(conn, case)
  502. else:
  503. self.save_existing_case(conn, case)
  504. with conn.cursor() as cursor:
  505. query1 = "DELETE FROM signatures WHERE signature_case = ? AND signature_username = ? AND signature_timestamp = ?"
  506. query2 = "INSERT INTO signatures (signature_case, signature_username, signature_timestamp) VALUES (?, ?, ?)"
  507. removals = set(storedsigs) - set(sigs)
  508. additions = set(sigs) - set(storedsigs)
  509. if removals:
  510. args = [(case.id, name, stamp) for (name, stamp) in removals]
  511. cursor.executemany(query1, args)
  512. if additions:
  513. args = []
  514. for name, stamp in additions:
  515. args.append((case.id, name, stamp))
  516. cursor.executemany(query2, args)
  517. msg = u" {0}: added {1} signatures and removed {2}"
  518. log = msg.format(case.id, len(additions), len(removals))
  519. self.logger.debug(log)
  520. def save_new_case(self, conn, case):
  521. """Save a brand new case to the database."""
  522. args = (case.id, case.title, case.status, case.last_action,
  523. case.file_user, case.file_time, case.modify_user,
  524. case.modify_time, case.volunteer_user, case.volunteer_time,
  525. case.close_time, case.parties_notified,
  526. case.very_old_notified, case.archived,
  527. case.last_volunteer_size)
  528. with conn.cursor() as cursor:
  529. query = "INSERT INTO cases VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"
  530. cursor.execute(query, args)
  531. log = u" {0}: inserted new case into database".format(case.id)
  532. self.logger.debug(log)
  533. def save_existing_case(self, conn, case):
  534. """Save an existing case to the database, updating as necessary."""
  535. with conn.cursor(oursql.DictCursor) as cursor:
  536. query = "SELECT * FROM cases WHERE case_id = ?"
  537. cursor.execute(query, (case.id,))
  538. stored = cursor.fetchone()
  539. with conn.cursor() as cursor:
  540. changes, args = [], []
  541. fields_to_check = [
  542. ("case_status", case.status),
  543. ("case_last_action", case.last_action),
  544. ("case_file_user", case.file_user),
  545. ("case_file_time", case.file_time),
  546. ("case_modify_user", case.modify_user),
  547. ("case_modify_time", case.modify_time),
  548. ("case_volunteer_user", case.volunteer_user),
  549. ("case_volunteer_time", case.volunteer_time),
  550. ("case_close_time", case.close_time),
  551. ("case_parties_notified", case.parties_notified),
  552. ("case_very_old_notified", case.very_old_notified),
  553. ("case_archived", case.archived),
  554. ("case_last_volunteer_size", case.last_volunteer_size)
  555. ]
  556. for column, data in fields_to_check:
  557. if data != stored[column]:
  558. changes.append(column + " = ?")
  559. args.append(data)
  560. msg = u" {0}: will alter {1} ('{2}' -> '{3}')"
  561. log = msg.format(case.id, column, stored[column], data)
  562. self.logger.debug(log)
  563. if changes:
  564. changes = ", ".join(changes)
  565. args.append(case.id)
  566. query = "UPDATE cases SET {0} WHERE case_id = ?".format(changes)
  567. cursor.execute(query, args)
  568. else:
  569. log = u" {0}: no changes to commit".format(case.id)
  570. self.logger.debug(log)
  571. def save(self, case, kwargs, start):
  572. """Save any changes to a specific case subpage."""
  573. page, text, newtext = case.page, case.old, case.body
  574. if text == newtext:
  575. self.logger.debug(u"Nothing to edit on [[{0}]]".format(page.title))
  576. return True
  577. worktime = time() - start
  578. if worktime < 60:
  579. log = "Waiting {0} seconds to avoid edit conflicts"
  580. self.logger.debug(log.format(int(60 - worktime)))
  581. sleep(60 - worktime)
  582. page.reload()
  583. if page.get() != text:
  584. log = "Someone has edited the page while we were working; restarting"
  585. self.logger.warn(log)
  586. self.run(**kwargs)
  587. return False
  588. page.edit(newtext, self.clerk_summary, minor=True, bot=True)
  589. log = u"Saved page [[{0}]]"
  590. self.logger.info(log.format(page.title))
  591. return True
  592. def send_notices(self, site, notices):
  593. """Send out any templated notices to users or pages."""
  594. if not notices:
  595. self.logger.info("No notices to send")
  596. return
  597. for notice in notices:
  598. target, template = notice.target, notice.template
  599. log = u"Trying to notify [[{0}]] with '{1}'"
  600. self.logger.debug(log.format(target, template))
  601. page = site.get_page(target)
  602. if page.namespace == constants.NS_USER_TALK:
  603. user = site.get_user(target.split(":", 1)[1])
  604. if not user.exists and not user.is_ip:
  605. log = u"Skipping [[{0}]]; user does not exist and is not an IP"
  606. self.logger.info(log.format(target))
  607. continue
  608. try:
  609. text = page.get()
  610. except exceptions.PageNotFoundError:
  611. text = ""
  612. if notice.too_late(text):
  613. log = u"Skipping [[{0}]]; was already notified with '{1}'"
  614. self.logger.info(log.format(page.title, template))
  615. continue
  616. text += ("\n" if text else "") + template
  617. try:
  618. page.edit(text, self.notify_summary, minor=False, bot=True)
  619. except exceptions.EditError as error:
  620. name, msg = type(error).name, error.message
  621. log = u"Couldn't leave notice on [[{0}]] because of {1}: {2}"
  622. self.logger.error(log.format(page.title, name, msg))
  623. else:
  624. log = u"Notified [[{0}]] with '{1}'"
  625. self.logger.info(log.format(page.title, template))
  626. self.logger.debug("Done sending notices")
  627. def update_chart(self, conn, site):
  628. """Update the chart of open or recently closed cases."""
  629. page = site.get_page(self.chart_title)
  630. self.logger.info(u"Updating case status at [[{0}]]".format(page.title))
  631. statuses = self.compile_chart(conn)
  632. text = page.get()
  633. newtext = re.sub(u"<!-- status begin -->(.*?)<!-- status end -->",
  634. "<!-- status begin -->\n" + statuses + "\n<!-- status end -->",
  635. text, flags=re.DOTALL)
  636. if newtext == text:
  637. self.logger.info("Chart unchanged; not saving")
  638. return
  639. newtext = re.sub("<!-- sig begin -->(.*?)<!-- sig end -->",
  640. "<!-- sig begin -->~~~ at ~~~~~<!-- sig end -->",
  641. newtext)
  642. page.edit(newtext, self.chart_summary, minor=True, bot=True)
  643. self.logger.info(u"Chart saved to [[{0}]]".format(page.title))
  644. def compile_chart(self, conn):
  645. """Actually generate the chart from the database."""
  646. chart = "{{" + self.tl_chart_header + "|small={{{small|}}}|collapsed={{{collapsed|}}}}}\n"
  647. query = "SELECT * FROM cases WHERE case_status != ?"
  648. with conn.cursor(oursql.DictCursor) as cursor:
  649. cursor.execute(query, (self.STATUS_UNKNOWN,))
  650. for case in cursor:
  651. chart += self.compile_row(case)
  652. chart += "{{" + self.tl_chart_footer + "|small={{{small|}}}}}"
  653. return chart
  654. def compile_row(self, case):
  655. """Generate a single row of the chart from a dict via the database."""
  656. data = u"|t={case_title}|d={title}|s={case_status}"
  657. data += "|cu={case_file_user}|cs={file_sortkey}|ct={file_time}"
  658. if case["case_volunteer_user"]:
  659. data += "|vu={case_volunteer_user}|vs={volunteer_sortkey}|vt={volunteer_time}"
  660. case["volunteer_time"] = self.format_time(case["case_volunteer_time"])
  661. case["volunteer_sortkey"] = int(mktime(case["case_volunteer_time"].timetuple()))
  662. data += "|mu={case_modify_user}|ms={modify_sortkey}|mt={modify_time}"
  663. title = case["case_title"].replace("_", " ")
  664. case["title"] = title[:47] + "..." if len(title) > 50 else title
  665. case["file_time"] = self.format_time(case["case_file_time"])
  666. case["file_sortkey"] = int(mktime(case["case_file_time"].timetuple()))
  667. case["modify_time"] = self.format_time(case["case_modify_time"])
  668. case["modify_sortkey"] = int(mktime(case["case_modify_time"].timetuple()))
  669. row = "{{" + self.tl_chart_row + data.format(**case)
  670. return row + "|sm={{{small|}}}}}\n"
  671. def format_time(self, dt):
  672. """Return a string telling the time since datetime occurred."""
  673. parts = [("year", 31536000), ("day", 86400), ("hour", 3600)]
  674. seconds = int((datetime.utcnow() - dt).total_seconds())
  675. msg = []
  676. for name, size in parts:
  677. num = seconds // size
  678. seconds -= num * size
  679. if num:
  680. chunk = "{0} {1}".format(num, name if num == 1 else name + "s")
  681. msg.append(chunk)
  682. return ", ".join(msg) + " ago" if msg else "0 hours ago"
  683. def purge_old_data(self, conn):
  684. """Delete old cases (> six months) from the database."""
  685. log = "Purging closed cases older than six months from the database"
  686. self.logger.info(log)
  687. query = """DELETE cases, signatures
  688. FROM cases JOIN signatures ON case_id = signature_case
  689. WHERE case_status = ?
  690. AND case_file_time < DATE_SUB(CURRENT_TIMESTAMP, INTERVAL 180 DAY)
  691. AND case_modify_time < DATE_SUB(CURRENT_TIMESTAMP, INTERVAL 180 DAY)
  692. """
  693. with conn.cursor() as cursor:
  694. cursor.execute(query, (self.STATUS_UNKNOWN,))
  695. class _Case(object):
  696. """A object representing a dispute resolution case."""
  697. def __init__(self, id_, title, status, last_action, file_user, file_time,
  698. modify_user, modify_time, volunteer_user, volunteer_time,
  699. close_time, parties_notified, archived, very_old_notified,
  700. last_volunteer_size, page=None, new=False):
  701. self.id = id_
  702. self.title = title
  703. self.status = status
  704. self.last_action = last_action
  705. self.file_user = file_user
  706. self.file_time = file_time
  707. self.modify_user = modify_user
  708. self.modify_time = modify_time
  709. self.volunteer_user = volunteer_user
  710. self.volunteer_time = volunteer_time
  711. self.close_time = close_time
  712. self.parties_notified = parties_notified
  713. self.very_old_notified = very_old_notified
  714. self.archived = archived
  715. self.last_volunteer_size = last_volunteer_size
  716. self.page = page
  717. self.new = new
  718. self.original_status = status
  719. self.body = None
  720. self.old = None
  721. class _Notice(object):
  722. """An object representing a notice to be sent to a user or a page."""
  723. def __init__(self, target, template, too_late=None):
  724. self.target = target
  725. self.template = template
  726. self.too_late = too_late