Additional IRC commands and bot tasks for EarwigBot https://en.wikipedia.org/wiki/User:EarwigBot
Du kan inte välja fler än 25 ämnen Ämnen måste starta med en bokstav eller siffra, kan innehålla bindestreck ('-') och vara max 35 tecken långa.

850 rader
35 KiB

  1. # Copyright (C) 2009-2019 Ben Kurtovic <ben.kurtovic@gmail.com>
  2. #
  3. # Permission is hereby granted, free of charge, to any person obtaining a copy
  4. # of this software and associated documentation files (the "Software"), to deal
  5. # in the Software without restriction, including without limitation the rights
  6. # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
  7. # copies of the Software, and to permit persons to whom the Software is
  8. # furnished to do so, subject to the following conditions:
  9. #
  10. # The above copyright notice and this permission notice shall be included in
  11. # all copies or substantial portions of the Software.
  12. #
  13. # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  14. # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  15. # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  16. # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  17. # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
  18. # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  19. # SOFTWARE.
  20. import re
  21. from collections import OrderedDict
  22. from datetime import datetime
  23. from os.path import expanduser
  24. from threading import Lock
  25. from time import sleep
  26. import mwparserfromhell
  27. import oursql
  28. from earwigbot import exceptions, wiki
  29. from earwigbot.tasks import Task
  30. _DEFAULT_PAGE_TEXT = """<noinclude><!-- You can edit anything on this page \
  31. except for content inside of <!-- stat begin/end -> and <!-- sig begin/end -> \
  32. without causing problems. Most of the chart can be modified by editing the \
  33. templates it uses, documented in [[Template:AfC statistics/doc]]. -->
  34. {{NOINDEX}}</noinclude>\
  35. <!-- stat begin --><!-- stat end -->
  36. <span style="font-style: italic; font-size: 85%%;">Last updated by \
  37. <!-- sig begin --><!-- sig end --></span>\
  38. <noinclude>{{Documentation|Template:%(pageroot)s/doc}}</noinclude>
  39. """
  40. _PER_CHART_LIMIT = 1000
  41. class AfCStatistics(Task):
  42. """A task to generate statistics for WikiProject Articles for Creation.
  43. Statistics are stored in a MySQL database ("u_earwig_afc_statistics")
  44. accessed with oursql. Statistics are synchronied with the live database
  45. every four minutes and saved once an hour, on the hour, to subpages of
  46. self.pageroot. In the live bot, this is "Template:AfC statistics".
  47. """
  48. name = "afc_statistics"
  49. number = 2
  50. # Chart status number constants:
  51. CHART_NONE = 0
  52. CHART_PEND = 1
  53. CHART_REVIEW = 3
  54. CHART_ACCEPT = 4
  55. CHART_DECLINE = 5
  56. CHART_MISPLACE = 6
  57. def setup(self):
  58. self.cfg = cfg = self.config.tasks.get(self.name, {})
  59. self.site = self.bot.wiki.get_site()
  60. self.revision_cache = {}
  61. # Set some wiki-related attributes:
  62. self.pageroot = cfg.get("page", "Template:AfC statistics")
  63. self.pending_cat = cfg.get("pending", "Pending AfC submissions")
  64. self.ignore_list = cfg.get("ignoreList", [])
  65. default_summary = (
  66. "Updating statistics for [[WP:WPAFC|WikiProject Articles for creation]]."
  67. )
  68. self.summary = self.make_summary(cfg.get("summary", default_summary))
  69. # Templates used in chart generation:
  70. templates = cfg.get("templates", {})
  71. self.tl_header = templates.get("header", "AfC statistics/header")
  72. self.tl_row = templates.get("row", "#invoke:AfC|row")
  73. self.tl_footer = templates.get("footer", "AfC statistics/footer")
  74. # Connection data for our SQL database:
  75. kwargs = cfg.get("sql", {})
  76. kwargs["read_default_file"] = expanduser("~/.my.cnf")
  77. self.conn_data = kwargs
  78. self.db_access_lock = Lock()
  79. def run(self, **kwargs):
  80. """Entry point for a task event.
  81. Depending on the kwargs passed, we will either synchronize our local
  82. statistics database with the site (self.sync()) or save it to the wiki
  83. (self.save()). We will additionally create an SQL connection with our
  84. local database.
  85. """
  86. action = kwargs.get("action")
  87. if not self.db_access_lock.acquire(False): # Non-blocking
  88. if action == "sync":
  89. self.logger.info("A sync is already ongoing; aborting")
  90. return
  91. self.logger.info("Waiting for database access lock")
  92. self.db_access_lock.acquire()
  93. try:
  94. self.site = self.bot.wiki.get_site()
  95. self.conn = oursql.connect(**self.conn_data)
  96. self.revision_cache = {}
  97. try:
  98. if action == "save":
  99. self.save(kwargs)
  100. elif action == "sync":
  101. self.sync(kwargs)
  102. finally:
  103. self.conn.close()
  104. finally:
  105. self.db_access_lock.release()
  106. #################### CHART BUILDING AND SAVING METHODS ####################
  107. def save(self, kwargs):
  108. """Save our local statistics to the wiki.
  109. After checking for emergency shutoff, the statistics chart is compiled,
  110. and then saved to subpages of self.pageroot using self.summary iff it
  111. has changed since last save.
  112. """
  113. self.logger.info("Saving chart")
  114. if kwargs.get("fromIRC"):
  115. summary = self.summary + " (!earwigbot)"
  116. else:
  117. if self.shutoff_enabled():
  118. return
  119. summary = self.summary
  120. statistics = self._compile_charts()
  121. for name, chart in statistics.iteritems():
  122. self._save_page(name, chart, summary)
  123. def _save_page(self, name, chart, summary):
  124. """Save a statistics chart to a single page."""
  125. page = self.site.get_page(f"{self.pageroot}/{name}")
  126. try:
  127. text = page.get()
  128. except exceptions.PageNotFoundError:
  129. text = _DEFAULT_PAGE_TEXT % {"pageroot": self.pageroot}
  130. newtext = re.sub(
  131. "<!-- stat begin -->(.*?)<!-- stat end -->",
  132. "<!-- stat begin -->" + chart + "<!-- stat end -->",
  133. text,
  134. flags=re.DOTALL,
  135. )
  136. if newtext == text:
  137. self.logger.info(f"Chart for {name} unchanged; not saving")
  138. return
  139. newtext = re.sub(
  140. "<!-- sig begin -->(.*?)<!-- sig end -->",
  141. "<!-- sig begin -->~~~ at ~~~~~<!-- sig end -->",
  142. newtext,
  143. )
  144. page.edit(newtext, summary, minor=True, bot=True)
  145. self.logger.info(f"Chart for {name} saved to [[{page.title}]]")
  146. def _compile_charts(self):
  147. """Compile and return all statistics information from our local db."""
  148. stats = OrderedDict()
  149. with self.conn.cursor(oursql.DictCursor) as cursor:
  150. cursor.execute("SELECT * FROM chart")
  151. for chart in cursor:
  152. name = chart["chart_name"]
  153. stats[name] = self._compile_chart(chart)
  154. return stats
  155. def _compile_chart(self, chart_info):
  156. """Compile and return a single statistics chart."""
  157. chart = self.tl_header + "|" + chart_info["chart_title"]
  158. if chart_info["chart_special_title"]:
  159. chart += "|" + chart_info["chart_special_title"]
  160. chart = "{{" + chart + "}}"
  161. query = "SELECT * FROM page JOIN row ON page_id = row_id WHERE row_chart = ?"
  162. with self.conn.cursor(oursql.DictCursor) as cursor:
  163. cursor.execute(query, (chart_info["chart_id"],))
  164. rows = cursor.fetchall()
  165. skipped = max(0, len(rows) - _PER_CHART_LIMIT)
  166. rows = rows[:_PER_CHART_LIMIT]
  167. for page in rows:
  168. chart += "\n" + self._compile_chart_row(page)
  169. footer = "{{" + self.tl_footer
  170. if skipped:
  171. footer += f"|skip={skipped}"
  172. footer += "}}"
  173. chart += "\n" + footer + "\n"
  174. return chart
  175. def _compile_chart_row(self, page):
  176. """Compile and return a single chart row.
  177. 'page' is a dict of page information, taken as a row from the page
  178. table, where keys are column names and values are their cell contents.
  179. """
  180. row = "{0}|s={page_status}|t={page_title}|z={page_size}|"
  181. if page["page_special_oldid"]:
  182. row += (
  183. "sr={page_special_user}|sd={page_special_time}|si={page_special_oldid}|"
  184. )
  185. row += "mr={page_modify_user}|md={page_modify_time}|mi={page_modify_oldid}"
  186. page["page_special_time"] = self._fmt_time(page["page_special_time"])
  187. page["page_modify_time"] = self._fmt_time(page["page_modify_time"])
  188. if page["page_notes"]:
  189. row += "|n=1{page_notes}"
  190. return "{{" + row.format(self.tl_row, **page) + "}}"
  191. def _fmt_time(self, date):
  192. """Format a datetime into the standard MediaWiki timestamp format."""
  193. return date.strftime("%H:%M, %d %b %Y")
  194. ######################## PRIMARY SYNC ENTRY POINTS ########################
  195. def sync(self, kwargs):
  196. """Synchronize our local statistics database with the site.
  197. Syncing involves, in order, updating tracked submissions that have
  198. been changed since last sync (self._update_tracked()), adding pending
  199. submissions that are not tracked (self._add_untracked()), and removing
  200. old submissions from the database (self._delete_old()).
  201. The sync will be canceled if SQL replication lag is greater than 600
  202. seconds, because this will lead to potential problems and outdated
  203. data, not to mention putting demand on an already overloaded server.
  204. Giving sync the kwarg "ignore_replag" will go around this restriction.
  205. """
  206. self.logger.info("Starting sync")
  207. replag = self.site.get_replag()
  208. self.logger.debug(f"Server replag is {replag}")
  209. if replag > 600 and not kwargs.get("ignore_replag"):
  210. msg = "Sync canceled as replag ({0} secs) is greater than ten minutes"
  211. self.logger.warn(msg.format(replag))
  212. return
  213. with self.conn.cursor() as cursor:
  214. self._update_tracked(cursor)
  215. self._add_untracked(cursor)
  216. self._update_stale(cursor)
  217. self._delete_old(cursor)
  218. self.logger.info("Sync completed")
  219. def _update_tracked(self, cursor):
  220. """Update tracked submissions that have been changed since last sync.
  221. This is done by iterating through every page in our database and
  222. comparing our stored latest revision ID with the actual latest revision
  223. ID from an SQL query. If they differ, we will update our information
  224. about the page (self._update_page()).
  225. If the page does not exist, we will remove it from our database with
  226. self._untrack_page().
  227. """
  228. self.logger.debug("Updating tracked submissions")
  229. query1 = """SELECT page_id, page_title, page_modify_oldid
  230. FROM page"""
  231. query2 = """SELECT page_latest, page_title, page_namespace
  232. FROM page WHERE page_id = ?"""
  233. cursor.execute(query1)
  234. for pageid, title, oldid in cursor.fetchall():
  235. result = list(self.site.sql_query(query2, (pageid,)))
  236. if not result:
  237. self._untrack_page(cursor, pageid)
  238. continue
  239. real_oldid, real_title, real_ns = result[0]
  240. if oldid == real_oldid:
  241. continue
  242. msg = "Updating page [[{0}]] (id: {1}) @ {2}"
  243. self.logger.debug(msg.format(title, pageid, oldid))
  244. msg = " {0}: oldid: {1} -> {2}"
  245. self.logger.debug(msg.format(pageid, oldid, real_oldid))
  246. real_title = real_title.decode("utf8").replace("_", " ")
  247. ns = self.site.namespace_id_to_name(real_ns)
  248. if ns:
  249. real_title = ":".join((ns, real_title))
  250. try:
  251. self._update_page(cursor, pageid, real_title)
  252. except Exception:
  253. e = "Error updating page [[{0}]] (id: {1})"
  254. self.logger.exception(e.format(real_title, pageid))
  255. def _add_untracked(self, cursor):
  256. """Add pending submissions that are not yet tracked.
  257. This is done by compiling a list of all currently tracked submissions
  258. and iterating through all members of self.pending_cat via SQL. If a
  259. page in the pending category is not tracked and is not in
  260. self.ignore_list, we will track it with self._track_page().
  261. """
  262. self.logger.debug("Adding untracked pending submissions")
  263. query1 = "SELECT page_id FROM page"
  264. query2 = """SELECT page_id, page_title, page_namespace
  265. FROM page
  266. INNER JOIN categorylinks ON page_id = cl_from
  267. WHERE cl_to = ?"""
  268. cursor.execute(query1)
  269. tracked = [pid for (pid,) in cursor.fetchall()]
  270. pend_cat = self.pending_cat.replace(" ", "_")
  271. for pageid, title, ns in self.site.sql_query(query2, (pend_cat,)):
  272. if pageid in tracked:
  273. continue
  274. title = title.decode("utf8").replace("_", " ")
  275. ns_name = self.site.namespace_id_to_name(ns)
  276. if ns_name:
  277. title = ":".join((ns_name, title))
  278. if title in self.ignore_list or ns == wiki.NS_CATEGORY:
  279. continue
  280. msg = f"Tracking page [[{title}]] (id: {pageid})"
  281. self.logger.debug(msg)
  282. try:
  283. self._track_page(cursor, pageid, title)
  284. except Exception:
  285. e = "Error tracking page [[{0}]] (id: {1})"
  286. self.logger.exception(e.format(title, pageid))
  287. def _update_stale(self, cursor):
  288. """Update submissions that haven't been updated in a long time.
  289. This is intended to update notes that change without typical update
  290. triggers, like when submitters are blocked. It also resolves conflicts
  291. when pages are tracked during high replag, potentially causing data to
  292. be inaccurate (like a missed decline). It updates no more than the ten
  293. stalest pages that haven't been updated in two days.
  294. """
  295. self.logger.debug("Updating stale submissions")
  296. query = """SELECT page_id, page_title, page_modify_oldid
  297. FROM page JOIN updatelog ON page_id = update_id
  298. WHERE ADDTIME(update_time, '48:00:00') < NOW()
  299. ORDER BY update_time ASC LIMIT 10"""
  300. cursor.execute(query)
  301. for pageid, title, oldid in cursor:
  302. msg = "Updating page [[{0}]] (id: {1}) @ {2}"
  303. self.logger.debug(msg.format(title, pageid, oldid))
  304. try:
  305. self._update_page(cursor, pageid, title)
  306. except Exception:
  307. e = "Error updating page [[{0}]] (id: {1})"
  308. self.logger.exception(e.format(title, pageid))
  309. def _delete_old(self, cursor):
  310. """Remove old submissions from the database.
  311. "Old" is defined as a submission that has been declined or accepted
  312. more than 36 hours ago. Pending submissions cannot be "old".
  313. """
  314. self.logger.debug("Removing old submissions from chart")
  315. query = """DELETE FROM page, row, updatelog USING page JOIN row
  316. ON page_id = row_id JOIN updatelog ON page_id = update_id
  317. WHERE row_chart IN (?, ?)
  318. AND ADDTIME(page_special_time, '36:00:00') < NOW()"""
  319. cursor.execute(query, (self.CHART_ACCEPT, self.CHART_DECLINE))
  320. ######################## PRIMARY PAGE ENTRY POINTS ########################
  321. def _untrack_page(self, cursor, pageid):
  322. """Remove a page, given by ID, from our database."""
  323. self.logger.debug(f"Untracking page (id: {pageid})")
  324. query = """DELETE FROM page, row, updatelog USING page JOIN row
  325. ON page_id = row_id JOIN updatelog ON page_id = update_id
  326. WHERE page_id = ?"""
  327. cursor.execute(query, (pageid,))
  328. def _track_page(self, cursor, pageid, title):
  329. """Update hook for when page is not in our database.
  330. A variety of SQL queries are used to gather information about the page,
  331. which is then saved to our database.
  332. """
  333. content = self._get_content(pageid)
  334. if content is None:
  335. msg = f"Could not get page content for [[{title}]]"
  336. self.logger.error(msg)
  337. return
  338. namespace = self.site.get_page(title).namespace
  339. status, chart = self._get_status_and_chart(content, namespace)
  340. if chart == self.CHART_NONE:
  341. msg = f"Could not find a status for [[{title}]]"
  342. self.logger.warn(msg)
  343. return
  344. m_user, m_time, m_id = self._get_modify(pageid)
  345. s_user, s_time, s_id = self._get_special(pageid, content, chart)
  346. notes = self._get_notes(chart, content, m_time, s_user)
  347. query1 = "INSERT INTO row VALUES (?, ?)"
  348. query2 = "INSERT INTO page VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"
  349. query3 = "INSERT INTO updatelog VALUES (?, ?)"
  350. cursor.execute(query1, (pageid, chart))
  351. cursor.execute(
  352. query2,
  353. (
  354. pageid,
  355. status,
  356. title,
  357. len(content),
  358. notes,
  359. m_user,
  360. m_time,
  361. m_id,
  362. s_user,
  363. s_time,
  364. s_id,
  365. ),
  366. )
  367. cursor.execute(query3, (pageid, datetime.utcnow()))
  368. def _update_page(self, cursor, pageid, title):
  369. """Update hook for when page is already in our database.
  370. A variety of SQL queries are used to gather information about the page,
  371. which is compared against our stored information. Differing information
  372. is then updated.
  373. """
  374. content = self._get_content(pageid)
  375. if content is None:
  376. msg = f"Could not get page content for [[{title}]]"
  377. self.logger.error(msg)
  378. return
  379. namespace = self.site.get_page(title).namespace
  380. status, chart = self._get_status_and_chart(content, namespace)
  381. if chart == self.CHART_NONE:
  382. self._untrack_page(cursor, pageid)
  383. return
  384. query = "SELECT * FROM page JOIN row ON page_id = row_id WHERE page_id = ?"
  385. with self.conn.cursor(oursql.DictCursor) as dict_cursor:
  386. dict_cursor.execute(query, (pageid,))
  387. result = dict_cursor.fetchall()[0]
  388. m_user, m_time, m_id = self._get_modify(pageid)
  389. if title != result["page_title"]:
  390. self._update_page_title(cursor, result, pageid, title)
  391. if m_id != result["page_modify_oldid"]:
  392. self._update_page_modify(
  393. cursor, result, pageid, len(content), m_user, m_time, m_id
  394. )
  395. if status != result["page_status"]:
  396. special = self._update_page_status(
  397. cursor, result, pageid, content, status, chart
  398. )
  399. s_user = special[0]
  400. else:
  401. s_user = result["page_special_user"]
  402. notes = self._get_notes(chart, content, m_time, s_user)
  403. if notes != result["page_notes"]:
  404. self._update_page_notes(cursor, result, pageid, notes)
  405. query = "UPDATE updatelog SET update_time = ? WHERE update_id = ?"
  406. cursor.execute(query, (datetime.utcnow(), pageid))
  407. ###################### PAGE ATTRIBUTE UPDATE METHODS ######################
  408. def _update_page_title(self, cursor, result, pageid, title):
  409. """Update the title of a page in our database."""
  410. query = "UPDATE page SET page_title = ? WHERE page_id = ?"
  411. cursor.execute(query, (title, pageid))
  412. msg = " {0}: title: {1} -> {2}"
  413. self.logger.debug(msg.format(pageid, result["page_title"], title))
  414. def _update_page_modify(self, cursor, result, pageid, size, m_user, m_time, m_id):
  415. """Update the last modified information of a page in our database."""
  416. query = """UPDATE page SET page_size = ?, page_modify_user = ?,
  417. page_modify_time = ?, page_modify_oldid = ?
  418. WHERE page_id = ?"""
  419. cursor.execute(query, (size, m_user, m_time, m_id, pageid))
  420. msg = " {0}: modify: {1} / {2} / {3} -> {4} / {5} / {6}"
  421. msg = msg.format(
  422. pageid,
  423. result["page_modify_user"],
  424. result["page_modify_time"],
  425. result["page_modify_oldid"],
  426. m_user,
  427. m_time,
  428. m_id,
  429. )
  430. self.logger.debug(msg)
  431. def _update_page_status(self, cursor, result, pageid, content, status, chart):
  432. """Update the status and "specialed" information of a page."""
  433. query1 = """UPDATE page JOIN row ON page_id = row_id
  434. SET page_status = ?, row_chart = ? WHERE page_id = ?"""
  435. query2 = """UPDATE page SET page_special_user = ?,
  436. page_special_time = ?, page_special_oldid = ?
  437. WHERE page_id = ?"""
  438. cursor.execute(query1, (status, chart, pageid))
  439. msg = " {0}: status: {1} ({2}) -> {3} ({4})"
  440. self.logger.debug(
  441. msg.format(
  442. pageid, result["page_status"], result["row_chart"], status, chart
  443. )
  444. )
  445. s_user, s_time, s_id = self._get_special(pageid, content, chart)
  446. if s_id != result["page_special_oldid"]:
  447. cursor.execute(query2, (s_user, s_time, s_id, pageid))
  448. msg = " {0}: special: {1} / {2} / {3} -> {4} / {5} / {6}"
  449. msg = msg.format(
  450. pageid,
  451. result["page_special_user"],
  452. result["page_special_time"],
  453. result["page_special_oldid"],
  454. s_user,
  455. s_time,
  456. s_id,
  457. )
  458. self.logger.debug(msg)
  459. return s_user, s_time, s_id
  460. def _update_page_notes(self, cursor, result, pageid, notes):
  461. """Update the notes (or warnings) of a page in our database."""
  462. query = "UPDATE page SET page_notes = ? WHERE page_id = ?"
  463. cursor.execute(query, (notes, pageid))
  464. msg = " {0}: notes: {1} -> {2}"
  465. self.logger.debug(msg.format(pageid, result["page_notes"], notes))
  466. ###################### DATA RETRIEVAL HELPER METHODS ######################
  467. def _get_content(self, pageid):
  468. """Get the current content of a page by ID from the API.
  469. The page's current revision ID is retrieved from SQL, and then
  470. an API query is made to get its content. This is the only API query
  471. used in the task's code.
  472. """
  473. query = "SELECT page_latest FROM page WHERE page_id = ?"
  474. result = self.site.sql_query(query, (pageid,))
  475. try:
  476. revid = int(list(result)[0][0])
  477. except IndexError:
  478. return None
  479. return self._get_revision_content(revid)
  480. def _get_revision_content(self, revid, tries=1):
  481. """Get the content of a revision by ID from the API."""
  482. if revid in self.revision_cache:
  483. return self.revision_cache[revid]
  484. res = self.site.api_query(
  485. action="query",
  486. prop="revisions",
  487. rvprop="content",
  488. rvslots="main",
  489. revids=revid,
  490. )
  491. try:
  492. revision = res["query"]["pages"].values()[0]["revisions"][0]
  493. content = revision["slots"]["main"]["*"]
  494. except KeyError:
  495. if tries == 0:
  496. raise
  497. sleep(5)
  498. return self._get_revision_content(revid, tries=tries - 1)
  499. self.revision_cache[revid] = content
  500. return content
  501. def _get_status_and_chart(self, content, namespace):
  502. """Determine the status and chart number of an AfC submission.
  503. The methodology used here is the same one I've been using for years
  504. (see also commands.afc_report), but with the new draft system taken
  505. into account. The order here is important: if there is more than one
  506. {{AfC submission}} template on a page, we need to know which one to
  507. use (revision history search to find the most recent isn't a viable
  508. idea :P).
  509. """
  510. statuses = self.get_statuses(content)
  511. if namespace == wiki.NS_MAIN:
  512. if statuses:
  513. return None, self.CHART_MISPLACE
  514. return "a", self.CHART_ACCEPT
  515. elif "R" in statuses:
  516. return "r", self.CHART_REVIEW
  517. elif "P" in statuses:
  518. return "p", self.CHART_PEND
  519. elif "T" in statuses:
  520. return None, self.CHART_NONE
  521. elif "D" in statuses:
  522. return "d", self.CHART_DECLINE
  523. return None, self.CHART_NONE
  524. def get_statuses(self, content):
  525. """Return a list of all AfC submission statuses in a page's text."""
  526. valid = ["P", "R", "T", "D"]
  527. aliases = {
  528. "submit": "P",
  529. "afc submission/submit": "P",
  530. "afc submission/reviewing": "R",
  531. "afc submission/pending": "P",
  532. "afc submission/draft": "T",
  533. "afc submission/declined": "D",
  534. }
  535. statuses = []
  536. code = mwparserfromhell.parse(content)
  537. for template in code.filter_templates():
  538. name = template.name.strip().lower()
  539. if name == "afc submission":
  540. if template.has(1, ignore_empty=True):
  541. status = template.get(1).value.strip().upper()
  542. statuses.append(status if status in valid else "P")
  543. else:
  544. statuses.append("P")
  545. elif name in aliases:
  546. statuses.append(aliases[name])
  547. return statuses
  548. def _get_modify(self, pageid):
  549. """Return information about a page's last edit ("modification").
  550. This consists of the most recent editor, modification time, and the
  551. lastest revision ID.
  552. """
  553. query = """SELECT actor_name, rev_timestamp, rev_id
  554. FROM revision
  555. JOIN page ON rev_id = page_latest
  556. JOIN actor ON rev_actor = actor_id
  557. WHERE page_id = ?"""
  558. result = self.site.sql_query(query, (pageid,))
  559. m_user, m_time, m_id = list(result)[0]
  560. timestamp = datetime.strptime(m_time, "%Y%m%d%H%M%S")
  561. return m_user.decode("utf8"), timestamp, m_id
  562. def _get_special(self, pageid, content, chart):
  563. """Return information about a page's "special" edit.
  564. I tend to use the term "special" as a verb a lot, which is bound to
  565. cause confusion. It is merely a short way of saying "the edit in which
  566. a declined submission was declined, an accepted submission was
  567. accepted, a submission in review was set as such, a pending submission
  568. was submitted, and a "misplaced" submission was created."
  569. This "information" consists of the special edit's editor, its time, and
  570. its revision ID. If the page's status is not something that involves
  571. "special"-ing, we will return None for all three. The same will be
  572. returned if we cannot determine when the page was "special"-ed.
  573. """
  574. charts = {
  575. self.CHART_NONE: (lambda pageid, content: None, None, None),
  576. self.CHART_MISPLACE: self.get_create,
  577. self.CHART_ACCEPT: self.get_accepted,
  578. self.CHART_REVIEW: self.get_reviewing,
  579. self.CHART_PEND: self.get_pending,
  580. self.CHART_DECLINE: self.get_decline,
  581. }
  582. return charts[chart](pageid, content)
  583. def get_create(self, pageid, content=None):
  584. """Return (creator, create_ts, create_revid) for the given page."""
  585. query = """SELECT actor_name, rev_timestamp, rev_id
  586. FROM revision
  587. JOIN actor ON rev_actor = actor_id
  588. WHERE rev_id = (SELECT MIN(rev_id) FROM revision WHERE rev_page = ?)"""
  589. result = self.site.sql_query(query, (pageid,))
  590. c_user, c_time, c_id = list(result)[0]
  591. timestamp = datetime.strptime(c_time, "%Y%m%d%H%M%S")
  592. return c_user.decode("utf8"), timestamp, c_id
  593. def get_accepted(self, pageid, content=None):
  594. """Return (acceptor, accept_ts, accept_revid) for the given page."""
  595. query = """SELECT actor_name, rev_timestamp, rev_id
  596. FROM revision
  597. JOIN actor ON rev_actor = actor_id
  598. JOIN comment ON rev_comment_id = comment_id
  599. WHERE rev_page = ?
  600. AND comment_text LIKE "% moved page [[%]] to [[%]]%"
  601. ORDER BY rev_timestamp DESC LIMIT 1"""
  602. result = self.site.sql_query(query, (pageid,))
  603. try:
  604. a_user, a_time, a_id = list(result)[0]
  605. except IndexError:
  606. return None, None, None
  607. timestamp = datetime.strptime(a_time, "%Y%m%d%H%M%S")
  608. return a_user.decode("utf8"), timestamp, a_id
  609. def get_reviewing(self, pageid, content=None):
  610. """Return (reviewer, review_ts, review_revid) for the given page."""
  611. return self._search_history(pageid, self.CHART_REVIEW, ["R"], [])
  612. def get_pending(self, pageid, content):
  613. """Return (submitter, submit_ts, submit_revid) for the given page."""
  614. res = self._get_status_helper(pageid, content, ("P", ""), ("u", "ts"))
  615. return res or self._search_history(pageid, self.CHART_PEND, ["P"], [])
  616. def get_decline(self, pageid, content):
  617. """Return (decliner, decline_ts, decline_revid) for the given page."""
  618. params = ("decliner", "declinets")
  619. res = self._get_status_helper(pageid, content, ("D"), params)
  620. return res or self._search_history(
  621. pageid, self.CHART_DECLINE, ["D"], ["R", "P", "T"]
  622. )
  623. def _get_status_helper(self, pageid, content, statuses, params):
  624. """Helper function for get_pending() and get_decline()."""
  625. submits = []
  626. code = mwparserfromhell.parse(content)
  627. for tmpl in code.filter_templates():
  628. status = tmpl.get(1).value.strip().upper() if tmpl.has(1) else "P"
  629. if tmpl.name.strip().lower() == "afc submission":
  630. if all([tmpl.has(par, ignore_empty=True) for par in params]):
  631. if status in statuses:
  632. data = [str(tmpl.get(par).value) for par in params]
  633. submits.append(data)
  634. if not submits:
  635. return None
  636. user, stamp = max(submits, key=lambda pair: pair[1])
  637. query = """SELECT rev_id
  638. FROM revision_userindex
  639. JOIN actor ON rev_actor = actor_id
  640. WHERE rev_page = ? AND actor_name = ? AND ABS(rev_timestamp - ?) <= 60
  641. ORDER BY ABS(rev_timestamp - ?) ASC LIMIT 1"""
  642. result = self.site.sql_query(query, (pageid, user, stamp, stamp))
  643. try:
  644. dtime = datetime.strptime(stamp, "%Y%m%d%H%M%S")
  645. return user, dtime, list(result)[0][0]
  646. except (ValueError, IndexError):
  647. return None
  648. def _search_history(self, pageid, chart, search_with, search_without):
  649. """Search through a page's history to find when a status was set.
  650. Linear search backwards in time for the edit right after the most
  651. recent edit that fails the (pseudocode) test:
  652. ``status_set(any(search_with)) && !status_set(any(search_without))``
  653. """
  654. query = """SELECT actor_name, rev_timestamp, rev_id
  655. FROM revision
  656. JOIN actor ON rev_actor = actor_id
  657. WHERE rev_page = ? ORDER BY rev_id DESC"""
  658. result = self.site.sql_query(query, (pageid,))
  659. counter = 0
  660. last = (None, None, None)
  661. for user, ts, revid in result:
  662. counter += 1
  663. if counter > 50:
  664. msg = "Exceeded 50 content lookups while searching history of page (id: {0}, chart: {1})"
  665. self.logger.warn(msg.format(pageid, chart))
  666. return None, None, None
  667. try:
  668. content = self._get_revision_content(revid)
  669. except exceptions.APIError:
  670. msg = "API error interrupted SQL query in _search_history() for page (id: {0}, chart: {1})"
  671. self.logger.exception(msg.format(pageid, chart))
  672. return None, None, None
  673. statuses = self.get_statuses(content)
  674. req = search_with and not any([s in statuses for s in search_with])
  675. if any([s in statuses for s in search_without]) or req:
  676. return last
  677. timestamp = datetime.strptime(ts, "%Y%m%d%H%M%S")
  678. last = (user.decode("utf8"), timestamp, revid)
  679. return last
  680. def _get_notes(self, chart, content, m_time, s_user):
  681. """Return any special notes or warnings about this page.
  682. copyvio: submission is a suspected copyright violation
  683. unsourced: submission lacks references completely
  684. no-inline: submission has no inline citations
  685. short: submission is less than a kilobyte in length
  686. resubmit: submission was resubmitted after a previous decline
  687. old: submission has not been touched in > 4 days
  688. rejected: submission was rejected, a more severe form of declined
  689. blocked: submitter is currently blocked
  690. """
  691. notes = ""
  692. ignored_charts = [self.CHART_NONE, self.CHART_ACCEPT]
  693. if chart in ignored_charts:
  694. return notes
  695. if chart == self.CHART_DECLINE:
  696. # Decline is special, as only the rejected note is meaningful
  697. code = mwparserfromhell.parse(content)
  698. for tmpl in code.filter_templates():
  699. if tmpl.name.strip().lower() == "afc submission":
  700. if tmpl.has("reject") and tmpl.get("reject").value:
  701. notes += "|nj=1" # Submission was rejected
  702. break
  703. return notes
  704. copyvios = self.config.tasks.get("afc_copyvios", {})
  705. regex = r"\{\{s*" + copyvios.get("template", "AfC suspected copyvio")
  706. if re.search(regex, content):
  707. notes += "|nc=1" # Submission is a suspected copyvio
  708. if not re.search(r"\<ref\s*(.*?)\>(.*?)\</ref\>", content, re.I | re.S):
  709. regex = r"(https?:)|\[//(?!{0})([^ \]\t\n\r\f\v]+?)"
  710. sitedomain = re.escape(self.site.domain)
  711. if re.search(regex.format(sitedomain), content, re.I | re.S):
  712. notes += "|ni=1" # Submission has no inline citations
  713. else:
  714. notes += "|nu=1" # Submission is completely unsourced
  715. if len(content) < 1000:
  716. notes += "|ns=1" # Submission is short
  717. statuses = self.get_statuses(content)
  718. if "D" in statuses and chart != self.CHART_MISPLACE:
  719. notes += "|nr=1" # Submission was resubmitted
  720. time_since_modify = (datetime.utcnow() - m_time).total_seconds()
  721. max_time = 4 * 24 * 60 * 60
  722. if time_since_modify > max_time:
  723. notes += "|no=1" # Submission hasn't been touched in over 4 days
  724. if chart == self.CHART_PEND and s_user:
  725. submitter = self.site.get_user(s_user)
  726. try:
  727. if submitter.blockinfo:
  728. notes += "|nb=1" # Submitter is blocked
  729. except exceptions.UserNotFoundError: # Likely an IP
  730. pass
  731. return notes