Additional IRC commands and bot tasks for EarwigBot https://en.wikipedia.org/wiki/User:EarwigBot
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

723 rivejä
30 KiB

  1. # -*- coding: utf-8 -*-
  2. #
  3. # Copyright (C) 2009-2013 Ben Kurtovic <ben.kurtovic@verizon.net>
  4. #
  5. # Permission is hereby granted, free of charge, to any person obtaining a copy
  6. # of this software and associated documentation files (the "Software"), to deal
  7. # in the Software without restriction, including without limitation the rights
  8. # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
  9. # copies of the Software, and to permit persons to whom the Software is
  10. # furnished to do so, subject to the following conditions:
  11. #
  12. # The above copyright notice and this permission notice shall be included in
  13. # all copies or substantial portions of the Software.
  14. #
  15. # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16. # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17. # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  18. # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19. # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
  20. # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  21. # SOFTWARE.
  22. from datetime import datetime
  23. import re
  24. from os.path import expanduser
  25. from threading import Lock
  26. from time import sleep
  27. import oursql
  28. from earwigbot import exceptions
  29. from earwigbot import wiki
  30. from earwigbot.tasks import Task
  31. class AFCStatistics(Task):
  32. """A task to generate statistics for WikiProject Articles for Creation.
  33. Statistics are stored in a MySQL database ("u_earwig_afc_statistics")
  34. accessed with oursql. Statistics are synchronied with the live database
  35. every four minutes and saved once an hour, on the hour, to self.pagename.
  36. In the live bot, this is "Template:AFC statistics".
  37. """
  38. name = "afc_statistics"
  39. number = 2
  40. # Chart status number constants:
  41. CHART_NONE = 0
  42. CHART_PEND = 1
  43. CHART_REVIEW = 3
  44. CHART_ACCEPT = 4
  45. CHART_DECLINE = 5
  46. CHART_MISPLACE = 6
  47. def setup(self):
  48. self.cfg = cfg = self.config.tasks.get(self.name, {})
  49. # Set some wiki-related attributes:
  50. self.pagename = cfg.get("page", "Template:AFC statistics")
  51. self.pending_cat = cfg.get("pending", "Pending AfC submissions")
  52. self.ignore_list = cfg.get("ignoreList", [])
  53. default_summary = "Updating statistics for [[WP:WPAFC|WikiProject Articles for creation]]."
  54. self.summary = self.make_summary(cfg.get("summary", default_summary))
  55. # Templates used in chart generation:
  56. templates = cfg.get("templates", {})
  57. self.tl_header = templates.get("header", "AFC statistics/header")
  58. self.tl_row = templates.get("row", "#invoke:AfC|row")
  59. self.tl_footer = templates.get("footer", "AFC statistics/footer")
  60. # Connection data for our SQL database:
  61. kwargs = cfg.get("sql", {})
  62. kwargs["read_default_file"] = expanduser("~/.my.cnf")
  63. self.conn_data = kwargs
  64. self.db_access_lock = Lock()
  65. def run(self, **kwargs):
  66. """Entry point for a task event.
  67. Depending on the kwargs passed, we will either synchronize our local
  68. statistics database with the site (self.sync()) or save it to the wiki
  69. (self.save()). We will additionally create an SQL connection with our
  70. local database.
  71. """
  72. action = kwargs.get("action")
  73. if not self.db_access_lock.acquire(False): # Non-blocking
  74. if action == "sync":
  75. self.logger.info("A sync is already ongoing; aborting")
  76. return
  77. self.logger.info("Waiting for database access lock")
  78. self.db_access_lock.acquire()
  79. try:
  80. self.site = self.bot.wiki.get_site()
  81. self.conn = oursql.connect(**self.conn_data)
  82. self.revision_cache = {}
  83. try:
  84. if action == "save":
  85. self.save(kwargs)
  86. elif action == "sync":
  87. self.sync(kwargs)
  88. elif action == "update":
  89. self.update(kwargs)
  90. finally:
  91. self.conn.close()
  92. finally:
  93. self.db_access_lock.release()
  94. def save(self, kwargs):
  95. """Save our local statistics to the wiki.
  96. After checking for emergency shutoff, the statistics chart is compiled,
  97. and then saved to self.pagename using self.summary iff it has changed
  98. since last save.
  99. """
  100. self.logger.info("Saving chart")
  101. if kwargs.get("fromIRC"):
  102. summary = self.summary + " (!earwigbot)"
  103. else:
  104. if self.shutoff_enabled():
  105. return
  106. summary = self.summary
  107. statistics = self.compile_charts()
  108. page = self.site.get_page(self.pagename)
  109. text = page.get()
  110. newtext = re.sub(u"<!-- stat begin -->(.*?)<!-- stat end -->",
  111. "<!-- stat begin -->\n" + statistics + "\n<!-- stat end -->",
  112. text, flags=re.DOTALL)
  113. if newtext == text:
  114. self.logger.info("Chart unchanged; not saving")
  115. return # Don't edit the page if we're not adding anything
  116. newtext = re.sub("<!-- sig begin -->(.*?)<!-- sig end -->",
  117. "<!-- sig begin -->~~~ at ~~~~~<!-- sig end -->",
  118. newtext)
  119. page.edit(newtext, summary, minor=True, bot=True)
  120. self.logger.info(u"Chart saved to [[{0}]]".format(page.title))
  121. def compile_charts(self):
  122. """Compile and return all statistics information from our local db."""
  123. stats = ""
  124. with self.conn.cursor() as cursor:
  125. cursor.execute("SELECT * FROM chart")
  126. for chart in cursor:
  127. stats += self.compile_chart(chart) + "\n"
  128. return stats[:-1] # Drop the last newline
  129. def compile_chart(self, chart_info):
  130. """Compile and return a single statistics chart."""
  131. chart_id, chart_title, special_title = chart_info
  132. chart = self.tl_header + "|" + chart_title
  133. if special_title:
  134. chart += "|" + special_title
  135. chart = "{{" + chart + "}}"
  136. query = "SELECT * FROM page JOIN row ON page_id = row_id WHERE row_chart = ?"
  137. with self.conn.cursor(oursql.DictCursor) as cursor:
  138. cursor.execute(query, (chart_id,))
  139. for page in cursor:
  140. chart += "\n" + self.compile_chart_row(page)
  141. chart += "\n{{" + self.tl_footer + "}}"
  142. return chart
  143. def compile_chart_row(self, page):
  144. """Compile and return a single chart row.
  145. 'page' is a dict of page information, taken as a row from the page
  146. table, where keys are column names and values are their cell contents.
  147. """
  148. row = u"{0}|s={page_status}|t={page_title}|z={page_size}|"
  149. if page["page_special_oldid"]:
  150. row += "sr={page_special_user}|sd={page_special_time}|si={page_special_oldid}|"
  151. row += "mr={page_modify_user}|md={page_modify_time}|mi={page_modify_oldid}"
  152. page["page_special_time"] = self.format_time(page["page_special_time"])
  153. page["page_modify_time"] = self.format_time(page["page_modify_time"])
  154. if page["page_notes"]:
  155. row += "|n=1{page_notes}"
  156. return "{{" + row.format(self.tl_row, **page) + "}}"
  157. def format_time(self, dt):
  158. """Format a datetime into the standard MediaWiki timestamp format."""
  159. return dt.strftime("%H:%M, %d %b %Y")
  160. def sync(self, kwargs):
  161. """Synchronize our local statistics database with the site.
  162. Syncing involves, in order, updating tracked submissions that have
  163. been changed since last sync (self.update_tracked()), adding pending
  164. submissions that are not tracked (self.add_untracked()), and removing
  165. old submissions from the database (self.delete_old()).
  166. The sync will be canceled if SQL replication lag is greater than 600
  167. seconds, because this will lead to potential problems and outdated
  168. data, not to mention putting demand on an already overloaded server.
  169. Giving sync the kwarg "ignore_replag" will go around this restriction.
  170. """
  171. self.logger.info("Starting sync")
  172. replag = self.site.get_replag()
  173. self.logger.debug("Server replag is {0}".format(replag))
  174. if replag > 600 and not kwargs.get("ignore_replag"):
  175. msg = "Sync canceled as replag ({0} secs) is greater than ten minutes"
  176. self.logger.warn(msg.format(replag))
  177. return
  178. with self.conn.cursor() as cursor:
  179. self.update_tracked(cursor)
  180. self.add_untracked(cursor)
  181. self.delete_old(cursor)
  182. self.logger.info("Sync completed")
  183. def update_tracked(self, cursor):
  184. """Update tracked submissions that have been changed since last sync.
  185. This is done by iterating through every page in our database and
  186. comparing our stored latest revision ID with the actual latest revision
  187. ID from an SQL query. If they differ, we will update our information
  188. about the page (self.update_page()).
  189. If the page does not exist, we will remove it from our database with
  190. self.untrack_page().
  191. """
  192. self.logger.debug("Updating tracked submissions")
  193. query = """SELECT s.page_id, s.page_title, s.page_modify_oldid,
  194. r.page_latest, r.page_title, r.page_namespace
  195. FROM page AS s
  196. LEFT JOIN {0}_p.page AS r ON s.page_id = r.page_id
  197. WHERE s.page_modify_oldid != r.page_latest"""
  198. cursor.execute(query.format(self.site.name))
  199. for pageid, title, oldid, real_oldid, real_title, real_ns in cursor:
  200. if not real_oldid:
  201. self.untrack_page(cursor, pageid)
  202. continue
  203. msg = u"Updating page [[{0}]] (id: {1}) @ {2}"
  204. self.logger.debug(msg.format(title, pageid, oldid))
  205. msg = u" {0}: oldid: {1} -> {2}"
  206. self.logger.debug(msg.format(pageid, oldid, real_oldid))
  207. real_title = real_title.decode("utf8").replace("_", " ")
  208. ns = self.site.namespace_id_to_name(real_ns)
  209. if ns:
  210. real_title = u":".join((ns, real_title))
  211. try:
  212. self.update_page(cursor, pageid, real_title)
  213. except Exception:
  214. e = u"Error updating page [[{0}]] (id: {1})"
  215. self.logger.exception(e.format(real_title, pageid))
  216. def add_untracked(self, cursor):
  217. """Add pending submissions that are not yet tracked.
  218. This is done by compiling a list of all currently tracked submissions
  219. and iterating through all members of self.pending_cat via SQL. If a
  220. page in the pending category is not tracked and is not in
  221. self.ignore_list, we will track it with self.track_page().
  222. """
  223. self.logger.debug("Adding untracked pending submissions")
  224. query = """SELECT r.page_id, r.page_title, r.page_namespace
  225. FROM {0}_p.page AS r
  226. INNER JOIN {0}_p.categorylinks AS c ON r.page_id = c.cl_from
  227. LEFT JOIN page AS s ON r.page_id = s.page_id
  228. WHERE s.page_id IS NULL AND c.cl_to = ?"""
  229. cursor.execute(query.format(self.site.name),
  230. (self.pending_cat.replace(" ", "_"),))
  231. for pageid, title, ns in cursor:
  232. title = title.decode("utf8").replace("_", " ")
  233. ns = self.site.namespace_id_to_name(ns)
  234. if ns:
  235. title = u":".join((ns, title))
  236. if title in self.ignore_list:
  237. continue
  238. msg = u"Tracking page [[{0}]] (id: {1})".format(title, pageid)
  239. self.logger.debug(msg)
  240. try:
  241. self.track_page(cursor, pageid, title)
  242. except Exception:
  243. e = u"Error tracking page [[{0}]] (id: {1})"
  244. self.logger.exception(e.format(title, pageid))
  245. def delete_old(self, cursor):
  246. """Remove old submissions from the database.
  247. "Old" is defined as a submission that has been declined or accepted
  248. more than 36 hours ago. Pending submissions cannot be "old".
  249. """
  250. self.logger.debug("Removing old submissions from chart")
  251. query = """DELETE FROM page, row USING page JOIN row
  252. ON page_id = row_id WHERE row_chart IN (?, ?)
  253. AND ADDTIME(page_special_time, '36:00:00') < NOW()"""
  254. cursor.execute(query, (self.CHART_ACCEPT, self.CHART_DECLINE))
  255. def update(self, kwargs):
  256. """Update a page by name, regardless of whether anything has changed.
  257. Mainly intended as a command to be used via IRC, e.g.:
  258. !tasks start afc_statistics action=update page=Foobar
  259. """
  260. title = kwargs.get("page")
  261. if not title:
  262. return
  263. title = title.replace("_", " ").decode("utf8")
  264. query = "SELECT page_id, page_modify_oldid FROM page WHERE page_title = ?"
  265. with self.conn.cursor() as cursor:
  266. cursor.execute(query, (title,))
  267. try:
  268. pageid, oldid = cursor.fetchall()[0]
  269. except IndexError:
  270. msg = u"Page [[{0}]] not found in database".format(title)
  271. self.logger.error(msg)
  272. msg = u"Updating page [[{0}]] (id: {1}) @ {2}"
  273. self.logger.info(msg.format(title, pageid, oldid))
  274. self.update_page(cursor, pageid, title)
  275. def untrack_page(self, cursor, pageid):
  276. """Remove a page, given by ID, from our database."""
  277. self.logger.debug("Untracking page (id: {0})".format(pageid))
  278. query = """DELETE FROM page, row USING page JOIN row
  279. ON page_id = row_id WHERE page_id = ?"""
  280. cursor.execute(query, (pageid,))
  281. def track_page(self, cursor, pageid, title):
  282. """Update hook for when page is not in our database.
  283. A variety of SQL queries are used to gather information about the page,
  284. which is then saved to our database.
  285. """
  286. content = self.get_content(title)
  287. if content is None:
  288. msg = u"Could not get page content for [[{0}]]".format(title)
  289. self.logger.error(msg)
  290. return
  291. namespace = self.site.get_page(title).namespace
  292. status, chart = self.get_status_and_chart(content, namespace)
  293. if chart == self.CHART_NONE:
  294. msg = u"Could not find a status for [[{0}]]".format(title)
  295. self.logger.warn(msg)
  296. return
  297. size = self.get_size(content)
  298. m_user, m_time, m_id = self.get_modify(pageid)
  299. s_user, s_time, s_id = self.get_special(pageid, chart)
  300. notes = self.get_notes(chart, content, m_time, s_user)
  301. query1 = "INSERT INTO row VALUES (?, ?)"
  302. query2 = "INSERT INTO page VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"
  303. cursor.execute(query1, (pageid, chart))
  304. cursor.execute(query2, (pageid, status, title, size, notes, m_user,
  305. m_time, m_id, s_user, s_time, s_id))
  306. def update_page(self, cursor, pageid, title):
  307. """Update hook for when page is already in our database.
  308. A variety of SQL queries are used to gather information about the page,
  309. which is compared against our stored information. Differing information
  310. is then updated.
  311. """
  312. content = self.get_content(title)
  313. if content is None:
  314. msg = u"Could not get page content for [[{0}]]".format(title)
  315. self.logger.error(msg)
  316. return
  317. namespace = self.site.get_page(title).namespace
  318. status, chart = self.get_status_and_chart(content, namespace)
  319. if chart == self.CHART_NONE:
  320. self.untrack_page(cursor, pageid)
  321. return
  322. query = "SELECT * FROM page JOIN row ON page_id = row_id WHERE page_id = ?"
  323. with self.conn.cursor(oursql.DictCursor) as dict_cursor:
  324. dict_cursor.execute(query, (pageid,))
  325. result = dict_cursor.fetchall()[0]
  326. size = self.get_size(content)
  327. m_user, m_time, m_id = self.get_modify(pageid)
  328. if title != result["page_title"]:
  329. self.update_page_title(cursor, result, pageid, title)
  330. if m_id != result["page_modify_oldid"]:
  331. self.update_page_modify(cursor, result, pageid, size, m_user,
  332. m_time, m_id)
  333. if status != result["page_status"]:
  334. special = self.update_page_status(cursor, result, pageid, status,
  335. chart)
  336. s_user = special[0]
  337. else:
  338. s_user = result["page_special_user"]
  339. notes = self.get_notes(chart, content, m_time, s_user)
  340. if notes != result["page_notes"]:
  341. self.update_page_notes(cursor, result, pageid, notes)
  342. def update_page_title(self, cursor, result, pageid, title):
  343. """Update the title of a page in our database."""
  344. query = "UPDATE page SET page_title = ? WHERE page_id = ?"
  345. cursor.execute(query, (title, pageid))
  346. msg = u" {0}: title: {1} -> {2}"
  347. self.logger.debug(msg.format(pageid, result["page_title"], title))
  348. def update_page_modify(self, cursor, result, pageid, size, m_user, m_time, m_id):
  349. """Update the last modified information of a page in our database."""
  350. query = """UPDATE page SET page_size = ?, page_modify_user = ?,
  351. page_modify_time = ?, page_modify_oldid = ?
  352. WHERE page_id = ?"""
  353. cursor.execute(query, (size, m_user, m_time, m_id, pageid))
  354. msg = u" {0}: modify: {1} / {2} / {3} -> {4} / {5} / {6}"
  355. msg = msg.format(pageid, result["page_modify_user"],
  356. result["page_modify_time"],
  357. result["page_modify_oldid"], m_user, m_time, m_id)
  358. self.logger.debug(msg)
  359. def update_page_status(self, cursor, result, pageid, status, chart):
  360. """Update the status and "specialed" information of a page."""
  361. query1 = """UPDATE page JOIN row ON page_id = row_id
  362. SET page_status = ?, row_chart = ? WHERE page_id = ?"""
  363. query2 = """UPDATE page SET page_special_user = ?,
  364. page_special_time = ?, page_special_oldid = ?
  365. WHERE page_id = ?"""
  366. cursor.execute(query1, (status, chart, pageid))
  367. msg = " {0}: status: {1} ({2}) -> {3} ({4})"
  368. self.logger.debug(msg.format(pageid, result["page_status"],
  369. result["row_chart"], status, chart))
  370. s_user, s_time, s_id = self.get_special(pageid, chart)
  371. if s_id != result["page_special_oldid"]:
  372. cursor.execute(query2, (s_user, s_time, s_id, pageid))
  373. msg = u" {0}: special: {1} / {2} / {3} -> {4} / {5} / {6}"
  374. msg = msg.format(pageid, result["page_special_user"],
  375. result["page_special_time"],
  376. result["page_special_oldid"], s_user, s_time, s_id)
  377. self.logger.debug(msg)
  378. return s_user, s_time, s_id
  379. def update_page_notes(self, cursor, result, pageid, notes):
  380. """Update the notes (or warnings) of a page in our database."""
  381. query = "UPDATE page SET page_notes = ? WHERE page_id = ?"
  382. cursor.execute(query, (notes, pageid))
  383. msg = " {0}: notes: {1} -> {2}"
  384. self.logger.debug(msg.format(pageid, result["page_notes"], notes))
  385. def get_content(self, title):
  386. """Get the current content of a page by title from the API.
  387. The page's current revision ID is retrieved from SQL, and then
  388. an API query is made to get its content. This is the only API query
  389. used in the task's code.
  390. """
  391. query = "SELECT page_latest FROM page WHERE page_title = ? AND page_namespace = ?"
  392. try:
  393. namespace, base = title.split(":", 1)
  394. except ValueError:
  395. base = title
  396. ns = wiki.NS_MAIN
  397. else:
  398. try:
  399. ns = self.site.namespace_name_to_id(namespace)
  400. except exceptions.NamespaceNotFoundError:
  401. base = title
  402. ns = wiki.NS_MAIN
  403. result = self.site.sql_query(query, (base.replace(" ", "_"), ns))
  404. try:
  405. revid = int(list(result)[0][0])
  406. except IndexError:
  407. return None
  408. return self.get_revision_content(revid)
  409. def get_revision_content(self, revid, tries=1):
  410. """Get the content of a revision by ID from the API."""
  411. if revid in self.revision_cache:
  412. return self.revision_cache[revid]
  413. res = self.site.api_query(action="query", prop="revisions",
  414. revids=revid, rvprop="content")
  415. try:
  416. content = res["query"]["pages"].values()[0]["revisions"][0]["*"]
  417. except KeyError:
  418. if tries > 0:
  419. sleep(5)
  420. return self.get_revision_content(revid, tries=tries - 1)
  421. self.revision_cache[revid] = content
  422. return content
  423. def get_status_and_chart(self, content, namespace):
  424. """Determine the status and chart number of an AFC submission.
  425. The methodology used here is the same one I've been using for years
  426. (see also commands.afc_report), but with the new draft system taken
  427. into account. The order here is important: if there is more than one
  428. {{AFC submission}} template on a page, we need to know which one to
  429. use (revision history search to find the most recent isn't a viable
  430. idea :P).
  431. """
  432. statuses = self.get_statuses(content)
  433. if "R" in statuses:
  434. status, chart = "r", self.CHART_REVIEW
  435. elif "P" in statuses:
  436. status, chart = "p", self.CHART_PEND
  437. elif "T" in statuses:
  438. status, chart = None, self.CHART_NONE
  439. elif "D" in statuses:
  440. status, chart = "d", self.CHART_DECLINE
  441. else:
  442. status, chart = None, self.CHART_NONE
  443. if namespace == wiki.NS_MAIN:
  444. if not statuses:
  445. status, chart = "a", self.CHART_ACCEPT
  446. else:
  447. status, chart = None, self.CHART_MISPLACE
  448. return status, chart
  449. def get_statuses(self, content):
  450. """Return a list of all AFC submission statuses in a page's text."""
  451. re_has_templates = "\{\{[aA][fF][cC] submission\s*(\}\}|\||/)"
  452. re_template = "\{\{[aA][fF][cC] submission\s*(.*?)\}\}"
  453. re_remove_embed = "(\{\{[aA][fF][cC] submission\s*(.*?))\{\{(.*?)\}\}(.*?)\}\}"
  454. valid = ["R", "P", "T", "D"]
  455. subtemps = {
  456. "/reviewing": "R",
  457. "/pending": "P",
  458. "/draft": "T",
  459. "/declined": "D"
  460. }
  461. statuses = []
  462. while re.search(re_has_templates, content):
  463. status = "P"
  464. match = re.search(re_template, content, re.S)
  465. if not match:
  466. return statuses
  467. temp = match.group(1)
  468. limit = 0
  469. while "{{" in temp and limit < 50:
  470. content = re.sub(re_remove_embed, "\\1\\4}}", content, 1, re.S)
  471. match = re.search(re_template, content, re.S)
  472. temp = match.group(1)
  473. limit += 1
  474. params = temp.split("|")
  475. try:
  476. subtemp, params = params[0].strip(), params[1:]
  477. except IndexError:
  478. status = "P"
  479. params = []
  480. else:
  481. if subtemp:
  482. status = subtemps.get(subtemp)
  483. params = []
  484. for param in params:
  485. param = param.strip().upper()
  486. if "=" in param:
  487. key, value = param.split("=", 1)
  488. if key.strip() == "1":
  489. status = value if value in valid else "P"
  490. break
  491. else:
  492. status = param if param in valid else "P"
  493. break
  494. statuses.append(status)
  495. content = re.sub(re_template, "", content, 1, re.S)
  496. return statuses
  497. def get_size(self, content):
  498. """Return a page's size in a short, pretty format."""
  499. return "{0} kB".format(round(len(content) / 1000.0, 1))
  500. def get_modify(self, pageid):
  501. """Return information about a page's last edit ("modification").
  502. This consists of the most recent editor, modification time, and the
  503. lastest revision ID.
  504. """
  505. query = """SELECT rev_user_text, rev_timestamp, rev_id FROM revision
  506. JOIN page ON rev_id = page_latest WHERE page_id = ?"""
  507. result = self.site.sql_query(query, (pageid,))
  508. m_user, m_time, m_id = list(result)[0]
  509. timestamp = datetime.strptime(m_time, "%Y%m%d%H%M%S")
  510. return m_user.decode("utf8"), timestamp, m_id
  511. def get_special(self, pageid, chart):
  512. """Return information about a page's "special" edit.
  513. I tend to use the term "special" as a verb a lot, which is bound to
  514. cause confusion. It is merely a short way of saying "the edit in which
  515. a declined submission was declined, an accepted submission was
  516. accepted, a submission in review was set as such, a pending submission
  517. was submitted, and a "misplaced" submission was created."
  518. This "information" consists of the special edit's editor, its time, and
  519. its revision ID. If the page's status is not something that involves
  520. "special"-ing, we will return None for all three. The same will be
  521. returned if we cannot determine when the page was "special"-ed, or if
  522. it was "special"-ed more than 100 edits ago.
  523. """
  524. if chart == self.CHART_NONE:
  525. return None, None, None
  526. elif chart == self.CHART_MISPLACE:
  527. return self.get_create(pageid)
  528. elif chart == self.CHART_ACCEPT:
  529. search_for = None
  530. search_not = ["R", "P", "T", "D"]
  531. elif chart == self.CHART_PEND:
  532. search_for = "P"
  533. search_not = []
  534. elif chart == self.CHART_REVIEW:
  535. search_for = "R"
  536. search_not = []
  537. elif chart == self.CHART_DECLINE:
  538. search_for = "D"
  539. search_not = ["R", "P", "T"]
  540. query = """SELECT rev_user_text, rev_timestamp, rev_id
  541. FROM revision WHERE rev_page = ? ORDER BY rev_id DESC"""
  542. result = self.site.sql_query(query, (pageid,))
  543. counter = 0
  544. last = (None, None, None)
  545. for user, ts, revid in result:
  546. counter += 1
  547. if counter > 50:
  548. msg = "Exceeded 50 content lookups while determining special for page (id: {0}, chart: {1})"
  549. self.logger.warn(msg.format(pageid, chart))
  550. return None, None, None
  551. try:
  552. content = self.get_revision_content(revid)
  553. except exceptions.APIError:
  554. msg = "API error interrupted SQL query in get_special() for page (id: {0}, chart: {1})"
  555. self.logger.exception(msg.format(pageid, chart))
  556. return None, None, None
  557. statuses = self.get_statuses(content)
  558. matches = [s in statuses for s in search_not]
  559. if any(matches) or (search_for and search_for not in statuses):
  560. return last
  561. timestamp = datetime.strptime(ts, "%Y%m%d%H%M%S")
  562. last = (user.decode("utf8"), timestamp, revid)
  563. return last
  564. def get_create(self, pageid):
  565. """Return information about a page's first edit ("creation").
  566. This consists of the page creator, creation time, and the earliest
  567. revision ID.
  568. """
  569. query = """SELECT rev_user_text, rev_timestamp, rev_id
  570. FROM revision WHERE rev_id =
  571. (SELECT MIN(rev_id) FROM revision WHERE rev_page = ?)"""
  572. result = self.site.sql_query(query, (pageid,))
  573. c_user, c_time, c_id = list(result)[0]
  574. timestamp = datetime.strptime(c_time, "%Y%m%d%H%M%S")
  575. return c_user.decode("utf8"), timestamp, c_id
  576. def get_notes(self, chart, content, m_time, s_user):
  577. """Return any special notes or warnings about this page.
  578. copyvio: submission is a suspected copyright violation
  579. unsourced: submission lacks references completely
  580. no-inline: submission has no inline citations
  581. short: submission is less than a kilobyte in length
  582. resubmit: submission was resubmitted after a previous decline
  583. old: submission has not been touched in > 4 days
  584. blocked: submitter is currently blocked
  585. """
  586. notes = ""
  587. ignored_charts = [self.CHART_NONE, self.CHART_ACCEPT, self.CHART_DECLINE]
  588. if chart in ignored_charts:
  589. return notes
  590. copyvios = self.config.tasks.get("afc_copyvios", {})
  591. regex = "\{\{\s*" + copyvios.get("template", "AfC suspected copyvio")
  592. if re.search(regex, content):
  593. notes += "|nc=1" # Submission is a suspected copyvio
  594. if not re.search("\<ref\s*(.*?)\>(.*?)\</ref\>", content, re.I | re.S):
  595. regex = "(https?:)|\[//(?!{0})([^ \]\\t\\n\\r\\f\\v]+?)"
  596. sitedomain = re.escape(self.site.domain)
  597. if re.search(regex.format(sitedomain), content, re.I | re.S):
  598. notes += "|ni=1" # Submission has no inline citations
  599. else:
  600. notes += "|nu=1" # Submission is completely unsourced
  601. if len(content) < 1000:
  602. notes += "|ns=1" # Submission is short
  603. statuses = self.get_statuses(content)
  604. if "D" in statuses and chart != self.CHART_MISPLACE:
  605. notes += "|nr=1" # Submission was resubmitted
  606. time_since_modify = (datetime.utcnow() - m_time).total_seconds()
  607. max_time = 4 * 24 * 60 * 60
  608. if time_since_modify > max_time:
  609. notes += "|no=1" # Submission hasn't been touched in over 4 days
  610. if chart == self.CHART_PEND and s_user:
  611. submitter = self.site.get_user(s_user)
  612. try:
  613. if submitter.blockinfo:
  614. notes += "|nb=1" # Submitter is blocked
  615. except exceptions.UserNotFoundError: # Likely an IP
  616. pass
  617. return notes