A Python robot that edits Wikipedia and interacts with people over IRC https://en.wikipedia.org/wiki/User:EarwigBot
選択できるのは25トピックまでです。 トピックは、先頭が英数字で、英数字とダッシュ('-')を使用した35文字以内のものにしてください。

687 行
28 KiB

  1. # -*- coding: utf-8 -*-
  2. from datetime import datetime
  3. import logging
  4. import re
  5. from os.path import expanduser
  6. from threading import Lock
  7. from time import sleep
  8. import oursql
  9. from classes import BaseTask
  10. import config
  11. import wiki
  12. # Chart status number constants:
  13. CHART_NONE = 0
  14. CHART_PEND = 1
  15. CHART_DRAFT = 2
  16. CHART_REVIEW = 3
  17. CHART_ACCEPT = 4
  18. CHART_DECLINE = 5
  19. CHART_MISPLACE = 6
  20. class Task(BaseTask):
  21. """A task to generate statistics for WikiProject Articles for Creation.
  22. Statistics are stored in a MySQL database ("u_earwig_afc_statistics")
  23. accessed with oursql. Statistics are synchronied with the live database
  24. every four minutes and saved once an hour, on the hour, to self.pagename.
  25. In the live bot, this is "Template:AFC statistics".
  26. """
  27. name = "afc_statistics"
  28. number = 2
  29. def __init__(self):
  30. self.cfg = cfg = config.tasks.get(self.name, {})
  31. # Set some wiki-related attributes:
  32. self.pagename = cfg.get("page", "Template:AFC statistics")
  33. self.pending_cat = cfg.get("pending", "Pending AfC submissions")
  34. self.ignore_list = cfg.get("ignoreList", [])
  35. default_summary = "Updating statistics for [[WP:WPAFC|WikiProject Articles for creation]]."
  36. self.summary = self.make_summary(cfg.get("summary", default_summary))
  37. # Templates used in chart generation:
  38. templates = cfg.get("templates", {})
  39. self.tl_header = templates.get("header", "AFC statistics/header")
  40. self.tl_row = templates.get("row", "AFC statistics/row")
  41. self.tl_footer = templates.get("footer", "AFC statistics/footer")
  42. # Connection data for our SQL database:
  43. kwargs = cfg.get("sql", {})
  44. kwargs["read_default_file"] = expanduser("~/.my.cnf")
  45. self.conn_data = kwargs
  46. self.db_access_lock = Lock()
  47. def run(self, **kwargs):
  48. """Entry point for a task event.
  49. Depending on the kwargs passed, we will either synchronize our local
  50. statistics database with the site (self.sync()) or save it to the wiki
  51. (self.save()). We will additionally create an SQL connection with our
  52. local database.
  53. """
  54. self.site = wiki.get_site()
  55. with self.db_access_lock:
  56. self.conn = oursql.connect(**self.conn_data)
  57. action = kwargs.get("action")
  58. try:
  59. if action == "save":
  60. self.save(**kwargs)
  61. elif action == "sync":
  62. self.sync(**kwargs)
  63. elif action == "update":
  64. self.update(**kwargs)
  65. finally:
  66. self.conn.close()
  67. def save(self, **kwargs):
  68. """Save our local statistics to the wiki.
  69. After checking for emergency shutoff, the statistics chart is compiled,
  70. and then saved to self.pagename using self.summary iff it has changed
  71. since last save.
  72. """
  73. self.logger.info("Saving chart")
  74. if kwargs.get("fromIRC"):
  75. summary = " ".join((self.summary, "(!earwigbot)"))
  76. else:
  77. if self.shutoff_enabled():
  78. return
  79. summary = self.summary
  80. statistics = self.compile_charts().encode("utf8")
  81. page = self.site.get_page(self.pagename)
  82. text = page.get().encode("utf8")
  83. newtext = re.sub("(<!-- stat begin -->)(.*?)(<!-- stat end -->)",
  84. statistics.join(("\\1\n", "\n\\3")), text,
  85. flags=re.DOTALL)
  86. if newtext == text:
  87. self.logger.info("Chart unchanged; not saving")
  88. return # Don't edit the page if we're not adding anything
  89. newtext = re.sub("(<!-- sig begin -->)(.*?)(<!-- sig end -->)",
  90. "\\1~~~ at ~~~~~\\3", newtext)
  91. page.edit(newtext, summary, minor=True, bot=True)
  92. self.logger.info("Chart saved to [[{0}]]".format(page.title()))
  93. def compile_charts(self):
  94. """Compile and return all statistics information from our local db."""
  95. stats = ""
  96. with self.conn.cursor() as cursor:
  97. cursor.execute("SELECT * FROM chart")
  98. for chart in cursor:
  99. stats += self.compile_chart(chart) + "\n"
  100. return stats[:-1] # Drop the last newline
  101. def compile_chart(self, chart_info):
  102. """Compile and return a single statistics chart."""
  103. chart_id, chart_title, special_title = chart_info
  104. chart = "|".join((self.tl_header, chart_title))
  105. if special_title:
  106. chart += "".join(("|", special_title))
  107. chart = "".join(("{{", chart, "}}"))
  108. query = "SELECT * FROM page JOIN row ON page_id = row_id WHERE row_chart = ?"
  109. with self.conn.cursor(oursql.DictCursor) as cursor:
  110. cursor.execute(query, (chart_id,))
  111. for page in cursor:
  112. chart += "\n" + self.compile_chart_row(page).decode("utf8")
  113. chart += "".join(("\n{{", self.tl_footer, "}}"))
  114. return chart
  115. def compile_chart_row(self, page):
  116. """Compile and return a single chart row.
  117. 'page' is a dict of page information, taken as a row from the page
  118. table, where keys are column names and values are their cell contents.
  119. """
  120. row = "{0}|s={page_status}|t={page_title}|h={page_short}|z={page_size}|"
  121. row += "sr={page_special_user}|sd={page_special_time}|si={page_special_oldid}|"
  122. row += "mr={page_modify_user}|md={page_modify_time}|mi={page_modify_oldid}"
  123. page["page_special_time"] = self.format_time(page["page_special_time"])
  124. page["page_modify_time"] = self.format_time(page["page_modify_time"])
  125. if page["page_notes"]:
  126. row += "|n=1{page_notes}"
  127. return "".join(("{{", row.format(self.tl_row, **page), "}}"))
  128. def format_time(self, timestamp):
  129. """Format a datetime into the standard MediaWiki timestamp format."""
  130. return timestamp.strftime("%H:%M, %d %b %Y")
  131. def sync(self, **kwargs):
  132. """Synchronize our local statistics database with the site.
  133. Syncing involves, in order, updating tracked submissions that have
  134. been changed since last sync (self.update_tracked()), adding pending
  135. submissions that are not tracked (self.add_untracked()), and removing
  136. old submissions from the database (self.delete_old()).
  137. The sync will be canceled if SQL replication lag is greater than 600
  138. seconds, because this will lead to potential problems and outdated
  139. data, not to mention putting demand on an already overloaded server.
  140. Giving sync the kwarg "ignore_replag" will go around this restriction.
  141. """
  142. self.logger.info("Starting sync")
  143. replag = self.site.get_replag()
  144. self.logger.debug("Server replag is {0}".format(replag))
  145. if replag > 600 and not kwargs.get("ignore_replag"):
  146. msg = "Sync canceled as replag ({0} secs) is greater than ten minutes."
  147. self.logger.warn(msg.format(replag))
  148. with self.conn.cursor() as cursor:
  149. self.update_tracked(cursor)
  150. self.add_untracked(cursor)
  151. self.delete_old(cursor)
  152. self.logger.info("Sync completed")
  153. def update_tracked(self, cursor):
  154. """Update tracked submissions that have been changed since last sync.
  155. This is done by iterating through every page in our database and
  156. comparing our stored latest revision ID with the actual latest revision
  157. ID from an SQL query. If they differ, we will update our information
  158. about the page (self.update_page()).
  159. If the page does not exist, we will remove it from our database with
  160. self.untrack_page().
  161. """
  162. self.logger.debug("Updating tracked submissions")
  163. query1 = "SELECT page_id, page_title, page_modify_oldid FROM page"
  164. query2 = """SELECT page_latest, page_title, page_namespace FROM page
  165. WHERE page_id = ?"""
  166. cursor.execute(query1)
  167. for pageid, title, oldid in cursor:
  168. result = list(self.site.sql_query(query2, (pageid,)))
  169. if not result:
  170. self.untrack_page(cursor, pageid)
  171. continue
  172. real_oldid = result[0][0]
  173. if oldid != real_oldid:
  174. msg = "Updating page [[{0}]] (id: {1}) @ {2}"
  175. self.logger.debug(msg.format(title, pageid, oldid))
  176. self.logger.debug(" {0} -> {1}".format(oldid, real_oldid))
  177. body = result[0][1].replace("_", " ")
  178. ns = self.site.namespace_id_to_name(result[0][2])
  179. if ns:
  180. real_title = ":".join((str(ns), body))
  181. else:
  182. real_title = body
  183. self.update_page(cursor, pageid, real_title)
  184. def add_untracked(self, cursor):
  185. """Add pending submissions that are not yet tracked.
  186. This is done by compiling a list of all currently tracked submissions
  187. and iterating through all members of self.pending_cat via SQL. If a
  188. page in the pending category is not tracked and is not in
  189. self.ignore_list, we will track it with self.track_page().
  190. """
  191. self.logger.debug("Adding untracked pending submissions")
  192. cursor.execute("SELECT page_id FROM page")
  193. tracked = [i[0] for i in cursor.fetchall()]
  194. category = self.site.get_category(self.pending_cat)
  195. pending = category.members(use_sql=True)
  196. for title, pageid in pending:
  197. if title.decode("utf8") in self.ignore_list:
  198. continue
  199. if pageid not in tracked:
  200. msg = "Tracking page [[{0}]] (id: {1})".format(title, pageid)
  201. self.logger.debug(msg)
  202. self.track_page(cursor, pageid, title)
  203. def delete_old(self, cursor):
  204. """Remove old submissions from the database.
  205. "Old" is defined as a submission that has been declined or accepted
  206. more than 36 hours ago. Pending submissions cannot be "old".
  207. """
  208. self.logger.debug("Removing old submissions from chart")
  209. query = """DELETE FROM page, row USING page JOIN row
  210. ON page_id = row_id WHERE row_chart IN (?, ?)
  211. AND ADDTIME(page_special_time, '36:00:00') < NOW()"""
  212. cursor.execute(query, (CHART_ACCEPT, CHART_DECLINE))
  213. def update(self, **kwargs):
  214. """Update a page by name, regardless of whether anything has changed.
  215. Mainly intended as a command to be used via IRC, e.g.:
  216. !tasks start afc_statistics action=update page=Foobar
  217. """
  218. title = kwargs.get("page")
  219. if not title:
  220. return
  221. title = title.replace("_", " ")
  222. query = "SELECT page_id, page_modify_oldid FROM page WHERE page_title = ?"
  223. with self.conn.cursor() as cursor:
  224. cursor.execute(query, (title,))
  225. try:
  226. pageid, oldid = cursor.fetchall()[0]
  227. except IndexError:
  228. msg = "Page [[{0}]] not found in database".format(title)
  229. self.logger.error(msg)
  230. msg = "Updating page [[{0}]] (id: {1}) @ {2}"
  231. self.logger.info(msg.format(title, pageid, oldid))
  232. self.update_page(cursor, pageid, title)
  233. def untrack_page(self, cursor, pageid):
  234. """Remove a page, given by ID, from our database."""
  235. self.logger.debug("Untracking page (id: {0})".format(pageid))
  236. query = """DELETE FROM page, row USING page JOIN row
  237. ON page_id = row_id WHERE page_id = ?"""
  238. cursor.execute(query, (pageid,))
  239. def track_page(self, cursor, pageid, title):
  240. """Update hook for when page is not in our database.
  241. A variety of SQL queries are used to gather information about the page,
  242. which is then saved to our database.
  243. """
  244. content = self.get_content(title)
  245. if content is None:
  246. msg = "Could not get page content for [[{0}]]".format(title)
  247. self.logger.error(msg)
  248. return
  249. namespace = self.site.get_page(title).namespace()
  250. status, chart = self.get_status_and_chart(content, namespace)
  251. if chart == CHART_NONE:
  252. msg = "Could not find a status for [[{0}]]".format(title)
  253. self.logger.warn(msg)
  254. return
  255. short = self.get_short_title(title)
  256. size = self.get_size(content)
  257. m_user, m_time, m_id = self.get_modify(pageid)
  258. s_user, s_time, s_id = self.get_special(pageid, chart)
  259. notes = self.get_notes(chart, content, m_time, s_user)
  260. query1 = "INSERT INTO row VALUES (?, ?)"
  261. query2 = "INSERT INTO page VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"
  262. cursor.execute(query1, (pageid, chart))
  263. cursor.execute(query2, (pageid, status, title.decode("utf8"),
  264. short.decode("utf8"), size, notes,
  265. m_user.decode("utf8"), m_time, m_id,
  266. s_user.decode("utf8"), s_time, s_id))
  267. def update_page(self, cursor, pageid, title):
  268. """Update hook for when page is already in our database.
  269. A variety of SQL queries are used to gather information about the page,
  270. which is compared against our stored information. Differing information
  271. is then updated.
  272. """
  273. content = self.get_content(title)
  274. if content is None:
  275. msg = "Could not get page content for [[{0}]]".format(title)
  276. self.logger.error(msg)
  277. return
  278. namespace = self.site.get_page(title).namespace()
  279. status, chart = self.get_status_and_chart(content, namespace)
  280. if chart == CHART_NONE:
  281. self.untrack_page(cursor, pageid)
  282. return
  283. query = "SELECT * FROM page JOIN row ON page_id = row_id WHERE page_id = ?"
  284. with self.conn.cursor(oursql.DictCursor) as dict_cursor:
  285. dict_cursor.execute(query, (pageid,))
  286. result = dict_cursor.fetchall()[0]
  287. size = self.get_size(content)
  288. m_user, m_time, m_id = self.get_modify(pageid)
  289. notes = self.get_notes(chart, content, m_time, result["page_special_user"])
  290. if title != result["page_title"]:
  291. self.update_page_title(cursor, result, pageid, title)
  292. if m_id != result["page_modify_oldid"]:
  293. self.update_page_modify(cursor, result, pageid, size, m_user, m_time, m_id)
  294. if status != result["page_status"]:
  295. self.update_page_status(cursor, result, pageid, status, chart)
  296. if notes != result["page_notes"]:
  297. self.update_page_notes(cursor, result, pageid, notes)
  298. def update_page_title(self, cursor, result, pageid, title):
  299. """Update the title and short_title of a page in our database."""
  300. query = "UPDATE page SET page_title = ?, page_short = ? WHERE page_id = ?"
  301. short = self.get_short_title(title)
  302. cursor.execute(query, (title.decode("utf8"), short.decode("utf8"),
  303. pageid))
  304. msg = " {0}: title: {1} -> {2}"
  305. self.logger.debug(msg.format(pageid, result["page_title"], title))
  306. def update_page_modify(self, cursor, result, pageid, size, m_user, m_time, m_id):
  307. """Update the last modified information of a page in our database."""
  308. query = """UPDATE page SET page_size = ?, page_modify_user = ?,
  309. page_modify_time = ?, page_modify_oldid = ?
  310. WHERE page_id = ?"""
  311. cursor.execute(query, (size, m_user.decode("utf8"), m_time, m_id,
  312. pageid))
  313. msg = " {0}: modify: {1} / {2} / {3} -> {4} / {5} / {6}"
  314. msg = msg.format(pageid, result["page_modify_user"],
  315. result["page_modify_time"],
  316. result["page_modify_oldid"], m_user, m_time, m_id)
  317. self.logger.debug(msg)
  318. def update_page_status(self, cursor, result, pageid, status, chart):
  319. """Update the status and "specialed" information of a page."""
  320. query1 = """UPDATE page JOIN row ON page_id = row_id
  321. SET page_status = ?, row_chart = ? WHERE page_id = ?"""
  322. query2 = """UPDATE page SET page_special_user = ?,
  323. page_special_time = ?, page_special_oldid = ?
  324. WHERE page_id = ?"""
  325. cursor.execute(query1, (status, chart, pageid))
  326. msg = " {0}: status: {1} ({2}) -> {3} ({4})"
  327. self.logger.debug(msg.format(pageid, result["page_status"],
  328. result["row_chart"], status, chart))
  329. s_user, s_time, s_id = self.get_special(pageid, chart)
  330. if s_id != result["page_special_oldid"]:
  331. cursor.execute(query2, (s_user.decode("utf8"), s_time, s_id,
  332. pageid))
  333. msg = "{0}: special: {1} / {2} / {3} -> {4} / {5} / {6}"
  334. msg = msg.format(pageid, result["page_special_user"],
  335. result["page_special_time"],
  336. result["page_special_oldid"], s_user, s_time, s_id)
  337. self.logger.debug(msg)
  338. def update_page_notes(self, cursor, result, pageid, notes):
  339. """Update the notes (or warnings) of a page in our database."""
  340. query = "UPDATE page SET page_notes = ? WHERE page_id = ?"
  341. cursor.execute(query, (notes, pageid))
  342. msg = " {0}: notes: {1} -> {2}"
  343. self.logger.debug(msg.format(pageid, result["page_notes"], notes))
  344. def get_content(self, title):
  345. """Get the current content of a page by title from the API.
  346. The page's current revision ID is retrieved from SQL, and then
  347. an API query is made to get its content. This is the only API query
  348. used in the task's code.
  349. """
  350. query = "SELECT page_latest FROM page WHERE page_title = ? AND page_namespace = ?"
  351. try:
  352. namespace, base = title.decode("utf8").split(":", 1)
  353. except ValueError:
  354. base = title.decode("utf8")
  355. ns = wiki.NS_MAIN
  356. else:
  357. try:
  358. ns = self.site.namespace_name_to_id(namespace)
  359. except wiki.NamespaceNotFoundError:
  360. base = title.decode("utf8")
  361. ns = wiki.NS_MAIN
  362. result = self.site.sql_query(query, (base.replace(" ", "_"), ns))
  363. revid = int(list(result)[0][0])
  364. return self.get_revision_content(revid)
  365. def get_revision_content(self, revid):
  366. """Get the content of a revision by ID from the API."""
  367. res = self.site.api_query(action="query", prop="revisions",
  368. revids=revid, rvprop="content")
  369. try:
  370. return res["query"]["pages"].values()[0]["revisions"][0]["*"]
  371. except KeyError:
  372. sleep(5)
  373. res = self.site.api_query(action="query", prop="revisions",
  374. revids=revid, rvprop="content")
  375. try:
  376. return res["query"]["pages"].values()[0]["revisions"][0]["*"]
  377. except KeyError:
  378. return None
  379. def get_status_and_chart(self, content, namespace):
  380. """Determine the status and chart number of an AFC submission.
  381. The methodology used here is the same one I've been using for years
  382. (see also commands.afc_report), but with the new draft system taken
  383. into account. The order here is important: if there is more than one
  384. {{AFC submission}} template on a page, we need to know which one to
  385. use (revision history search to find the most recent isn't a viable
  386. idea :P).
  387. """
  388. statuses = self.get_statuses(content)
  389. if "R" in statuses:
  390. status, chart = "r", CHART_REVIEW
  391. elif "H" in statuses:
  392. status, chart = "p", CHART_DRAFT
  393. elif "P" in statuses:
  394. status, chart = "p", CHART_PEND
  395. elif "T" in statuses:
  396. status, chart = None, CHART_NONE
  397. elif "D" in statuses:
  398. status, chart = "d", CHART_DECLINE
  399. else:
  400. status, chart = None, CHART_NONE
  401. if namespace == wiki.NS_MAIN:
  402. if not statuses:
  403. status, chart = "a", CHART_ACCEPT
  404. else:
  405. status, chart = None, CHART_MISPLACE
  406. return status, chart
  407. def get_statuses(self, content):
  408. """Return a list of all AFC submission statuses in a page's text."""
  409. re_has_templates = "\{\{[aA][fF][cC] submission\s*(\}\}|\||/)"
  410. re_template = "\{\{[aA][fF][cC] submission\s*(.*?)\}\}"
  411. re_remove_embed = "(\{\{[aA][fF][cC] submission\s*(.*?))\{\{(.*?)\}\}(.*?)\}\}"
  412. valid = ["R", "H", "P", "T", "D"]
  413. subtemps = {
  414. "/reviewing": "R",
  415. "/onhold": "H",
  416. "/pending": "P",
  417. "/draft": "T",
  418. "/declined": "D"
  419. }
  420. statuses = []
  421. while re.search(re_has_templates, content):
  422. status = "P"
  423. match = re.search(re_template, content, re.S)
  424. if not match:
  425. return statuses
  426. temp = match.group(1)
  427. limit = 0
  428. while "{{" in temp and limit < 50:
  429. content = re.sub(re_remove_embed, "\\1\\4}}", content, 1, re.S)
  430. match = re.search(re_template, content, re.S)
  431. temp = match.group(1)
  432. limit += 1
  433. params = temp.split("|")
  434. try:
  435. subtemp, params = params[0].strip(), params[1:]
  436. except IndexError:
  437. status = "P"
  438. params = []
  439. else:
  440. if subtemp:
  441. status = subtemps.get(subtemp)
  442. params = []
  443. for param in params:
  444. param = param.strip().upper()
  445. if "=" in param:
  446. key, value = param.split("=", 1)
  447. if key.strip() == "1":
  448. status = value if value in valid else "P"
  449. break
  450. else:
  451. status = param if param in valid else "P"
  452. break
  453. statuses.append(status)
  454. content = re.sub(re_template, "", content, 1, re.S)
  455. return statuses
  456. def get_short_title(self, title):
  457. """Shorten a title so we can display it in a chart using less space.
  458. Basically, this just means removing the "Wikipedia talk:Articles for
  459. creation" part from the beginning. If it is longer than 50 characters,
  460. we'll shorten it down to 47 and add an poor-man's ellipsis at the end.
  461. """
  462. short = re.sub("Wikipedia(\s*talk)?\:Articles\sfor\screation\/", "", title)
  463. if len(short) > 50:
  464. short = "".join((short[:47], "..."))
  465. return short
  466. def get_size(self, content):
  467. """Return a page's size in a short, pretty format."""
  468. return "{0} kB".format(round(len(content) / 1000.0, 1))
  469. def get_modify(self, pageid):
  470. """Return information about a page's last edit ("modification").
  471. This consists of the most recent editor, modification time, and the
  472. lastest revision ID.
  473. """
  474. query = """SELECT rev_user_text, rev_timestamp, rev_id FROM revision
  475. JOIN page ON rev_id = page_latest WHERE page_id = ?"""
  476. result = self.site.sql_query(query, (pageid,))
  477. m_user, m_time, m_id = list(result)[0]
  478. return m_user, datetime.strptime(m_time, "%Y%m%d%H%M%S"), m_id
  479. def get_special(self, pageid, chart):
  480. """Return information about a page's "special" edit.
  481. I tend to use the term "special" as a verb a lot, which is bound to
  482. cause confusion. It is merely a short way of saying "the edit in which
  483. a declined submission was declined, an accepted submission was
  484. accepted, a submission in review was set as such, a pending submission
  485. was submitted, and a "misplaced" submission was created."
  486. This "information" consists of the special edit's editor, its time, and
  487. its revision ID. If the page's status is not something that involves
  488. "special"-ing, we will return None for all three. The same will be
  489. returned if we cannot determine when the page was "special"-ed, or if
  490. it was "special"-ed more than 250 edits ago.
  491. """
  492. if chart ==CHART_NONE:
  493. return None, None, None
  494. elif chart == CHART_MISPLACE:
  495. return self.get_create(pageid)
  496. elif chart == CHART_ACCEPT:
  497. search_for = None
  498. search_not = ["R", "H", "P", "T", "D"]
  499. elif chart == CHART_DRAFT:
  500. search_for = "H"
  501. search_not = []
  502. elif chart == CHART_PEND:
  503. search_for = "P"
  504. search_not = []
  505. elif chart == CHART_REVIEW:
  506. search_for = "R"
  507. search_not = []
  508. elif chart == CHART_DECLINE:
  509. search_for = "D"
  510. search_not = ["R", "H", "P", "T"]
  511. query = """SELECT rev_user_text, rev_timestamp, rev_id
  512. FROM revision WHERE rev_page = ? ORDER BY rev_id DESC"""
  513. result = self.site.sql_query(query, (pageid,))
  514. counter = 0
  515. last = (None, None, None)
  516. for user, ts, revid in result:
  517. counter += 1
  518. if counter > 100:
  519. msg = "Exceeded 100 content lookups while determining special for page (id: {0}, chart: {1})"
  520. self.logger.warn(msg.format(pageid, chart))
  521. return None, None, None
  522. content = self.get_revision_content(revid)
  523. statuses = self.get_statuses(content)
  524. matches = [s in statuses for s in search_not]
  525. if search_for:
  526. if search_for not in statuses or any(matches):
  527. return last
  528. else:
  529. if any(matches):
  530. return last
  531. last = (user, datetime.strptime(ts, "%Y%m%d%H%M%S"), revid)
  532. return last
  533. def get_create(self, pageid):
  534. """Return information about a page's first edit ("creation").
  535. This consists of the page creator, creation time, and the earliest
  536. revision ID.
  537. """
  538. query = """SELECT rev_user_text, rev_timestamp, rev_id
  539. FROM revision WHERE rev_id =
  540. (SELECT MIN(rev_id) FROM revision WHERE rev_page = ?)"""
  541. result = self.site.sql_query(query, (pageid,))
  542. c_user, c_time, c_id = list(result)[0]
  543. return c_user, datetime.strptime(c_time, "%Y%m%d%H%M%S"), c_id
  544. def get_notes(self, chart, content, m_time, s_user):
  545. """Return any special notes or warnings about this page.
  546. resubmit: submission was resubmitted after a previous decline
  547. short: submission is fewer than 500 bytes
  548. no-inline: submission has no inline citations
  549. unsourced: submission lacks references completely
  550. old: submission has not been touched in > 4 days
  551. blocked: submitter is currently blocked
  552. """
  553. notes = ""
  554. ignored_charts = [CHART_NONE, CHART_ACCEPT, CHART_DECLINE]
  555. if chart in ignored_charts:
  556. return notes
  557. statuses = self.get_statuses(content)
  558. if "D" in statuses and chart != CHART_MISPLACE:
  559. notes += "|nr=1" # Submission was resubmitted
  560. if len(content) < 500:
  561. notes += "|ns=1" # Submission is short
  562. if not re.search("\<ref\s*(.*?)\>(.*?)\</ref\>", content, re.I|re.S):
  563. if re.search("https?:\/\/(.*?)\.", content, re.I|re.S):
  564. notes += "|ni=1" # Submission has no inline citations
  565. else:
  566. notes += "|nu=1" # Submission is completely unsourced
  567. time_since_modify = (datetime.now() - m_time).seconds
  568. max_time = 4 * 24 * 60 * 60
  569. if time_since_modify > max_time:
  570. notes += "|no=1" # Submission hasn't been touched in over 4 days
  571. if chart in [CHART_PEND, CHART_DRAFT]:
  572. submitter = self.site.get_user(s_user)
  573. try:
  574. if submitter.blockinfo():
  575. notes += "|nb=1" # Submitter is blocked
  576. except wiki.UserNotFoundError: # Likely an IP
  577. pass
  578. return notes