A Python robot that edits Wikipedia and interacts with people over IRC https://en.wikipedia.org/wiki/User:EarwigBot
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

afc_statistics.py 14 KiB

13 years ago
13 years ago
13 years ago
13 years ago
13 years ago
13 years ago
13 years ago
13 years ago
13 years ago
13 years ago
13 years ago
13 years ago
13 years ago
13 years ago
13 years ago
13 years ago
13 years ago
13 years ago
13 years ago
13 years ago
13 years ago
13 years ago
13 years ago
13 years ago
13 years ago
13 years ago
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365
  1. # -*- coding: utf-8 -*-
  2. from datetime import datetime
  3. import re
  4. from os.path import expanduser
  5. from threading import Lock
  6. import oursql
  7. from classes import BaseTask
  8. import config
  9. import wiki
  10. class Task(BaseTask):
  11. """A task to generate statistics for WikiProject Articles for Creation.
  12. Statistics are stored in a MySQL database ("u_earwig_afc_statistics")
  13. accessed with oursql. Statistics are updated live while watching the recent
  14. changes IRC feed and saved once an hour, on the hour, to self.pagename.
  15. In the live bot, this is "Template:AFC statistics".
  16. """
  17. name = "afc_statistics"
  18. number = 2
  19. def __init__(self):
  20. self.cfg = cfg = config.tasks.get(self.name, {})
  21. # Set some wiki-related attributes:
  22. self.pagename = cfg.get("page", "Template:AFC statistics")
  23. self.pending_cat = cfg.get("pending", "Pending_AfC_submissions")
  24. self.ignore_list = cfg.get("ignore_list", [])
  25. default_summary = "Updating statistics for [[WP:WPAFC|WikiProject Articles for creation]]."
  26. self.summary = self.make_summary(cfg.get("summary", default_summary))
  27. # Templates used in chart generation:
  28. templates = cfg.get("templates", {})
  29. self.tl_header = templates.get("header", "AFC statistics/header")
  30. self.tl_row = templates.get("row", "AFC statistics/row")
  31. self.tl_footer = templates.get("footer", "AFC statistics/footer")
  32. # Connection data for our SQL database:
  33. kwargs = cfg.get("sql", {})
  34. kwargs["read_default_file"] = expanduser("~/.my.cnf")
  35. self.conn_data = kwargs
  36. self.db_access_lock = Lock()
  37. def run(self, **kwargs):
  38. self.site = wiki.get_site()
  39. self.conn = oursql.connect(**self.conn_data)
  40. action = kwargs.get("action")
  41. if not action:
  42. return
  43. methods = {
  44. "save": self.save,
  45. "sync": self.sync,
  46. "edit": self.process_edit,
  47. "move": self.process_move,
  48. "delete": self.process_delete,
  49. "restore": self.process_edit,
  50. }
  51. method = methods.get(action)
  52. if method:
  53. try:
  54. method(**kwargs)
  55. finally:
  56. self.conn.close()
  57. def save(self, **kwargs):
  58. if kwargs.get("fromIRC"):
  59. summary = " ".join((self.summary, "(!earwigbot)"))
  60. else:
  61. if self.shutoff_enabled():
  62. return
  63. summary = self.summary
  64. statistics = self.compile_charts()
  65. page = self.site.get_page(self.pagename)
  66. text = page.get()
  67. newtext = re.sub("(<!-- stat begin -->)(.*?)(<!-- stat end -->)",
  68. statistics.join(("\\1\n", "\n\\3")), text,
  69. flags=re.DOTALL)
  70. if newtext == text:
  71. return # Don't edit the page if we're not adding anything
  72. newtext = re.sub("(<!-- sig begin -->)(.*?)(<!-- sig end -->)",
  73. "\\1~~~ at ~~~~~\\3", newtext)
  74. page.edit(newtext, summary, minor=True, bot=True)
  75. def compile_charts(self):
  76. stats = ""
  77. with self.conn.cursor() as cursor, self.db_access_lock:
  78. cursor.execute("SELECT * FROM chart")
  79. for chart in cursor:
  80. stats += self.compile_chart(chart) + "\n"
  81. return stats[:-1] # Drop the last newline
  82. def compile_chart(self, chart_info):
  83. chart_id, chart_title, special_title = chart_info
  84. chart = "|".join((self.tl_header, chart_title))
  85. if special_title:
  86. chart += "".join(("|", special_title))
  87. chart = "".join(("{{", chart, "}}"))
  88. query = "SELECT * FROM page JOIN row ON page_id = row_id WHERE row_chart = ?"
  89. with self.conn.cursor(oursql.DictCursor) as cursor:
  90. cursor.execute(query, (chart_id,))
  91. for page in cursor:
  92. chart += "\n" + self.compile_chart_row(page)
  93. chart += "".join(("\n{{", self.tl_footer, "}}"))
  94. return chart
  95. def compile_chart_row(self, page):
  96. row = "{0}|s={page_status}|t={page_title}|h={page_short}|z={page_size}|"
  97. row += "cr={page_create_user}|cd={page_create_time}|ci={page_create_oldid}|"
  98. row += "mr={page_modify_user}|md={page_modify_time}|mi={page_modify_oldid}|"
  99. page["page_create_time"] = self.format_time(page["page_create_time"])
  100. page["page_modify_time"] = self.format_time(page["page_modify_time"])
  101. if page["page_special_user"]:
  102. row += "sr={page_special_user}|sd={page_special_time}|si={page_special_oldid}|"
  103. page["page_special_time"] = self.format_time(page["page_special_time"])
  104. if page["page_notes"]:
  105. row += "n=1{page_notes}"
  106. return "".join(("{{", row.format(self.tl_row, **page), "}}"))
  107. def format_time(self, timestamp):
  108. return timestamp.strftime("%H:%M, %d %B %Y")
  109. def sync(self, **kwargs):
  110. with self.conn.cursor() as cursor, self.db_access_lock:
  111. self.sync_deleted(cursor) # Remove deleted subs
  112. self.sync_oldids(cursor) # Make sure all subs are up to date
  113. self.sync_pending(cursor) # Add missing pending subs
  114. self.sync_old(cursor) # Remove old declined and accepted subs
  115. def sync_deleted(self, cursor):
  116. query1 = "SELECT page_id FROM page"
  117. query2 = "SELECT page_id FROM page WHERE page_id = ?"
  118. cursor.execute(query1)
  119. for page in cursor:
  120. result = self.site.sql_query(query2, (page[0],))
  121. if not list(result):
  122. self.untrack_page(cursor, pageid=page[0])
  123. def sync_oldids(self, cursor):
  124. query1 = "SELECT page_id, page_title, page_modify_oldid FROM page"
  125. query2 = "SELECT page_latest, page_title FROM page WHERE page_id = ?"
  126. cursor.execute(query1)
  127. for page_id, title, oldid in cursor:
  128. result = list(self.site.sql_query(query2, (page_id,)))
  129. try:
  130. real_oldid = result[0][0]
  131. real_title = result[0][1]
  132. except IndexError: # Page doesn't exist!
  133. self.untrack_page(cursor, pageid=page_id)
  134. continue
  135. if real_oldid != oldid:
  136. self.update_page(cursor, real_title)
  137. def sync_pending(self, cursor):
  138. query1 = """SELECT page_id FROM page JOIN row ON page_id = row_id
  139. WHERE row_chart IN (1, 2, 3)"""
  140. query2 = """SELECT cl_from, page_title, page_namespace
  141. FROM categorylinks JOIN page ON cl_from = page_id
  142. WHERE cl_to = ?"""
  143. cursor.execute(query1)
  144. tracked = [i[0] for i in cursor.fetchall()]
  145. result = self.site.sql_query(query2, (self.pending_cat,))
  146. for pageid, title, ns in result:
  147. title = ":".join((self.site.namespace_id_to_name(ns), title))
  148. if title.replace("_", " ") in self.ignore_list:
  149. continue
  150. if pageid not in tracked:
  151. self.track_page(cursor, title)
  152. def sync_old(self, cursor):
  153. query = """DELETE FROM page, row USING page JOIN row
  154. ON page_id = row_id WHERE row_chart IN (4, 5)
  155. AND ADDTIME(page_special_time, '36:00:00') < NOW()"""
  156. cursor.execute(query)
  157. def process_edit(self, page, **kwargs):
  158. if page in self.ignore_list:
  159. return
  160. with self.conn.cursor() as cursor, self.db_access_lock:
  161. self.sync_page(cursor, page)
  162. def process_move(self, page, **kwargs):
  163. query1 = "SELECT * FROM page WHERE page_title = ?"
  164. query2 = "SELECT page_latest FROM page WHERE page_namespace = ? AND page_title = ?"
  165. query3 = "UPDATE page SET page_title = ?, page_modify_oldid = ? WHERE page_title = ?"
  166. source, dest = page
  167. with self.conn.cursor() as cursor, self.db_access_lock:
  168. cursor.execute(query1, (source,))
  169. result = cursor.fetchall()
  170. if result:
  171. res = self.site.sql_query(query2, self.split_title(dest))
  172. try:
  173. new_oldid = list(res)[0][0]
  174. except IndexError:
  175. new_oldid = result[0][11]
  176. cursor.execute(query3, (dest, new_oldid, source))
  177. else:
  178. self.track_page(cursor, dest)
  179. def process_delete(self, page, **kwargs):
  180. query = "SELECT page_id FROM page WHERE page_namespace = ? AND page_title = ?"
  181. with self.conn.cursor() as cursor, self.db_access_lock:
  182. result = self.site.sql_query(query, self.split_title(page))
  183. if list(result):
  184. self.sync_page(cursor, page)
  185. else:
  186. self.untrack_page(cursor, title=page)
  187. def sync_page(self, cursor, page):
  188. query = "SELECT * FROM page WHERE page_title = ?"
  189. cursor.execute(query, (page,))
  190. result = cursor.fetchall()
  191. if result:
  192. self.update_page(cursor, page)
  193. else:
  194. self.track_page(cursor, page)
  195. def untrack_page(self, cursor, pageid=None, title=None):
  196. query = "DELETE FROM page, row USING page JOIN row ON page_id = row_id WHERE "
  197. if pageid:
  198. query += "page_id = ?"
  199. cursor.execute(query, (pageid,))
  200. elif title:
  201. query += "page_title = ?"
  202. cursor.execute(query, (title,))
  203. def track_page(self, cursor, title):
  204. """Update hook for when page is not in our database."""
  205. page = self.site.get_page(title)
  206. status, chart = self.get_status_and_chart(page)
  207. if not status or status in ("accept", "decline"):
  208. return
  209. pageid = page.pageid()
  210. title = page.title()
  211. short = self.get_short_title(title)
  212. size = len(page.get())
  213. notes = self.get_notes(page)
  214. c_user, c_time, c_id = self.get_create(pageid)
  215. m_user, m_time, m_id = self.get_modify(pageid)
  216. s_user, s_time, s_id = self.get_special(page, status)
  217. query1 = "INSERT INTO row VALUES (?, ?)"
  218. query2 = "INSERT INTO page VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"
  219. cursor.execute(query1, (pageid, chart))
  220. cursor.execute(query2, (pageid, status, title, short, size, notes,
  221. c_user, c_time, c_id, m_user, m_time, m_id,
  222. s_user, s_time, s_id))
  223. def update_page(self, cursor, title):
  224. """Update hook for when page is in our database."""
  225. page = self.site.get_page(title)
  226. status, chart = self.get_status_and_chart(page)
  227. if not status:
  228. self.untrack_page(cursor, title=title)
  229. pageid = page.pageid()
  230. title = page.title()
  231. size = len(page.get())
  232. notes = self.get_notes(page)
  233. m_user, m_time, m_id = self.get_modify(pageid)
  234. query = "SELECT * FROM page JOIN row ON page_id = row_id WHERE page_id = ?"
  235. with self.conn.cursor(oursql.DictCursor) as dict_cursor:
  236. dict_cursor.execute(query, (pageid,))
  237. result = dict_cursor.fetchall()[0]
  238. if title != result["page_title"]:
  239. query = "UPDATE page SET page_title = ?, page_short = ? WHERE page_id = ?"
  240. short = self.get_short_title(title)
  241. cursor.execute(query, (title, short, pageid))
  242. if m_id != result["page_modify_oldid"]:
  243. query = """UPDATE page SET page_size = ?, page_modify_user = ?,
  244. page_modify_time = ?, page_modify_oldid = ?
  245. WHERE page_id = ?"""
  246. cursor.execute(query, (size, m_user, m_time, m_id, pageid))
  247. if status != result["page_status"]:
  248. query1 = """UPDATE page JOIN row ON page_id = row_id
  249. SET page_status = ?, row_chart = ? WHERE page_id = ?"""
  250. query2 = """UPDATE page SET page_special_user = ?,
  251. page_special_time = ?, page_special_oldid = ?
  252. WHERE page_id = ?"""
  253. cursor.execute(query1, (status, chart, pageid))
  254. s_user, s_time, s_id = self.get_special(page, status)
  255. if s_id != result["page_special_oldid"]:
  256. cursor.execute(query2, (s_user, s_time, s_id, pageid))
  257. if notes != result["page_notes"]:
  258. query = "UPDATE page SET page_notes = ? WHERE page_id = ?"
  259. cursor.execute(query, (notes, pageid))
  260. def split_title(self, title):
  261. namespace, body = title.split(":", 1)[0]
  262. if not body:
  263. return 0, title
  264. try:
  265. ns = self.site.namespace_name_to_id(namespace)
  266. except wiki.NamespaceNotFoundError:
  267. return 0, title
  268. return ns, body
  269. def get_status_and_chart(self, page):
  270. content = page.get()
  271. if page.is_redirect():
  272. target = page.get_redirect_target()
  273. if self.site.get_page(target).namespace() == 0:
  274. return "accept", 4
  275. return None, 0
  276. elif re.search("\{\{afc submission\|r\|(.*?)\}\}", content, re.I):
  277. return "review", 3
  278. elif re.search("\{\{afc submission\|h\|(.*?)\}\}", content, re.I):
  279. return "pend", 2
  280. elif re.search("\{\{afc submission\|\|(.*?)\}\}", content, re.I):
  281. return "pend", 1
  282. elif re.search("\{\{afc submission\|t\|(.*?)\}\}", content, re.I):
  283. return None, 0
  284. elif re.search("\{\{afc submission\|d\|(.*?)\}\}", content, re.I):
  285. return "decline", 5
  286. return None, 0
  287. def get_short_title(self, title):
  288. short = re.sub("Wikipedia(\s*talk)?\:Articles\sfor\screation\/", "", title)
  289. if len(short) > 50:
  290. short = "".join((short[:47], "..."))
  291. return short
  292. def get_create(self, pageid):
  293. query1 = "SELECT MIN(rev_id) FROM revision WHERE rev_page = ?"
  294. query2 = "SELECT rev_user_text, rev_timestamp, rev_id FROM revision WHERE rev_id = ?"
  295. result1 = self.site.sql_query(query1, (pageid,))
  296. rev_id = list(result1)[0][0]
  297. result2 = self.site.sql_query(query2, (rev_id,))
  298. m_user, m_time, m_id = list(result2)[0]
  299. return m_user, datetime.strptime(m_time, "%Y%m%d%H%M%S"), m_id
  300. def get_modify(self, pageid):
  301. query = """SELECT rev_user_text, rev_timestamp, rev_id FROM revision
  302. JOIN page ON rev_id = page_latest WHERE page_id = ?"""
  303. result = self.site.sql_query(query, (pageid,))
  304. m_user, m_time, m_id = list(result)[0]
  305. return m_user, datetime.strptime(m_time, "%Y%m%d%H%M%S"), m_id
  306. def get_special(self, page, status):
  307. return None, None, None
  308. def get_notes(self, page):
  309. return None