Additional IRC commands and bot tasks for EarwigBot https://en.wikipedia.org/wiki/User:EarwigBot
Nelze vybrat více než 25 témat Téma musí začínat písmenem nebo číslem, může obsahovat pomlčky („-“) a může být dlouhé až 35 znaků.

rc_monitor.py 8.9 KiB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248
  1. # -*- coding: utf-8 -*-
  2. #
  3. # Copyright (C) 2016 Ben Kurtovic <ben.kurtovic@gmail.com>
  4. #
  5. # Permission is hereby granted, free of charge, to any person obtaining a copy
  6. # of this software and associated documentation files (the "Software"), to deal
  7. # in the Software without restriction, including without limitation the rights
  8. # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
  9. # copies of the Software, and to permit persons to whom the Software is
  10. # furnished to do so, subject to the following conditions:
  11. #
  12. # The above copyright notice and this permission notice shall be included in
  13. # all copies or substantial portions of the Software.
  14. #
  15. # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16. # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17. # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  18. # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19. # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
  20. # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  21. # SOFTWARE.
  22. from collections import namedtuple
  23. from datetime import datetime
  24. from difflib import ndiff
  25. from Queue import Queue
  26. import re
  27. from threading import Thread
  28. from earwigbot.commands import Command
  29. from earwigbot.exceptions import APIError
  30. from earwigbot.irc import RC
  31. from earwigbot.wiki import constants
  32. _Diff = namedtuple("_Diff", ["added", "removed"])
  33. class RCMonitor(Command):
  34. """Monitors the recent changes feed for certain edits and reports them to a
  35. dedicated channel."""
  36. name = "rc_monitor"
  37. commands = ["rc_monitor", "rcm"]
  38. hooks = ["msg", "rc"]
  39. def setup(self):
  40. try:
  41. self._channel = self.config.commands[self.name]["channel"]
  42. except KeyError:
  43. self._channel = None
  44. log = ('Cannot use without a report channel set as '
  45. 'config.commands["{0}"]["channel"]')
  46. self.logger.warn(log.format(self.name))
  47. self._stats = {
  48. "start": datetime.utcnow(),
  49. "edits": 0,
  50. "hits": 0,
  51. "max_backlog": 0
  52. }
  53. self._levels = {}
  54. self._issues = {}
  55. self._descriptions = {}
  56. self._redirects = {}
  57. self._queue = Queue()
  58. self._thread = Thread(target=self._callback, name="rc_monitor")
  59. self._thread.daemon = True
  60. self._thread.running = True
  61. self._prepare_reports()
  62. self._thread.start()
  63. def check(self, data):
  64. if not self._channel:
  65. return
  66. return isinstance(data, RC) or (
  67. data.is_command and data.command in self.commands)
  68. def process(self, data):
  69. if isinstance(data, RC):
  70. newlen = self._queue.qsize() + 1
  71. self._queue.put(data)
  72. if newlen > self._stats["max_backlog"]:
  73. self._stats["max_backlog"] = newlen
  74. return
  75. if not self.config.irc["permissions"].is_admin(data):
  76. self.reply(data, "You must be a bot admin to use this command.")
  77. return
  78. since = self._stats["start"].strftime("%H:%M:%S, %d %B %Y")
  79. seconds = (datetime.utcnow() - self._stats["start"]).total_seconds()
  80. rate = self._stats["edits"] / seconds
  81. msg = ("\x02{edits:,}\x0F edits checked since {since} "
  82. "(\x02{rate:.2f}\x0F edits/sec); \x02{hits:,}\x0F hits; "
  83. "\x02{qsize:,}\x0F-edit backlog (\x02{max_backlog:,}\x0F max).")
  84. self.reply(data, msg.format(
  85. since=since, rate=rate, qsize=self._queue.qsize(), **self._stats))
  86. def unload(self):
  87. self._thread.running = False
  88. self._queue.put(None)
  89. def _prepare_reports(self):
  90. """Set up internal tables for storing report information."""
  91. routine = 1
  92. alert = 2
  93. urgent = 3
  94. self._levels = {
  95. routine: "routine",
  96. alert: "alert",
  97. urgent: "URGENT"
  98. }
  99. self._issues = {
  100. "g10": alert
  101. }
  102. self._descriptions = {
  103. "g10": "CSD G10 nomination"
  104. }
  105. def _get_diff(self, oldrev, newrev):
  106. """Return the difference between two revisions.
  107. A diff is a 2-tuple: (list of lines added, list of lines removed).
  108. """
  109. site = self.bot.wiki.get_site()
  110. try:
  111. result = site.api_query(
  112. action="query", prop="revisions", rvprop="ids|content",
  113. rvslots="main",
  114. revids=(oldrev + "|" + newrev) if oldrev else newrev)
  115. except APIError:
  116. return None
  117. try:
  118. pages = result["query"]["pages"].values()
  119. except IndexError:
  120. return None
  121. if len(pages) != 1:
  122. return None
  123. revs = pages[0]["revisions"]
  124. if not oldrev:
  125. try:
  126. text = revs[0]["slots"]["main"]["*"]
  127. except (IndexError, KeyError):
  128. return None
  129. return _Diff(text.splitlines(), [])
  130. try:
  131. oldtext = [rv["slots"]["main"]["*"] for rv in revs
  132. if rv["revid"] == int(oldrev)][0]
  133. newtext = [rv["slots"]["main"]["*"] for rv in revs
  134. if rv["revid"] == int(newrev)][0]
  135. except (IndexError, KeyError):
  136. return None
  137. lines = list(ndiff(oldtext.splitlines(), newtext.splitlines()))
  138. added = [line[2:] for line in lines if line[:2] == "+ "]
  139. removed = [line[2:] for line in lines if line[:2] == "- "]
  140. return _Diff(added, removed)
  141. def _fetch_redirects(self, template):
  142. """Return a list of valid names for a given template."""
  143. site = self.bot.wiki.get_site()
  144. try:
  145. result = site.api_query(
  146. action="query", list="backlinks", blfilterredir="redirects",
  147. blnamespace=constants.NS_TEMPLATE, bllimit=50,
  148. bltitle="Template:" + template)
  149. except APIError:
  150. return []
  151. redirs = {link["title"].split(":", 1)[1].lower()
  152. for link in result["query"]["backlinks"]}
  153. redirs.add(template)
  154. return redirs
  155. def _fast_template_search(self, template):
  156. """Return a compiled regular expression for matching templates."""
  157. if template not in self._redirects:
  158. redirects = self._fetch_redirects(template)
  159. if not redirects:
  160. return None
  161. self._redirects[template] = redirects
  162. search = "|".join(r"(template:)?" + re.escape(tmpl).replace(r"\ ", r"[ _]")
  163. for tmpl in self._redirects[template])
  164. return re.compile(r"\{\{\s*(" + search + r")\s*(\||\}\})", re.U|re.I)
  165. def _evaluate_csd(self, diff):
  166. """Evaluate a diff for CSD tagging."""
  167. regex = self._fast_template_search("db-g10")
  168. if not regex:
  169. return []
  170. if any(regex.search(line) for line in diff.added):
  171. return ["g10"]
  172. return []
  173. def _evaluate(self, event):
  174. """Return heuristic information about the given RC event."""
  175. oldrev = re.search(r"(?:\?|&)oldid=(.*?)(?:&|$)", event.url)
  176. newrev = re.search(r"(?:\?|&)diff=(.*?)(?:&|$)", event.url)
  177. if not oldrev:
  178. return []
  179. if newrev:
  180. diff = self._get_diff(oldrev.group(1), newrev.group(1))
  181. else:
  182. diff = self._get_diff(None, oldrev.group(1))
  183. if not diff:
  184. return []
  185. issues = []
  186. issues.extend(self._evaluate_csd(diff))
  187. issues.sort(key=lambda issue: self._issues[issue], reverse=True)
  188. return issues
  189. def _format(self, rc, report):
  190. """Format a RC event for the report channel."""
  191. level = self._levels[max(self._issues[issue] for issue in report)]
  192. descr = ", ".join(self._descriptions[issue] for issue in report)
  193. notify = " ".join("!rcm-" + issue for issue in report)
  194. cmnt = rc.comment if len(rc.comment) <= 50 else rc.comment[:47] + "..."
  195. msg = ("[\x02{level}\x0F] ({descr}) [\x02{notify}\x0F]\x0306 * "
  196. "\x0314[[\x0307{title}\x0314]]\x0306 * \x0303{user}\x0306 * "
  197. "\x0302{url}\x0306 * \x0310{comment}")
  198. return msg.format(
  199. level=level, descr=descr, notify=notify, title=rc.page,
  200. user=rc.user, url=rc.url, comment=cmnt)
  201. def _handle_event(self, event):
  202. """Process a recent change event."""
  203. if not event.is_edit or "B" in event.flags:
  204. return
  205. report = self._evaluate(event)
  206. self._stats["edits"] += 1
  207. if report:
  208. self.say(self._channel, self._format(event, report))
  209. self._stats["hits"] += 1
  210. def _callback(self):
  211. """Internal callback for the RC monitor thread."""
  212. while self._thread.running:
  213. event = self._queue.get()
  214. if not self._thread.running:
  215. break
  216. self._handle_event(event)