Additional IRC commands and bot tasks for EarwigBot https://en.wikipedia.org/wiki/User:EarwigBot
Nelze vybrat více než 25 témat Téma musí začínat písmenem nebo číslem, může obsahovat pomlčky („-“) a může být dlouhé až 35 znaků.

afc_copyvios.py 10 KiB

před 11 roky
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232
  1. # -*- coding: utf-8 -*-
  2. #
  3. # Copyright (C) 2009-2014 Ben Kurtovic <ben.kurtovic@gmail.com>
  4. #
  5. # Permission is hereby granted, free of charge, to any person obtaining a copy
  6. # of this software and associated documentation files (the "Software"), to deal
  7. # in the Software without restriction, including without limitation the rights
  8. # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
  9. # copies of the Software, and to permit persons to whom the Software is
  10. # furnished to do so, subject to the following conditions:
  11. #
  12. # The above copyright notice and this permission notice shall be included in
  13. # all copies or substantial portions of the Software.
  14. #
  15. # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16. # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17. # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  18. # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19. # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
  20. # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  21. # SOFTWARE.
  22. from hashlib import sha256
  23. from os.path import expanduser
  24. from threading import Lock
  25. # from urllib import quote
  26. import mwparserfromhell
  27. import oursql
  28. from earwigbot.tasks import Task
  29. class AFCCopyvios(Task):
  30. """A task to check newly-edited [[WP:AFC]] submissions for copyright
  31. violations."""
  32. name = "afc_copyvios"
  33. number = 1
  34. def setup(self):
  35. cfg = self.config.tasks.get(self.name, {})
  36. self.template = cfg.get("template", "AfC suspected copyvio")
  37. self.ignore_list = cfg.get("ignoreList", [])
  38. self.min_confidence = cfg.get("minConfidence", 0.5)
  39. self.max_queries = cfg.get("maxQueries", 10)
  40. self.max_time = cfg.get("maxTime", 150)
  41. self.cache_results = cfg.get("cacheResults", False)
  42. default_summary = "Tagging suspected [[WP:COPYVIO|copyright violation]] of {url}."
  43. self.summary = self.make_summary(cfg.get("summary", default_summary))
  44. default_tags = [
  45. "Db-g12", "Db-copyvio", "Copyvio", "Copyviocore" "Copypaste"]
  46. self.tags = default_tags + cfg.get("tags", [])
  47. # Connection data for our SQL database:
  48. kwargs = cfg.get("sql", {})
  49. kwargs["read_default_file"] = expanduser("~/.my.cnf")
  50. self.conn_data = kwargs
  51. self.db_access_lock = Lock()
  52. def run(self, **kwargs):
  53. """Entry point for the bot task.
  54. Takes a page title in kwargs and checks it for copyvios, adding
  55. {{self.template}} at the top if a copyvio has been detected. A page is
  56. only checked once (processed pages are stored by page_id in an SQL
  57. database).
  58. """
  59. if self.shutoff_enabled():
  60. return
  61. title = kwargs["page"]
  62. page = self.bot.wiki.get_site().get_page(title)
  63. with self.db_access_lock:
  64. self.conn = oursql.connect(**self.conn_data)
  65. self.process(page)
  66. def process(self, page):
  67. """Detect copyvios in 'page' and add a note if any are found."""
  68. title = page.title
  69. if title in self.ignore_list:
  70. msg = u"Skipping [[{0}]], in ignore list"
  71. self.logger.info(msg.format(title))
  72. return
  73. pageid = page.pageid
  74. if self.has_been_processed(pageid):
  75. msg = u"Skipping [[{0}]], already processed"
  76. self.logger.info(msg.format(title))
  77. return
  78. code = mwparserfromhell.parse(page.get())
  79. if not self.is_pending(code):
  80. msg = u"Skipping [[{0}]], not a pending submission"
  81. self.logger.info(msg.format(title))
  82. return
  83. tag = self.is_tagged(code)
  84. if tag:
  85. msg = u"Skipping [[{0}]], already tagged with '{1}'"
  86. self.logger.info(msg.format(title, tag))
  87. return
  88. self.logger.info(u"Checking [[{0}]]".format(title))
  89. result = page.copyvio_check(self.min_confidence, self.max_queries,
  90. self.max_time)
  91. url = result.url
  92. orig_conf = "{0}%".format(round(result.confidence * 100, 2))
  93. if result.violation:
  94. if self.handle_violation(title, page, result, url, orig_conf):
  95. self._trial_reporter(title, False, url, orig_conf, result.queries, result.time, msg)
  96. self.log_processed(pageid)
  97. return
  98. else:
  99. msg = u"No violations detected in [[{0}]] (best: {1} at {2} confidence)"
  100. self.logger.info(msg.format(title, url, orig_conf))
  101. self._trial_reporter(title, False, url, orig_conf, result.queries, result.time, msg)
  102. self.log_processed(pageid)
  103. if self.cache_results:
  104. self.cache_result(page, result)
  105. def handle_violation(self, title, page, result, url, orig_conf):
  106. """Handle a page that passed its initial copyvio check."""
  107. # Things can change in the minute that it takes to do a check.
  108. # Confirm that a violation still holds true:
  109. page.load()
  110. content = page.get()
  111. tag = self.is_tagged(mwparserfromhell.parse(content))
  112. if tag:
  113. msg = u"A violation was detected in [[{0}]], but it was tagged"
  114. msg += u" in the mean time with '{1}' (best: {2} at {3} confidence)"
  115. self.logger.info(msg.format(title, tag, url, orig_conf))
  116. return True
  117. confirm = page.copyvio_compare(url, self.min_confidence)
  118. new_conf = "{0}%".format(round(confirm.confidence * 100, 2))
  119. if not confirm.violation:
  120. msg = u"A violation was detected in [[{0}]], but couldn't be confirmed."
  121. msg += u" It may have just been edited (best: {1} at {2} -> {3} confidence)"
  122. self.logger.info(msg.format(title, url, orig_conf, new_conf))
  123. return True
  124. msg = u"Found violation: [[{0}]] -> {1} ({2} confidence)"
  125. self.logger.info(msg.format(title, url, new_conf))
  126. # safeurl = quote(url.encode("utf8"), safe="/:").decode("utf8")
  127. # template = u"\{\{{0}|url={1}|confidence={2}\}\}\n"
  128. # template = template.format(self.template, safeurl, new_conf)
  129. # newtext = template + content
  130. # if "{url}" in self.summary:
  131. # page.edit(newtext, self.summary.format(url=url))
  132. # else:
  133. # page.edit(newtext, self.summary)
  134. self._trial_reporter(title, True, url, new_conf, result.queries, result.time, msg)
  135. def _trial_reporter(self, title, violation, url, conf, queries, time, msg):
  136. from datetime import datetime
  137. date = datetime.now().strftime("%Y-%m-%d %H:%M:%S UTC")
  138. data = u"\n" + ("-" * 80) + u"\n"
  139. data += u"[[{0}]] at {1}\n".format(title, date)
  140. data += u"Violation? {0}\n".format("***YES***" if violation else "No")
  141. data += u"Confidence: {0}\n".format(conf)
  142. data += u"Best match: {0}\n".format(url)
  143. data += u"Num queries: {0}\n".format(queries)
  144. data += u"Time: {0}\n".format(time)
  145. data += u"Log message: {0}\n".format(msg)
  146. with open("/data/project/earwigbot/public_html/copyvio_bot_trial.txt", "a") as fp:
  147. fp.write(data.encode("utf8"))
  148. def is_tagged(self, code):
  149. """Return whether a page contains a copyvio check template."""
  150. for template in code.ifilter_templates():
  151. for tag in self.tags:
  152. if template.name.matches(tag):
  153. return tag
  154. def is_pending(self, code):
  155. """Return whether a page is a pending AFC submission."""
  156. other_statuses = ["r", "t", "d"]
  157. tmpls = ["submit", "afc submission/submit", "afc submission/pending"]
  158. for template in code.ifilter_templates():
  159. name = template.name.strip().lower()
  160. if name == "afc submission":
  161. if not template.has(1):
  162. return True
  163. if template.get(1).value.strip().lower() not in other_statuses:
  164. return True
  165. elif name in tmpls:
  166. return True
  167. return False
  168. def has_been_processed(self, pageid):
  169. """Returns True if pageid was processed before, otherwise False."""
  170. query = "SELECT 1 FROM processed WHERE page_id = ?"
  171. with self.conn.cursor() as cursor:
  172. cursor.execute(query, (pageid,))
  173. results = cursor.fetchall()
  174. return True if results else False
  175. def log_processed(self, pageid):
  176. """Adds pageid to our database of processed pages.
  177. Raises an exception if the page has already been processed.
  178. """
  179. query = "INSERT INTO processed VALUES (?)"
  180. with self.conn.cursor() as cursor:
  181. cursor.execute(query, (pageid,))
  182. def cache_result(self, page, result):
  183. """Store the check's result in a cache table temporarily.
  184. The cache contains the page's ID, a hash of its content, the URL of the
  185. best match, the time of caching, and the number of queries used. It
  186. will replace any existing cache entries for that page.
  187. The cache is intended for EarwigBot's complementary Toolserver web
  188. interface, in which copyvio checks can be done separately from the bot.
  189. The cache saves time and money by saving the result of the web search
  190. but neither the result of the comparison nor any actual text (which
  191. could violate data retention policy). Cache entries are (intended to
  192. be) retained for three days; this task does not remove old entries
  193. (that is handled by the Toolserver component).
  194. This will only be called if ``cache_results == True`` in the task's
  195. config, which is ``False`` by default.
  196. """
  197. pageid = page.pageid
  198. hash = sha256(page.get().encode("utf8")).hexdigest()
  199. query1 = "SELECT 1 FROM cache WHERE cache_id = ?"
  200. query2 = "DELETE FROM cache WHERE cache_id = ?"
  201. query3 = "INSERT INTO cache VALUES (?, ?, ?, CURRENT_TIMESTAMP, ?, ?)"
  202. with self.conn.cursor() as cursor:
  203. cursor.execute(query1, (pageid,))
  204. if cursor.fetchall():
  205. cursor.execute(query2, (pageid,))
  206. args = (pageid, hash, result.url, result.queries, 0)
  207. cursor.execute(query3, args)