A copyright violation detector running on Wikimedia Cloud Services https://tools.wmflabs.org/copyvios/
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
 
 
 
 
 

162 lines
4.4 KiB

  1. #! /usr/bin/env python
  2. import functools
  3. import hashlib
  4. import json
  5. import logging
  6. import os
  7. import time
  8. import traceback
  9. from collections.abc import Callable
  10. from logging.handlers import TimedRotatingFileHandler
  11. from typing import Any, ParamSpec
  12. from earwigbot.wiki.copyvios import globalize
  13. from flask import Flask, Response, make_response, request
  14. from flask_mako import MakoTemplates, TemplateError, render_template
  15. from copyvios.api import format_api_error, handle_api_request
  16. from copyvios.cache import cache
  17. from copyvios.checker import CopyvioCheckError, do_check
  18. from copyvios.cookies import get_new_cookies
  19. from copyvios.misc import get_notice
  20. from copyvios.query import CheckQuery
  21. from copyvios.settings import process_settings
  22. from copyvios.sites import update_sites
  23. app = Flask(__name__)
  24. MakoTemplates(app)
  25. hand = TimedRotatingFileHandler("logs/app.log", when="midnight", backupCount=7)
  26. hand.setLevel(logging.DEBUG)
  27. app.logger.addHandler(hand)
  28. app.logger.info(f"Flask server started {time.asctime()}")
  29. globalize(num_workers=8)
  30. AnyResponse = Response | str | bytes
  31. P = ParamSpec("P")
  32. def catch_errors(func: Callable[P, AnyResponse]) -> Callable[P, AnyResponse]:
  33. @functools.wraps(func)
  34. def inner(*args: P.args, **kwargs: P.kwargs) -> AnyResponse:
  35. try:
  36. return func(*args, **kwargs)
  37. except TemplateError as exc:
  38. app.logger.error(f"Caught exception:\n{exc.text}")
  39. return render_template("error.mako", traceback=exc.text)
  40. except Exception:
  41. app.logger.exception("Caught exception:")
  42. return render_template("error.mako", traceback=traceback.format_exc())
  43. return inner
  44. @app.after_request
  45. def add_new_cookies(response: Response) -> Response:
  46. for cookie in get_new_cookies():
  47. response.headers.add("Set-Cookie", cookie)
  48. return response
  49. @app.after_request
  50. def write_access_log(response: Response) -> Response:
  51. app.logger.debug(
  52. f"{time.asctime()} {request.method} {request.path} "
  53. f"{request.values.to_dict()} -> {response.status_code}"
  54. )
  55. return response
  56. @functools.lru_cache
  57. def _get_hash(path: str, mtime: float) -> str:
  58. # mtime is used as part of the cache key
  59. with open(path, "rb") as fp:
  60. return hashlib.sha1(fp.read()).hexdigest()
  61. def external_url_handler(
  62. error: Exception, endpoint: str, values: dict[str, Any]
  63. ) -> str:
  64. if endpoint == "static" and "file" in values:
  65. assert app.static_folder is not None
  66. path = os.path.join(app.static_folder, values["file"])
  67. mtime = os.path.getmtime(path)
  68. hashstr = _get_hash(path, mtime)
  69. return f"/static/{values['file']}?v={hashstr}"
  70. raise error
  71. app.url_build_error_handlers.append(external_url_handler)
  72. @app.route("/")
  73. @catch_errors
  74. def index() -> AnyResponse:
  75. notice = get_notice()
  76. update_sites()
  77. query = CheckQuery.from_get_args()
  78. try:
  79. result = do_check(query)
  80. error = None
  81. except CopyvioCheckError as exc:
  82. result = None
  83. error = exc
  84. return render_template(
  85. "index.mako",
  86. notice=notice,
  87. query=query,
  88. result=result,
  89. error=error,
  90. )
  91. @app.route("/settings", methods=["GET", "POST"])
  92. @catch_errors
  93. def settings() -> AnyResponse:
  94. status = process_settings() if request.method == "POST" else None
  95. update_sites()
  96. default = cache.bot.wiki.get_site()
  97. kwargs = {
  98. "status": status,
  99. "default_lang": default.lang,
  100. "default_project": default.project,
  101. }
  102. return render_template("settings.mako", **kwargs)
  103. @app.route("/api")
  104. @catch_errors
  105. def api() -> AnyResponse:
  106. return render_template("api.mako", help=True)
  107. @app.route("/api.json")
  108. @catch_errors
  109. def api_json() -> AnyResponse:
  110. if not request.args:
  111. return render_template("api.mako", help=True)
  112. format = request.args.get("format", "json")
  113. if format in ["json", "jsonfm"]:
  114. update_sites()
  115. try:
  116. result = handle_api_request()
  117. except Exception as exc:
  118. result = format_api_error("unhandled_exception", exc)
  119. else:
  120. errmsg = f"Unknown format: {format!r}"
  121. result = format_api_error("unknown_format", errmsg)
  122. if format == "jsonfm":
  123. return render_template("api.mako", help=False, result=result)
  124. resp = make_response(json.dumps(result))
  125. resp.mimetype = "application/json"
  126. resp.headers["Access-Control-Allow-Origin"] = "*"
  127. return resp
  128. if __name__ == "__main__":
  129. app.run()