Mostly finish Jinja refactoring

README.md

@@ -32,10 +32,14 @@ Installation
   In `.earwigbot/config.yml`, fill out the connection info for the database by
   adding the following to the `wiki` section:
 
-        copyvios:
-            engine: mysql
-            host: <hostname of database server>
-            db: <name of database>
+      copyvios:
+        oauth:
+          consumer_token: <oauth consumer token>
+          consumer_secret: <oauth consumer secret>
+        sql:
+          engine: mysql
+          host: <hostname of database server>
+          db: <name of database>
 
 Running
 =======

app.py

@@ -3,54 +3,63 @@
 import functools
 import hashlib
 import json
-import logging
 import os
 import time
 import traceback
-from collections.abc import Callable
-from logging.handlers import TimedRotatingFileHandler
-from typing import Any, ParamSpec
+from typing import Any
 
 from earwigbot.wiki.copyvios import globalize
-from flask import Flask, Response, make_response, request
-from flask_mako import MakoTemplates, TemplateError, render_template
+from flask import Response, make_response, render_template, request
 
+from copyvios import app
 from copyvios.api import format_api_error, handle_api_request
+from copyvios.attribution import get_attribution_info
+from copyvios.background import get_background
 from copyvios.cache import cache
-from copyvios.checker import CopyvioCheckError, do_check
-from copyvios.cookies import get_new_cookies
-from copyvios.misc import get_notice
+from copyvios.checker import (
+    T_POSSIBLE,
+    T_SUSPECT,
+    CopyvioCheckError,
+    ErrorCode,
+    do_check,
+)
+from copyvios.cookies import get_cookies, get_new_cookies
+from copyvios.highlighter import highlight_delta
+from copyvios.misc import get_notice, get_permalink
 from copyvios.query import CheckQuery
 from copyvios.settings import process_settings
 from copyvios.sites import update_sites
 
-app = Flask(__name__)
-MakoTemplates(app)
+AnyResponse = Response | str | bytes
 
-hand = TimedRotatingFileHandler("logs/app.log", when="midnight", backupCount=7)
-hand.setLevel(logging.DEBUG)
-app.logger.addHandler(hand)
 app.logger.info(f"Flask server started {time.asctime()}")
 
 globalize(num_workers=8)
 
-AnyResponse = Response | str | bytes
-P = ParamSpec("P")
-
 
-def catch_errors(func: Callable[P, AnyResponse]) -> Callable[P, AnyResponse]:
-    @functools.wraps(func)
-    def inner(*args: P.args, **kwargs: P.kwargs) -> AnyResponse:
-        try:
-            return func(*args, **kwargs)
-        except TemplateError as exc:
-            app.logger.error(f"Caught exception:\n{exc.text}")
-            return render_template("error.mako", traceback=exc.text)
-        except Exception:
-            app.logger.exception("Caught exception:")
-            return render_template("error.mako", traceback=traceback.format_exc())
-
-    return inner
+@app.errorhandler(Exception)
+def handle_errors(exc: Exception) -> AnyResponse:
+    if app.debug:
+        raise  # Use built-in debugger
+    app.logger.exception("Caught exception:")
+    return render_template("error.html.jinja", traceback=traceback.format_exc())
+
+
+@app.context_processor
+def setup_context() -> dict[str, Any]:
+    return {
+        "T_POSSIBLE": T_POSSIBLE,
+        "T_SUSPECT": T_SUSPECT,
+        "ErrorCode": ErrorCode,
+        "cache": cache,
+        "dump_json": json.dumps,
+        "get_attribution_info": get_attribution_info,
+        "get_background": get_background,
+        "get_cookies": get_cookies,
+        "get_notice": get_notice,
+        "get_permalink": get_permalink,
+        "highlight_delta": highlight_delta,
+    }
 
 
 @app.after_request
@@ -92,51 +101,47 @@ app.url_build_error_handlers.append(external_url_handler)
 
 
 @app.route("/")
-@catch_errors
 def index() -> AnyResponse:
-    notice = get_notice()
     update_sites()
     query = CheckQuery.from_get_args()
     try:
         result = do_check(query)
         error = None
     except CopyvioCheckError as exc:
+        app.logger.exception(f"Copyvio check failed on {query}")
         result = None
         error = exc
+
     return render_template(
-        "index.mako",
-        notice=notice,
+        "index.html.jinja",
         query=query,
         result=result,
         error=error,
+        splash=not result,
     )
 
 
 @app.route("/settings", methods=["GET", "POST"])
-@catch_errors
 def settings() -> AnyResponse:
     status = process_settings() if request.method == "POST" else None
     update_sites()
-    default = cache.bot.wiki.get_site()
-    kwargs = {
-        "status": status,
-        "default_lang": default.lang,
-        "default_project": default.project,
-    }
-    return render_template("settings.mako", **kwargs)
+    return render_template(
+        "settings.html.jinja",
+        status=status,
+        default_site=cache.bot.wiki.get_site(),
+        splash=True,
+    )
 
 
 @app.route("/api")
-@catch_errors
 def api() -> AnyResponse:
-    return render_template("api.mako", help=True)
+    return render_template("api_help.html.jinja")
 
 
 @app.route("/api.json")
-@catch_errors
 def api_json() -> AnyResponse:
     if not request.args:
-        return render_template("api.mako", help=True)
+        return render_template("api_help.html.jinja")
 
     format = request.args.get("format", "json")
     if format in ["json", "jsonfm"]:
@@ -144,18 +149,26 @@ def api_json() -> AnyResponse:
         try:
             result = handle_api_request()
         except Exception as exc:
+            app.logger.exception("API request failed")
             result = format_api_error("unhandled_exception", exc)
     else:
         errmsg = f"Unknown format: {format!r}"
         result = format_api_error("unknown_format", errmsg)
 
     if format == "jsonfm":
-        return render_template("api.mako", help=False, result=result)
+        return render_template("api_result.html.jinja", result=result)
     resp = make_response(json.dumps(result))
     resp.mimetype = "application/json"
     resp.headers["Access-Control-Allow-Origin"] = "*"
     return resp
 
 
+if app.debug:
+    # Silence browser 404s when testing
+    @app.route("/favicon.ico")
+    def favicon() -> AnyResponse:
+        return app.send_static_file("favicon.ico")
+
+
 if __name__ == "__main__":
     app.run()

pyproject.toml

@@ -11,8 +11,6 @@ dependencies = [
     "earwigbot[sql,copyvios] >= 0.4",
     "mwparserfromhell >= 0.6",
     "flask >= 3.0",
-    "flask-mako >= 0.4",
-    "mako >= 1.3.5",
     "requests >= 2.32.3",
     "pydantic >= 2.9.2",
     "SQLAlchemy >= 2.0.32",

src/copyvios/__init__.py

@@ -0,0 +1,13 @@
+import logging
+from logging.handlers import TimedRotatingFileHandler
+
+from flask import Flask
+
+app = Flask("copyvios")
+
+app.jinja_options["trim_blocks"] = True
+app.jinja_options["lstrip_blocks"] = True
+
+hand = TimedRotatingFileHandler("logs/app.log", when="midnight", backupCount=7)
+hand.setLevel(logging.DEBUG)
+app.logger.addHandler(hand)

src/copyvios/api.py

@@ -62,8 +62,8 @@ def _serialize_detail(result: CopyvioCheckResult) -> dict[str, Any] | None:
     if not result.best:
         return None
     source_chain, delta = result.best.chains
-    article = highlight_delta(None, result.article_chain, delta)
-    source = highlight_delta(None, source_chain, delta)
+    article = highlight_delta(result.article_chain, delta)
+    source = highlight_delta(source_chain, delta)
     return {"article": article, "source": source}
 
 
@@ -136,7 +136,11 @@ def _hook_check(query: APIQuery) -> dict[str, Any]:
 
 def _hook_sites(query: APIQuery) -> dict[str, Any]:
     update_sites()
-    return {"status": "ok", "langs": cache.langs, "projects": cache.projects}
+    return {
+        "status": "ok",
+        "langs": [[lang.code, lang.name] for lang in cache.langs],
+        "projects": [[project.code, project.name] for project in cache.projects],
+    }
 
 
 _HOOKS = {

src/copyvios/background.py

@@ -13,7 +13,6 @@ from typing import Self
 
 from earwigbot import exceptions
 from earwigbot.wiki import Site
-from flask import g
 
 from .cache import cache
 from .cookies import get_cookies
@@ -79,16 +78,20 @@ def _get_fresh_from_potd() -> BackgroundInfo | None:
     site = _get_commons_site()
     date = datetime.now(UTC).strftime("%Y-%m-%d")
     page = site.get_page(f"Template:Potd/{date}")
-    regex = r"\{\{Potd filename\|(?:1=)?(.*?)\|.*?\}\}"
+    filename = None
     try:
-        match = re.search(regex, page.get())
+        code = page.parse()
+        for tmpl in code.ifilter_templates(
+            matches=lambda tmpl: tmpl.name.matches("Potd filename")
+        ):
+            filename = tmpl.get(1).value.strip_code().strip()
+            break
     except exceptions.EarwigBotError:
         logger.exception(f"Failed to load today's POTD from {page.title!r}")
         return None
-    if not match:
+    if not filename:
         logger.exception(f"Failed to extract POTD from {page.title!r}")
         return None
-    filename = match.group(1)
     return _load_file(site, filename)
 
 
@@ -144,7 +147,10 @@ def _get_background(selected: str) -> BackgroundInfo | None:
     return _BACKGROUND_CACHE[selected]
 
 
-def get_background(selected: str) -> str:
+def get_background(selected: str) -> tuple[str | None, str | None]:
+    if selected == "plain":
+        return None, None
+
     cookies = get_cookies()
     if "CopyviosScreenCache" in cookies:
         cookie = cookies["CopyviosScreenCache"].value
@@ -155,8 +161,6 @@ def get_background(selected: str) -> str:
     background = _get_background(selected)
     if background:
         bg_url = _build_url(screen, background)
-        g.descurl = background.descurl
+        return bg_url, background.descurl
     else:
-        bg_url = ""
-        g.descurl = None
-    return bg_url
+        return None, None

src/copyvios/cache.py

@@ -38,7 +38,7 @@ def setup_connection(dbapi_connection: Any, connection_record: Any) -> None:
 
 
 def _get_engine(bot: Bot) -> sqlalchemy.Engine:
-    args = bot.config.wiki["copyvios"].copy()
+    args = bot.config.wiki.get("copyvios", {}).get("sql", {}).copy()
     engine_name = args.pop("engine", "mysql").lower()
 
     if engine_name == "mysql":

src/copyvios/checker.py

@@ -101,8 +101,7 @@ def _get_results(
             result = _perform_check(query, page, conn)
         finally:
             conn.close()
-        if turnitin_result:
-            result.metadata.turnitin_result = turnitin_result
+        result.metadata.turnitin_result = turnitin_result
 
     elif query.action == "compare":
         if not query.url:
@@ -229,7 +228,7 @@ def _get_cached_results(
     )
     data = cursor.fetchall()
 
-    if not data:  # TODO: do something less hacky for this edge case
+    if not data:  # TODO: Do something less hacky for this edge case
         article_chain = CopyvioChecker(page).article_chain
         result = CopyvioCheckResult(
             False, [], queries, check_time, article_chain, possible_miss

src/copyvios/highlighter.py

@@ -13,9 +13,7 @@ from earwigbot.wiki.copyvios.markov import (
 )
 
 
-def highlight_delta(
-    context, chain: MarkovChain, delta: MarkovChainIntersection | None
-) -> str:
+def highlight_delta(chain: MarkovChain, delta: MarkovChainIntersection | None) -> str:
     degree = chain.degree - 1
     highlights = [False] * degree
     block: deque[str | Sentinel] = deque([Sentinel.START] * degree)

src/copyvios/misc.py

@@ -10,11 +10,15 @@ __all__ = [
 import datetime
 import os
 import sqlite3
+import urllib.parse
 from typing import TypeVar
 
 import pymysql
+from flask import g, request
 
+from . import app
 from .cache import cache
+from .query import CheckQuery
 
 T = TypeVar("T")
 
@@ -50,17 +54,34 @@ def get_notice() -> str | None:
         return None
 
 
-def httpsfix(context, url: str) -> str:
-    if url.startswith("http://"):
-        url = url[len("http:") :]
-    return url
+def get_permalink(query: CheckQuery) -> str:
+    params = {
+        "lang": query.orig_lang,
+        "project": query.project,
+        "oldid": query.oldid or g.page.lastrevid,
+        "action": query.action,
+    }
+    if query.action == "search":
+        params["use_engine"] = int(query.use_engine)
+        params["use_links"] = int(query.use_links)
+    elif query.action == "compare":
+        params["url"] = query.url
+    return f"{request.script_root}/?{urllib.parse.urlencode(params)}"
 
 
 def parse_wiki_timestamp(timestamp: str) -> datetime.datetime:
     return datetime.datetime.strptime(timestamp, "%Y%m%d%H%M%S")
 
 
-def urlstrip(context, url: str) -> str:
+@app.template_filter()
+def httpsfix(url: str) -> str:
+    if url.startswith("http://"):
+        url = url[len("http:") :]
+    return url
+
+
+@app.template_filter()
+def urlstrip(url: str) -> str:
     if url.startswith("http://"):
         url = url[7:]
     if url.startswith("https://"):

static/api.css → src/copyvios/static/api.css

@@ -11,15 +11,6 @@ pre {
     max-width: 1200px;
 }
 
-.json {
-    font-family: monospace;
-}
-
-.indent {
-    display: inline-block;
-    padding-left: 2em;
-}
-
 .code {
     font-family: monospace;
 }

src/copyvios/static/api.min.css

@@ -0,0 +1 @@
+h1,h2{font-family:sans-serif}pre{white-space:pre-wrap}#help{margin:auto;max-width:1200px}.code{font-family:monospace}.resp-cond,.resp-desc,.resp-dtype{background-color:#eee;padding:0 .25em}.resp-dtype{color:#009}.resp-cond:before,.resp-dtype:before{content:"("}.resp-cond:after,.resp-dtype:after{content:")"}.resp-desc{color:#050}.resp-cond{color:#900;font-style:italic}.param-key{color:#009;font-weight:700}.param-val{color:#900;font-weight:700}.parameters{margin:1em 0}.parameters tr:first-child{color:#fff;font-family:sans-serif;font-size:1.17em}.parameters tr:first-child th{background-color:#369}.parameters td,.parameters th{padding:.2em .5em}.parameters th{background-color:#f0f0f0}.parameters td:first-child{font-family:monospace}.parameters tr:nth-child(2n+3){background-color:#e0e0e0}.parameters tr:nth-child(2n+4){background-color:#f0f0f0}a:link,a:visited{color:#373;text-decoration:none}a:hover{color:#040}a:active,a:hover{text-decoration:underline}a:active{color:#404}.no-color:link,.no-color:visited{color:#000;text-decoration:none}.no-color:active,.no-color:hover{color:#000;text-decoration:underline}

src/copyvios/templates/api_help.html.jinja

@@ -0,0 +1,285 @@
+<!DOCTYPE html>
+<html lang="en">
+    <head>
+        <meta charset="utf-8">
+        <title>API - Earwig's Copyvio Detector</title>
+        <link rel="stylesheet" href="{{ request.script_root }}{{ url_for('static', file='api.min.css') }}" type="text/css" />
+    </head>
+    <body>
+        <div id="help">
+            <h1>Copyvio Detector API</h1>
+            <p>This is the first version of the <a href="https://en.wikipedia.org/wiki/Application_programming_interface">API</a> for <a href="{{ request.script_root }}/">Earwig's Copyvio Detector</a>. Please <a href="https://github.com/earwig/copyvios/issues">report any issues</a> you encounter.</p>
+            <h2>Requests</h2>
+            <p>The API responds to GET requests made to <span class="code">https://copyvios.toolforge.org/api.json</span>. Parameters are described in the tables below:</p>
+            <table class="parameters">
+                <tr>
+                    <th colspan="4">Always</th>
+                </tr>
+                <tr>
+                    <th>Parameter</th>
+                    <th>Values</th>
+                    <th>Required?</th>
+                    <th>Description</th>
+                </tr>
+                <tr>
+                    <td>action</td>
+                    <td><span class="code">compare</span>, <span class="code">search</span>, <span class="code">sites</span></td>
+                    <td>Yes</td>
+                    <td>The API will do URL comparisons in <span class="code">compare</span> mode, run full copyvio checks in <span class="code">search</span> mode, and list all known site languages and projects in <span class="code">sites</span> mode.</td>
+                </tr>
+                <tr>
+                    <td>format</td>
+                    <td><span class="code">json</span>, <span class="code">jsonfm</span></td>
+                    <td>No&nbsp;(default:&nbsp;<span class="code">json</span>)</td>
+                    <td>The default output format is <a href="https://www.json.org/">JSON</a>. <span class="code">jsonfm</span> mode produces the same output, but renders it as a formatted HTML document for debugging.</td>
+                </tr>
+                <tr>
+                    <td>version</td>
+                    <td>integer</td>
+                    <td>No (default: <span class="code">1</span>)</td>
+                    <td>Currently, the API only has one version. You can skip this parameter, but it is recommended to include it for forward compatibility.</td>
+                </tr>
+            </table>
+            <table class="parameters">
+                <tr>
+                    <th colspan="4"><span class="code">compare</span> Mode</th>
+                </tr>
+                <tr>
+                    <th>Parameter</th>
+                    <th>Values</th>
+                    <th>Required?</th>
+                    <th>Description</th>
+                </tr>
+                <tr>
+                    <td>project</td>
+                    <td>string</td>
+                    <td>Yes</td>
+                    <td>The project code of the site the page lives on. Examples are <span class="code">wikipedia</span> and <span class="code">wiktionary</span>. A list of acceptable values can be retrieved using <span class="code">action=sites</span>.</td>
+                </tr>
+                <tr>
+                    <td>lang</td>
+                    <td>string</td>
+                    <td>Yes</td>
+                    <td>The language code of the site the page lives on. Examples are <span class="code">en</span> and <span class="code">de</span>. A list of acceptable values can be retrieved using <span class="code">action=sites</span>.</td>
+                </tr>
+                <tr>
+                    <td>title</td>
+                    <td>string</td>
+                    <td>Yes&nbsp;(either&nbsp;<span class="code">title</span>&nbsp;or&nbsp;<span class="code">oldid</span>)</td>
+                    <td>The title of the page or article to make a comparison against. Namespace must be included if the page isn't in the mainspace.</td>
+                </tr>
+                <tr>
+                    <td>oldid</td>
+                    <td>integer</td>
+                    <td>Yes (either <span class="code">title</span> or <span class="code">oldid</span>)</td>
+                    <td>The revision ID (also called oldid) of the page revision to make a comparison against. If both a title and oldid are given, the oldid will be used.</td>
+                </tr>
+                <tr>
+                    <td>url</td>
+                    <td>string</td>
+                    <td>Yes</td>
+                    <td>The URL of the suspected violation source that will be compared to the page.</td>
+                </tr>
+                <tr>
+                    <td>detail</td>
+                    <td>boolean</td>
+                    <td>No (default: <span class="code">false</span>)</td>
+                    <td>Whether to include the detailed HTML text comparison available in the regular interface. If not, only the similarity percentage is available.</td>
+                </tr>
+            </table>
+            <table class="parameters">
+                <tr>
+                    <th colspan="4"><span class="code">search</span> Mode</th>
+                </tr>
+                <tr>
+                    <th>Parameter</th>
+                    <th>Values</th>
+                    <th>Required?</th>
+                    <th>Description</th>
+                </tr>
+                <tr>
+                    <td>project</td>
+                    <td>string</td>
+                    <td>Yes</td>
+                    <td>The project code of the site the page lives on. Examples are <span class="code">wikipedia</span> and <span class="code">wiktionary</span>. A list of acceptable values can be retrieved using <span class="code">action=sites</span>.</td>
+                </tr>
+                <tr>
+                    <td>lang</td>
+                    <td>string</td>
+                    <td>Yes</td>
+                    <td>The language code of the site the page lives on. Examples are <span class="code">en</span> and <span class="code">de</span>. A list of acceptable values can be retrieved using <span class="code">action=sites</span>.</td>
+                </tr>
+                <tr>
+                    <td>title</td>
+                    <td>string</td>
+                    <td>Yes&nbsp;(either&nbsp;<span class="code">title</span>&nbsp;or&nbsp;<span class="code">oldid</span>)</td>
+                    <td>The title of the page or article to make a check against. Namespace must be included if the page isn't in the mainspace.</td>
+                </tr>
+                <tr>
+                    <td>oldid</td>
+                    <td>integer</td>
+                    <td>Yes (either <span class="code">title</span> or <span class="code">oldid</span>)</td>
+                    <td>The revision ID (also called oldid) of the page revision to make a check against. If both a title and oldid are given, the oldid will be used.</td>
+                </tr>
+                <tr>
+                    <td>use_engine</td>
+                    <td>boolean</td>
+                    <td>No (default: <span class="code">true</span>)</td>
+                    <td>Whether to use a search engine (<a href="https://developers.google.com/custom-search/">Google</a>) as a source of URLs to compare against the page.</td>
+                </tr>
+                <tr>
+                    <td>use_links</td>
+                    <td>boolean</td>
+                    <td>No (default: <span class="code">true</span>)</td>
+                    <td>Whether to compare the page against external links found in its wikitext.</td>
+                </tr>
+                <tr>
+                    <td>nocache</td>
+                    <td>boolean</td>
+                    <td>No (default: <span class="code">false</span>)</td>
+                    <td>Whether to bypass search results cached from previous checks. It is recommended that you don't pass this option unless a user specifically asks for it.</td>
+                </tr>
+                <tr>
+                    <td>noredirect</td>
+                    <td>boolean</td>
+                    <td>No (default: <span class="code">false</span>)</td>
+                    <td>Whether to avoid following redirects if the given page is a redirect.</td>
+                </tr>
+                <tr>
+                    <td>noskip</td>
+                    <td>boolean</td>
+                    <td>No (default: <span class="code">false</span>)</td>
+                    <td>If a suspected source is found during a check to have a sufficiently high similarity value, the check will end prematurely, and other pending URLs will be skipped. Passing this option will prevent this behavior, resulting in complete (but more time-consuming) checks.</td>
+                </tr>
+            </table>
+            <h2>Responses</h2>
+            <p>The JSON response object always contains a <span class="code">status</span> key, whose value is either <span class="code">ok</span> or <span class="code">error</span>. If an error has occurred, the response will look like this:</p>
+            <pre>{
+"status": "error",
+"error": {
+    "code": <span class="resp-dtype">string</span> <span class="resp-desc">error code</span>,
+    "info": <span class="resp-dtype">string</span> <span class="resp-desc">human-readable description of error</span>
+}
+}</pre>
+            <p>Valid responses for <span class="code">action=compare</span> and <span class="code">action=search</span> are formatted like this:</p>
+            <pre>{
+"status": "ok",
+"meta": {
+    "time":       <span class="resp-dtype">float</span> <span class="resp-desc">time to generate results, in seconds</span>,
+    "queries":    <span class="resp-dtype">int</span> <span class="resp-desc">number of search engine queries made</span>,
+    "cached":     <span class="resp-dtype">boolean</span> <span class="resp-desc">whether these results are cached from an earlier search (always false in the case of action=compare)</span>,
+    "redirected": <span class="resp-dtype">boolean</span> <span class="resp-desc">whether a redirect was followed</span>,
+    <span class="resp-cond">only if cached=true</span> "cache_time": <span class="resp-dtype">string</span> <span class="resp-desc">human-readable time of the original search that the results are cached from</span>
+},
+"page": {
+    "title": <span class="resp-dtype">string</span> <span class="resp-desc">the normalized title of the page checked</span>,
+    "url":   <span class="resp-dtype">string</span> <span class="resp-desc">the full URL of the page checked</span>
+},
+<span class="resp-cond">only if redirected=true</span> "original_page": {
+    "title": <span class="resp-dtype">string</span> <span class="resp-desc">the normalized title of the original page whose redirect was followed</span>,
+    "url":   <span class="resp-dtype">string</span> <span class="resp-desc">the full URL of the original page whose redirect was followed</span>
+},
+"best": {
+    "url":        <span class="resp-dtype">string</span> <span class="resp-desc">the URL of the best match found, or null if no matches were found</span>,
+    "confidence": <span class="resp-dtype">float</span> <span class="resp-desc">the similarity of a violation in the best match, or 0.0 if no matches were found</span>,
+    "violation":  <span class="resp-dtype">string</span> <span class="resp-desc">one of "suspected", "possible", or "none"</span>
+},
+"sources": [
+    {
+        "url":        <span class="resp-dtype">string</span> <span class="resp-desc">the URL of the source</span>,
+        "confidence": <span class="resp-dtype">float</span> <span class="resp-desc">the similarity of the source to the page checked as a ratio between 0.0 and 1.0</span>,
+        "violation":  <span class="resp-dtype">string</span> <span class="resp-desc">one of "suspected", "possible", or "none"</span>,
+        "skipped":    <span class="resp-dtype">boolean</span> <span class="resp-desc">whether the source was skipped due to the check finishing early (see note about noskip above) or an exclusion</span>,
+        "excluded":    <span class="resp-dtype">boolean</span> <span class="resp-desc">whether the source was skipped for being in the excluded URL list</span>
+    },
+    ...
+],
+<span class="resp-cond">only if action=compare and detail=true</span> "detail": {
+    "article": <span class="resp-dtype">string</span> <span class="resp-desc">article text, with shared passages marked with HTML</span>,
+    "source":  <span class="resp-dtype">string</span> <span class="resp-desc">source text, with shared passages marked with HTML</span>
+}
+}</pre>
+            <p>In the case of <span class="code">action=search</span>, <span class="code">sources</span> will contain one entry for each source checked (or skipped if the check ends early), sorted by similarity, with skipped and excluded sources at the bottom.</p>
+            <p>In the case of <span class="code">action=compare</span>, <span class="code">best</span> will always contain information about the URL that was given, so <span class="code">response["best"]["url"]</span> will never be <span class="code">null</span>. Also, <span class="code">sources</span> will always contain one entry, with the same data as <span class="code">best</span>, since only one source is checked in comparison mode.</p>
+            <p>Valid responses for <span class="code">action=sites</span> are formatted like this:</p>
+            <pre>{
+"status": "ok",
+"langs": [
+    [
+        <span class="resp-dtype">string</span> <span class="resp-desc">language code</span>,
+        <span class="resp-dtype">string</span> <span class="resp-desc">human-readable language name</span>
+    ],
+    ...
+],
+"projects": [
+    [
+        <span class="resp-dtype">string</span> <span class="resp-desc">project code</span>,
+        <span class="resp-dtype">string</span> <span class="resp-desc">human-readable project name</span>
+    ],
+    ...
+]
+}</pre>
+            <h2>Etiquette</h2>
+            <p>The tool uses the same workers to handle all requests, so making concurrent API calls is only going to slow you down. Most operations are not rate-limited, but full searches with <span class="code">use_engine=True</span> are globally limited to around a thousand per day. Be respectful!</p>
+            <p>Aside from testing, you must set a reasonable <a href="https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/User-Agent">user agent</a> that identifies your bot and and gives some way to contact you. You may be blocked if using an improper user agent (for example, the default user agent set by your HTTP library), or if your bot makes requests too frequently.</p>
+            <h2>Example</h2>
+            <p><a class="no-color" href="https://copyvios.toolforge.org/api.json?version=1&amp;action=search&amp;project=wikipedia&amp;lang=en&amp;title=User:EarwigBot/Copyvios/Tests/2"><span class="code">https://copyvios.toolforge.org/api.json?<span class="param-key">version</span>=<span class="param-val">1</span>&amp;<span class="param-key">action</span>=<span class="param-val">search</span>&amp;<span class="param-key">project</span>=<span class="param-val">wikipedia</span>&amp;<span class="param-key">lang</span>=<span class="param-val">en</span>&amp;<span class="param-key">title</span>=<span class="param-val">User:EarwigBot/Copyvios/Tests/2</span></span></a></p>
+            <pre>{
+"status": "ok",
+"meta": {
+    "time": 2.2474379539489746,
+    "queries": 1,
+    "cached": false,
+    "redirected": false
+},
+"page": {
+    "title": "User:EarwigBot/Copyvios/Tests/2",
+    "url": "https://en.wikipedia.org/wiki/User:EarwigBot/Copyvios/Tests/2"
+},
+"best": {
+    "url": "http://www.whitehouse.gov/administration/president-obama/",
+    "confidence": 0.9886608511242603,
+    "violation": "suspected"
+}
+"sources": [
+    {
+        "url": "http://www.whitehouse.gov/administration/president-obama/",
+        "confidence": 0.9886608511242603,
+        "violation": "suspected",
+        "skipped": false,
+        "excluded": false
+    },
+    {
+        "url": "http://maige2009.blogspot.com/2013/07/barack-h-obama-is-44th-president-of.html",
+        "confidence": 0.9864798816568047,
+        "violation": "suspected",
+        "skipped": false,
+        "excluded": false
+    },
+    {
+        "url": "http://jeuxdemonstre-apkdownload.rhcloud.com/luo-people-of-kenya-and-tanzania---wikipedia--the-free",
+        "confidence": 0.0,
+        "violation": "none",
+        "skipped": false,
+        "excluded": false
+    },
+    {
+        "url": "http://www.whitehouse.gov/about/presidents/barackobama",
+        "confidence": 0.0,
+        "violation": "none",
+        "skipped": true,
+        "excluded": false
+    },
+    {
+        "url": "http://jeuxdemonstre-apkdownload.rhcloud.com/president-barack-obama---the-white-house",
+        "confidence": 0.0,
+        "violation": "none",
+        "skipped": true,
+        "excluded": false
+    }
+]
+}
+</pre>
+        </div>
+    </body>
+</html>

src/copyvios/templates/api_result.html.jinja

@@ -0,0 +1,14 @@
+<!DOCTYPE html>
+<html lang="en">
+    <head>
+        <meta charset="utf-8">
+        <title>API - Earwig's Copyvio Detector</title>
+        <link rel="stylesheet" href="{{ request.script_root }}{{ url_for('static', file='api.min.css') }}" type="text/css" />
+    </head>
+    <body>
+        <div id="result">
+            <p>You are using <span class="code">jsonfm</span> output mode, which renders JSON data as a formatted HTML document. This is intended for testing and debugging only.</p>
+            <pre>{{ dump_json(result, indent=4) | e }}</pre>
+        </div>
+    </body>
+</html>

src/copyvios/templates/error.html.jinja

@@ -0,0 +1,11 @@
+{% extends "support/base.html.jinja" %}
+{% block title %}
+    Error! - {{ super() }}
+{% endblock %}
+{% block content %}
+    <h2>Error!</h2>
+    <p>An error occurred. If it hasn't been reported (<a href="https://github.com/earwig/copyvios/issues">try to check</a>), please <a href="https://github.com/earwig/copyvios/issues/new">file an issue</a> or <a href="mailto:wikipedia.earwig@gmail.com">email me</a>. Include the following information:</p>
+    <div id="info-box" class="red-box">
+        <pre>{{ traceback | trim | e }}</pre>
+    </div>
+{% endblock %}

src/copyvios/templates/index.html.jinja

@@ -0,0 +1,54 @@
+{% extends "support/base.html.jinja" %}
+{% block title %}
+    {% if g.page %}
+        {{ g.page.title | e }} - {{ super() }}
+    {% else %}
+        {{ super() }}
+    {% endif %}
+{% endblock %}
+{% block content %}
+    {% if query.submitted %}
+        {% if error %}
+            <div id="info-box" class="red-box"><p>
+                {% if error.code == ErrorCode.BAD_ACTION %}
+                    Unknown action: <b><span class="mono">{{ query.action | e }}</span></b>.
+                {% elif error.code == ErrorCode.NO_SEARCH_METHOD %}
+                    No copyvio search methods were selected. A check can only be made using the search engine, links present in the page, Turnitin, or some combination of these.
+                {% elif error.code == ErrorCode.BAD_OLDID %}
+                    The revision ID <code>{{ query.oldid | e }}</code> is invalid. It should be an integer.
+                {% elif error.code == ErrorCode.NO_URL %}
+                    Compare mode requires a URL to be entered. Enter one in the text box below, or choose copyvio search mode to look for content similar to the article elsewhere on the web.
+                {% elif error.code == ErrorCode.BAD_URI %}
+                    Unsupported URI scheme: <a href="{{ query.url | e }}">{{ query.url | e }}</a>.
+                {% elif error.code == ErrorCode.NO_DATA %}
+                    Couldn't find any text in <a href="{{ query.url | e }}">{{ query.url | e }}</a>. <i>Note:</i> only HTML documents, plain text pages, and PDFs are supported, and content generated by JavaScript or found inside iframes is ignored.
+                {% elif error.code == ErrorCode.TIMEOUT %}
+                    The URL <a href="{{ query.url | e }}">{{ query.url | e }}</a> timed out before any data could be retrieved.
+                {% elif error.code == ErrorCode.SEARCH_ERROR %}
+                    An error occurred while using the search engine ({{ error.__cause__ }}). <i>Note:</i> there is a daily limit on the number of search queries the tool is allowed to make. You may <a href="{{ request.url | httpsfix | e }}&amp;use_engine=0">repeat the check without using the search engine</a>.
+                {% else %}
+                    An unknown error occurred.
+                {% endif %}
+            </p></div>
+        {% elif not g.site %}
+            <div id="info-box" class="red-box">
+                <p>The given site (project=<b><span class="mono">{{ query.project | e }}</span></b>, language=<b><span class="mono">{{ query.lang | e }}</span></b>) doesn't seem to exist. It may also be closed or private. <a href="https://{{ query.lang | e }}.{{ query.project | e }}.org/">Confirm its URL.</a></p>
+            </div>
+        {% elif query.oldid and not result %}
+            <div id="info-box" class="red-box">
+                <p>The revision ID couldn't be found: <a href="https://{{ g.site.domain | e }}/w/index.php?oldid={{ query.oldid | e }}">{{ query.oldid | e }}</a>.</p>
+            </div>
+        {% elif query.title and not result %}
+            <div id="info-box" class="red-box">
+                <p>The page couldn't be found: <a href="{{ g.page.url }}">{{ g.page.title | e }}</a>.</p>
+            </div>
+        {% endif %}
+    {% endif %}
+    <p>This tool attempts to detect <a href="https://en.wikipedia.org/wiki/WP:COPYVIO">copyright violations</a> in articles. In <i>search mode</i>, it will check for similar content elsewhere on the web using <a href="https://developers.google.com/custom-search/">Google</a>, external links present in the text of the page, or <a href="https://en.wikipedia.org/wiki/Wikipedia:Turnitin">Turnitin</a> (via <a href="https://en.wikipedia.org/wiki/User:EranBot">EranBot</a>), depending on which options are selected. In <i>compare mode</i>, the tool will compare the article to a specific webpage without making additional searches, like the <a href="https://dupdet.toolforge.org/">Duplication Detector</a>.</p>
+    <p>Running a full check can take up to a minute if other websites are slow or if the tool is under heavy use. Please be patient. If you get a timeout, wait a moment and refresh the page.</p>
+    <p>Be aware that other websites can copy from Wikipedia, so check the results carefully, especially for older or well-developed articles. Specific websites can be skipped by adding them to the <a href="https://en.wikipedia.org/wiki/User:EarwigBot/Copyvios/Exclusions">excluded URL list</a>.</p>
+    {% include "support/cv_form.html.jinja" %}
+    {% if result %}
+        {% include "support/cv_result.html.jinja" %}
+    {% endif %}
+{% endblock %}

src/copyvios/templates/settings.html.jinja

@@ -0,0 +1,93 @@
+{% extends "support/base.html.jinja" %}
+{% block title %}
+    Settings - {{ super() }}
+{% endblock %}
+{% block content %}
+    {% if status %}
+        <div id="info-box" class="green-box">
+            <p>{{ status | safe }}</p>
+        </div>
+    {% endif %}
+    <h2>Settings</h2>
+    <p>This page contains some configurable options for the copyvio detector. Settings are saved as cookies.</p>
+    <form action="{{ request.script_root }}/settings" method="post">
+        <h3>Default site</h2>
+        <div class="oo-ui-layout oo-ui-labelElement oo-ui-fieldLayout oo-ui-fieldLayout-align-top">
+            <div class="oo-ui-fieldLayout-body">
+                <div class="oo-ui-fieldLayout-field">
+                    <div class="oo-ui-widget oo-ui-widget-enabled">
+                        <div class="oo-ui-layout oo-ui-horizontalLayout">
+                            <div class="oo-ui-widget oo-ui-widget-enabled oo-ui-inputWidget oo-ui-dropdownInputWidget oo-ui-dropdownInputWidget-php">
+                                <select name="lang" required="" class="oo-ui-inputWidget-input oo-ui-indicator-down">
+                                    {% set selected_lang = cookies["CopyviosDefaultLang"].value if "CopyviosDefaultLang" in cookies else default_site.lang %}
+                                    {% for lang in cache.langs %}
+                                        {% if lang.code == selected_lang %}
+                                            <option value="{{ lang.code | e }}" selected="selected">{{ lang.name | e }}</option>
+                                        {% else %}
+                                            <option value="{{ lang.code | e }}">{{ lang.name | e }}</option>
+                                        {% endif %}
+                                    {% endfor %}
+                                </select>
+                            </div>
+                            <div class="oo-ui-widget oo-ui-widget-enabled oo-ui-inputWidget oo-ui-dropdownInputWidget oo-ui-dropdownInputWidget-php">
+                                <select name="project" required="" class="oo-ui-inputWidget-input oo-ui-indicator-down">
+                                    {% set selected_project = cookies["CopyviosDefaultProject"].value if "CopyviosDefaultProject" in cookies else default_site.project %}
+                                    {% for project in cache.projects %}
+                                        {% if project.code == selected_project %}
+                                            <option value="{{ project.code | e }}" selected="selected">{{ project.name | e }}</option>
+                                        {% else %}
+                                            <option value="{{ project.code | e }}">{{ project.name | e }}</option>
+                                        {% endif %}
+                                    {% endfor %}
+                                </select>
+                            </div>
+                        </div>
+                    </div>
+                </div>
+            </div>
+        </div>
+
+        <h3>Background</h2>
+        {% set background_options = [
+                ("list", 'Randomly select from <a href="https://commons.wikimedia.org/wiki/User:The_Earwig/POTD">a subset</a> of previous <a href="https://commons.wikimedia.org/">Wikimedia Commons</a> <a href="https://commons.wikimedia.org/wiki/Commons:Picture_of_the_day">Pictures of the Day</a> that work well as widescreen backgrounds, refreshed daily (default).'),
+                ("potd", 'Use the current Commons Picture of the Day, unfiltered. Certain POTDs may be unsuitable as backgrounds due to their aspect ratio or subject matter.'),
+                ("plain", "Use a plain background."),
+            ]
+        %}
+        <div class="oo-ui-layout oo-ui-labelElement oo-ui-fieldLayout oo-ui-fieldLayout-align-top">
+            <div class="oo-ui-fieldLayout-body">
+                <div class="oo-ui-fieldLayout-field">
+                    <div class="oo-ui-widget oo-ui-widget-enabled oo-ui-inputWidget oo-ui-radioSelectInputWidget">
+                        {% for value, desc in background_options %}
+                            <div class="oo-ui-layout oo-ui-labelElement oo-ui-fieldLayout oo-ui-fieldLayout-align-inline">
+                                <div class="oo-ui-fieldLayout-body">
+                                    <span class="oo-ui-fieldLayout-field">
+                                        <span class="oo-ui-widget oo-ui-widget-enabled oo-ui-inputWidget oo-ui-radioInputWidget">
+                                            <input id="background-{{ value | e }}" class="oo-ui-inputWidget-input" type="radio" name="background" value="{{ value | e }}" {{ 'checked="checked"' if value == selected else '' | safe }}><span></span>
+                                        </span>
+                                    </span>
+                                    <span class="oo-ui-fieldLayout-header">
+                                        <label for="background-{{ value | e }}" class="oo-ui-labelElement-label">{{ desc | safe }}</label>
+                                    </span>
+                                </div>
+                            </div>
+                        {% endfor %}
+                    </div>
+                </div>
+            </div>
+        </div>
+
+        <input type="hidden" name="action" value="set"/>
+        <div class="oo-ui-layout oo-ui-fieldLayout oo-ui-fieldLayout-align-left">
+            <div class="oo-ui-fieldLayout-body">
+                <span class="oo-ui-fieldLayout-field">
+                    <span class="oo-ui-widget oo-ui-widget-enabled oo-ui-inputWidget oo-ui-buttonElement oo-ui-buttonElement-framed oo-ui-labelElement oo-ui-flaggedElement-primary oo-ui-flaggedElement-progressive oo-ui-labelElement oo-ui-buttonInputWidget">
+                        <button type="submit" class="oo-ui-inputWidget-input oo-ui-buttonElement-button">
+                            <span class="oo-ui-labelElement-label">Save</span>
+                        </button>
+                    </span>
+                </span>
+            </div>
+        </div>
+    </form>
+{% endblock %}

src/copyvios/templates/support/base.html.jinja

@@ -0,0 +1,50 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+    <meta charset="utf-8">
+    <title>{% block title %}Earwig's Copyvio Detector{% endblock %}</title>
+    <meta name="viewport" content="width=device-width, initial-scale=1">
+    <link rel="stylesheet" href="https://tools-static.wmflabs.org/cdnjs/ajax/libs/oojs-ui/0.41.3/oojs-ui-core-wikimediaui.min.css" integrity="sha512-xL+tTXAo7a4IAwNrNqBcOGWSqJF6ip0jg4SEda2mapAUxPzfOZQ7inazR4TvSCblHQjwtTOkUDIFtnpaSrg3xg==" crossorigin="anonymous" referrerpolicy="no-referrer"/>
+    <link rel="stylesheet" href="https://tools-static.wmflabs.org/cdnjs/ajax/libs/oojs-ui/0.41.3/oojs-ui-images-wikimediaui.min.css" integrity="sha512-A0LSCuOGH1+SyLhOs4eSKGbNgIEGXgIGh4ytb0GRj9GSUsjmmK6LFzB/E0o9ymRUvD+q7bZyv74XpboQt5qFvQ==" crossorigin="anonymous" referrerpolicy="no-referrer"/>
+    <link rel="stylesheet" href="{{ request.script_root }}{{ url_for('static', file='style.min.css') }}"/>
+    <script src="https://tools-static.wmflabs.org/cdnjs/ajax/libs/jquery/3.3.1/jquery.min.js"></script>
+    <script src="{{ request.script_root }}{{ url_for('static', file='script.min.js') }}"></script>
+</head>
+{% set cookies = get_cookies() %}
+{% set selected = cookies["CopyviosBackground"].value if "CopyviosBackground" in cookies else "list" %}
+{% set bg_url, desc_url = get_background(selected) %}
+{% if bg_url %}
+    <body onload="update_screen_size()" style="background-image: url('{{ bg_url | e }}');">
+{% else %}
+    <body>
+{% endif %}
+    <div id="container"{{' class="splash"' if splash else ''}}>
+        <div id="content">
+            <header>
+                <h1><a href="/">Earwig&apos;s <strong>Copyvio Detector</strong></a></h1>
+                <a id="settings-link" href="/settings">Settings</a>
+            </header>
+            <main>
+                {% set notice = get_notice() %}
+                {% if notice %}
+                    <div id="notice-box" class="gray-box">
+                        {{ notice | safe }}
+                    </div>
+                {% endif %}
+                {% block content required %}{% endblock %}
+            </main>
+        </div>
+        <div class="padding"></div>
+    </div>
+    <footer>
+        <ul>
+            <li>Maintained by <a href="https://en.wikipedia.org/wiki/User:The_Earwig">Ben Kurtovic</a></li>
+            <li><a href="{{ request.script_root }}/api">API</a></li>
+            <li><a href="https://github.com/earwig/copyvios">Source code</a></li>
+            {% if desc_url %}
+                <li><a href="{{ desc_url | e }}">Background image</a></li>
+            {% endif %}
+        </ul>
+    </footer>
+</body>
+</html>

src/copyvios/templates/support/cv_form.html.jinja

@@ -0,0 +1,113 @@
+<form id="cv-form" action="{{ request.script_root }}/" method="get">
+    <div class="oo-ui-layout oo-ui-horizontalLayout">
+        <label class="site oo-ui-widget oo-ui-widget-enabled oo-ui-labelElement-label oo-ui-labelElement oo-ui-labelWidget">Site</label>
+        <div class="oo-ui-widget oo-ui-widget-enabled oo-ui-inputWidget oo-ui-dropdownInputWidget oo-ui-dropdownInputWidget-php">
+            <select name="lang" required="" class="oo-ui-inputWidget-input oo-ui-indicator-down" title="Language">
+                {% set selected_lang = query.orig_lang if query.orig_lang else cookies["CopyviosDefaultLang"].value if "CopyviosDefaultLang" in cookies else cache.bot.wiki.get_site().lang %}
+                {% for lang in cache.langs %}
+                    {% if lang.code == selected_lang %}
+                        <option value="{{ lang.code | e }}" selected="selected">{{ lang.name | e }}</option>
+                    {% else %}
+                        <option value="{{ lang.code | e }}">{{ lang.name | e }}</option>
+                    {% endif %}
+                {% endfor %}
+            </select>
+        </div>
+        <div class="oo-ui-widget oo-ui-widget-enabled oo-ui-inputWidget oo-ui-dropdownInputWidget oo-ui-dropdownInputWidget-php">
+            <select name="project" required="" class="oo-ui-inputWidget-input oo-ui-indicator-down" title="Project">
+                {% set selected_project = query.project if query.project else cookies["CopyviosDefaultProject"].value if "CopyviosDefaultProject" in cookies else cache.bot.wiki.get_site().project %}
+                {% for project in cache.projects %}
+                    {% if project.code == selected_project %}
+                        <option value="{{ project.code | e }}" selected="selected">{{ project.name | e }}</option>
+                    {% else %}
+                        <option value="{{ project.code | e }}">{{ project.name | e }}</option>
+                    {% endif %}
+                {% endfor %}
+            </select>
+        </div>
+    </div>
+    <div class="oo-ui-layout oo-ui-horizontalLayout">
+        <label for="cv-title" class="page oo-ui-widget oo-ui-widget-enabled oo-ui-labelElement-label oo-ui-labelElement oo-ui-labelWidget">Page</label>
+        <div class="page-title oo-ui-widget oo-ui-widget-enabled oo-ui-inputWidget oo-ui-textInputWidget oo-ui-textInputWidget-type-text oo-ui-textInputWidget-php">
+            <input id="cv-title" type="text" class="oo-ui-inputWidget-input" name="title" placeholder="Title" title="Page title"
+                {% if query.title %}
+                    value="{{ g.page.title if g.page else query.title | e}}"
+                {% endif %}
+            >
+        </div>
+        <label class="oo-ui-widget oo-ui-widget-enabled oo-ui-labelElement-label oo-ui-labelElement oo-ui-labelWidget">or</label>
+        <div class="page-oldid oo-ui-widget oo-ui-widget-enabled oo-ui-inputWidget oo-ui-textInputWidget oo-ui-textInputWidget-type-text oo-ui-textInputWidget-php">
+            <input id="cv-oldid" type="text" class="oo-ui-inputWidget-input" name="oldid" placeholder="Revision ID" title="Revision ID"
+                {% if query.oldid %}
+                    value="{{ query.oldid | e }}"
+                {% endif %}
+            >
+        </div>
+    </div>
+    <div class="oo-ui-layout oo-ui-horizontalLayout">
+        <span class="oo-ui-widget oo-ui-widget-enabled">
+            <span class="oo-ui-widget oo-ui-widget-enabled oo-ui-inputWidget oo-ui-radioInputWidget">
+                <input id="action-search" class="oo-ui-inputWidget-input" type="radio" name="action" value="search" {{ 'checked="checked"' if (query.action == "search" or not query.action) else "" }}><span></span>
+            </span>
+            <label for="action-search" class="action oo-ui-widget oo-ui-widget-enabled oo-ui-labelElement-label oo-ui-labelElement oo-ui-labelWidget">Copyvio search</label>
+        </span>
+
+        <input type="hidden" name="use_engine" value="0">
+        <span class="oo-ui-widget oo-ui-widget-enabled">
+            <span class="cv-search-oo-ui oo-ui-widget oo-ui-widget-enabled oo-ui-inputWidget oo-ui-checkboxInputWidget">
+                <input id="cv-cb-engine" class="cv-search oo-ui-inputWidget-input" type="checkbox" name="use_engine" value="1" {{ 'checked="checked"' if query.use_engine else "" }}>
+                <span class="oo-ui-checkboxInputWidget-checkIcon oo-ui-widget oo-ui-widget-enabled oo-ui-iconElement-icon oo-ui-icon-check oo-ui-iconElement oo-ui-labelElement-invisible oo-ui-iconWidget oo-ui-image-invert"></span>
+            </span>
+            <label for="cv-cb-engine" class="oo-ui-widget oo-ui-widget-enabled oo-ui-labelElement-label oo-ui-labelElement oo-ui-labelWidget">Use search engine</label>
+        </span>
+
+        <input type="hidden" name="use_links" value="0">
+        <span class="oo-ui-widget oo-ui-widget-enabled">
+            <span class="cv-search-oo-ui oo-ui-widget oo-ui-widget-enabled oo-ui-inputWidget oo-ui-checkboxInputWidget">
+                <input id="cv-cb-links" class="cv-search oo-ui-inputWidget-input" type="checkbox" name="use_links" value="1" {{ 'checked="checked"' if query.use_links else "" }}>
+                <span class="oo-ui-checkboxInputWidget-checkIcon oo-ui-widget oo-ui-widget-enabled oo-ui-iconElement-icon oo-ui-icon-check oo-ui-iconElement oo-ui-labelElement-invisible oo-ui-iconWidget oo-ui-image-invert"></span>
+            </span>
+            <label for="cv-cb-links" class="oo-ui-widget oo-ui-widget-enabled oo-ui-labelElement-label oo-ui-labelElement oo-ui-labelWidget">Use links in page</label>
+        </span>
+
+        <input type="hidden" name="turnitin" value="0">
+        <span class="oo-ui-widget oo-ui-widget-enabled">
+            <span class="cv-search-oo-ui oo-ui-widget oo-ui-widget-enabled oo-ui-inputWidget oo-ui-checkboxInputWidget">
+                <input id="cv-cb-turnitin" class="cv-search oo-ui-inputWidget-input" type="checkbox" name="turnitin" value="1" {{ 'checked="checked"' if query.turnitin else "" }}>
+                <span class="oo-ui-checkboxInputWidget-checkIcon oo-ui-widget oo-ui-widget-enabled oo-ui-iconElement-icon oo-ui-icon-check oo-ui-iconElement oo-ui-labelElement-invisible oo-ui-iconWidget oo-ui-image-invert"></span>
+            </span>
+            <label for="cv-cb-turnitin" class="oo-ui-widget oo-ui-widget-enabled oo-ui-labelElement-label oo-ui-labelElement oo-ui-labelWidget">Use Turnitin</label>
+        </span>
+    </div>
+    <div class="oo-ui-layout oo-ui-horizontalLayout">
+        <span class="oo-ui-widget oo-ui-widget-enabled">
+            <span class="oo-ui-widget oo-ui-widget-enabled oo-ui-inputWidget oo-ui-radioInputWidget">
+                <input id="action-compare" class="oo-ui-inputWidget-input" type="radio" name="action" value="compare" {{ 'checked="checked"' if query.action == "compare" else "" }}><span></span>
+            </span>
+            <label for="action-compare" class="action oo-ui-widget oo-ui-widget-enabled oo-ui-labelElement-label oo-ui-labelElement oo-ui-labelWidget">Copyvio compare</label>
+        </span>
+        <div class="compare-url cv-compare-oo-ui oo-ui-widget oo-ui-widget-enabled oo-ui-inputWidget oo-ui-textInputWidget oo-ui-textInputWidget-type-text oo-ui-textInputWidget-php">
+            <input type="text" class="cv-compare oo-ui-inputWidget-input" name="url" placeholder="URL" title="URL to compare"
+                {% if query.url %}
+                    value="{{ query.url | e }}"
+                {% endif %}
+            >
+        </div>
+    </div>
+    {% if query.nocache or (result and result.metadata.cached) %}
+        <div class="oo-ui-layout oo-ui-horizontalLayout">
+            <span class="cv-search-oo-ui oo-ui-widget oo-ui-widget-enabled oo-ui-inputWidget oo-ui-checkboxInputWidget">
+                <input id="cb-nocache" class="oo-ui-inputWidget-input" type="checkbox" name="nocache" value="1" {{ 'checked="checked"' if query.nocache else "" }}>
+                <span class="oo-ui-checkboxInputWidget-checkIcon oo-ui-widget oo-ui-widget-enabled oo-ui-iconElement-icon oo-ui-icon-check oo-ui-iconElement oo-ui-labelElement-invisible oo-ui-iconWidget oo-ui-image-invert"></span>
+            </span>
+            <label for="cb-nocache">Bypass cache</label>
+        </div>
+    {% endif %}
+    <div class="oo-ui-layout oo-ui-horizontalLayout">
+        <span class="oo-ui-widget oo-ui-widget-enabled oo-ui-buttonElement oo-ui-buttonElement-framed oo-ui-labelElement oo-ui-flaggedElement-primary oo-ui-flaggedElement-progressive oo-ui-buttonWidget">
+            <button type="submit" class="oo-ui-inputWidget-input oo-ui-buttonElement-button">
+                <span class="oo-ui-labelElement-label">Submit</span>
+            </button>
+        </span>
+    </div>
+</form>

src/copyvios/templates/support/cv_result.html.jinja

@@ -0,0 +1,147 @@
+<div id="generation-time">
+    Results
+    {% if result.metadata.cached %}
+        <a id="cv-cached" href="#">cached<span>To save time (and money), this tool will retain the results of checks for up to 72 hours. This includes the URLs of the checked sources, but neither their content nor the content of the article. Future checks on the same page (assuming it remains unchanged) will not involve additional search queries, but a fresh comparison against the source URL will be made. If the page is modified, a new check will be run.</span></a> from <abbr title="{{ result.metadata.cache_time }}">{{ result.metadata.cache_age }} ago</abbr>. Originally
+    {% endif %}
+    generated in <span class="mono">{{ result.time | round(3) }}</span>
+    {% if query.action == "search" %}
+        seconds using <span class="mono">{{ result.queries }}</span> quer{{ "y" if result.queries == 1 else "ies" }}.
+    {% else %}
+        seconds.
+    {% endif %}
+    <a href="{{ get_permalink(query) | e }}">Permalink.</a>
+</div>
+
+<div id="cv-result" class="{{ 'red' if result.confidence >= T_SUSPECT else 'yellow' if result.confidence >= T_POSSIBLE else 'green' }}-box">
+    <table id="cv-result-head-table">
+        <colgroup>
+            <col>
+            <col>
+            <col>
+        </colgroup>
+        <tr>
+            <td>
+                <a href="{{ g.page.url | e }}">{{ g.page.title | e }}</a>
+                {% if query.oldid %}
+                    @<a href="https://{{ g.site.domain | e }}/w/index.php?oldid={{ query.oldid | e }}">{{ query.oldid | e }}</a>
+                {% endif %}
+                {% if query.redirected_from %}
+                    <br />
+                    <span id="redirected-from">Redirected from <a href="https://{{ g.site.domain | e }}/w/index.php?title={{ result.metadata.redirected_from.title | e }}&amp;redirect=no">{{ result.metadata.redirected_from.title | e }}</a>. <a href="{{ request.url | httpsfix | e }}&amp;noredirect=1">Check original.</a></span>
+                {% endif %}
+            </td>
+            <td>
+                <div>
+                    {% if result.confidence >= T_SUSPECT %}
+                        Violation suspected
+                    {% elif result.confidence >= T_POSSIBLE %}
+                        Violation possible
+                    {% elif result.sources %}
+                        Violation unlikely
+                    {% else %}
+                        No violation
+                    {% endif %}
+                </div>
+                <div>{{ (result.confidence * 100) | round(1) }}%</div>
+                <div>similarity</div>
+            </td>
+            <td>
+                {% if result.url %}
+                    <a href="{{ result.url | e }}">{{ result.url | urlstrip | e }}</a>
+                {% else %}
+                    <span id="result-head-no-sources">No matches found.</span>
+                {% endif %}
+            </td>
+        </tr>
+    </table>
+</div>
+
+{% set attrib = get_attribution_info(g.site, g.page) %}
+{% if attrib %}
+    <div id="attribution-warning" class="yellow-box">
+        This article contains an attribution template: <code>{{ '{{' }}<a href="{{ attrib[1] | e }}">{{ attrib[0] | e }}</a>{{ '}}' }}</code>. Please verify that any potential copyvios are not from properly attributed sources.
+    </div>
+{% endif %}
+
+{% if result.metadata.turnitin_result %}
+    <div id="turnitin-container" class="{{ 'red' if result.metadata.turnitin_result.reports else 'green' }}-box">
+        <div id="turnitin-title">Turnitin Results</div>
+        {% if result.metadata.turnitin_result.reports %}
+            <table id="turnitin-table"><tbody>
+            {% for report in turnitin_result.reports %}
+                <tr><td class="turnitin-table-cell"><a href="https://eranbot.toolforge.org/ithenticate.py?rid={{ report.reportid }}">Report {{ report.reportid }}</a> for text added at <a href="https://{{ query.lang | e }}.wikipedia.org/w/index.php?title={{ query.title | e }}&amp;diff={{ report.diffid }}"> {{ report.time_posted.strftime("%H:%M, %d %B %Y (UTC)") }}</a>:
+                <ul>
+                {% for source in report.sources %}
+                    <li>{{ source['percent'] }}% of revision text ({{ source['words'] }} words) found at <a href="{{ source['url'] | e }}">{{ source['url'] | e }}</a></li>
+                {% endfor %}
+                </ul></td></tr>
+            {% endfor %}
+            </tbody></table>
+        {% else %}
+            <div id="turnitin-summary">No matching sources found.</div>
+        {% endif %}
+    </div>
+{% endif %}
+
+{% if query.action == "search" %}
+    {% set skips = False %}
+    <div id="sources-container">
+        <div id="sources-title">Checked Sources</div>
+        {% if result.sources %}
+            <table id="cv-result-sources">
+                <colgroup>
+                    <col>
+                    <col>
+                    <col>
+                </colgroup>
+                <tr>
+                    <th>URL</th>
+                    <th>Similarity</th>
+                    <th>Compare</th>
+                </tr>
+                {% for i, source in enumerate(result.sources) %}
+                    <tr {{ 'class="source-default-hidden"' if i >= 10 else 'id="source-row-selected"' if i == 0 else "" }}>
+                        <td><a {{ 'id="source-selected"' if i == 0 else "" }} class="source-url" href="{{ source.url | e }}">{{ source.url | e }}</a></td>
+                        <td>
+                            {% if source.excluded %}
+                                <span class="source-excluded">Excluded</span>
+                            {% elif source.skipped %}
+                                {% set skips = True %}
+                                <span class="source-skipped">Skipped</span>
+                            {% else %}
+                                <span class="source-similarity {{ "source-suspect" if source.confidence >= T_SUSPECT else "source-possible" if source.confidence >= T_POSSIBLE else "source-novio" }}">{{ source.confidence * 100 | round(1) }}%</span>
+                            {% endif %}
+                        </td>
+                        <td>
+                            <a href="{{ request.script_root | e }}/?lang={{ query.lang | e }}&amp;project={{ query.project | e }}&amp;oldid={{ query.oldid or g.page.lastrevid | e }}&amp;action=compare&amp;url={{ source.url | e }}">Compare</a>
+                        </td>
+                    </tr>
+                {% endfor %}
+            </table>
+        {% else %}
+            <div class="cv-source-footer">
+                No sources checked.
+            </div>
+        {% endif %}
+        {% if len(result.sources) > 10 %}
+            <div id="cv-additional" class="cv-source-footer">
+                {{ len(result.sources) - 10 }} URL{{ "s" if len(result.sources) > 11 else "" }} with lower similarity hidden. <a id="show-additional-sources" href="#">Show them.</a>
+            </div>
+        {% endif %}
+        {% if skips or result.possible_miss %}
+            <div class="cv-source-footer">
+                The search ended early because a match was found with high similarity. <a href="{{ request.url | httpsfix | e }}&amp;noskip=1">Do a complete check.</a>
+            </div>
+        {% endif %}
+    </div>
+{% endif %}
+<table id="cv-chain-table">
+    <tr>
+        <td class="cv-chain-cell">Article: <div class="cv-chain-detail">
+            <p>{{ highlight_delta(result.article_chain, result.best.chains[1] if result.best else None) }}</p>
+        </div></td>
+        <td class="cv-chain-cell">Source: <div class="cv-chain-detail">
+            <p>{{ highlight_delta(result.best.chains[0], result.best.chains[1]) if result.best else "" }}</p>
+        </div></td>
+    </tr>
+</table>

static/api.min.css

@@ -1 +0,0 @@
-h1,h2{font-family:sans-serif}pre{white-space:pre-wrap}#help{margin:auto;max-width:1200px}.json{font-family:monospace}.indent{display:inline-block;padding-left:2em}.code{font-family:monospace}.resp-cond,.resp-desc,.resp-dtype{background-color:#eee;padding:0 .25em}.resp-dtype{color:#009}.resp-cond:before,.resp-dtype:before{content:"("}.resp-cond:after,.resp-dtype:after{content:")"}.resp-desc{color:#050}.resp-cond{color:#900;font-style:italic}.param-key{color:#009;font-weight:700}.param-val{color:#900;font-weight:700}.parameters{margin:1em 0}.parameters tr:first-child{color:#fff;font-family:sans-serif;font-size:1.17em}.parameters tr:first-child th{background-color:#369}.parameters td,.parameters th{padding:.2em .5em}.parameters th{background-color:#f0f0f0}.parameters td:first-child{font-family:monospace}.parameters tr:nth-child(2n+3){background-color:#e0e0e0}.parameters tr:nth-child(2n+4){background-color:#f0f0f0}a:link,a:visited{color:#373;text-decoration:none}a:hover{color:#040}a:active,a:hover{text-decoration:underline}a:active{color:#404}.no-color:link,.no-color:visited{color:#000;text-decoration:none}.no-color:active,.no-color:hover{color:#000;text-decoration:underline}

templates/api.mako

@@ -1,326 +0,0 @@
-<%!
-    from json import dumps
-    from flask import url_for
-%>\
-<%def name="do_indent(size)">
-    <br />
-    % for i in xrange(size):
-        <div class="indent"></div>
-    % endfor
-</%def>\
-<%def name="walk_json(obj, indent=0)">
-    % if isinstance(obj, type({})):
-        {
-        % for key in obj:
-            ${do_indent(indent + 1)}
-            "${key | h}": ${walk_json(obj[key], indent + 1)}${"," if not loop.last else ""}
-        % endfor
-        ${do_indent(indent)}
-        }
-    % elif isinstance(obj, (list, tuple, set)):
-        [
-        % for elem in obj:
-            ${do_indent(indent + 1)}
-            ${walk_json(elem, indent + 1)}${"," if not loop.last else ""}
-        % endfor
-        ${do_indent(indent)}
-        ]
-    % else:
-        ${dumps(obj) | h}
-    % endif
-</%def>\
-<!DOCTYPE html>
-<html lang="en">
-    <head>
-        <meta charset="utf-8">
-        <title>API | Earwig's Copyvio Detector</title>
-        <link rel="stylesheet" href="${request.script_root}${url_for('static', file='api.min.css')}" type="text/css" />
-    </head>
-    <body>
-        % if help:
-            <div id="help">
-                <h1>Copyvio Detector API</h1>
-                <p>This is the first version of the <a href="https://en.wikipedia.org/wiki/Application_programming_interface">API</a> for <a href="${request.script_root}/">Earwig's Copyvio Detector</a>. Please <a href="https://github.com/earwig/copyvios/issues">report any issues</a> you encounter.</p>
-                <h2>Requests</h2>
-                <p>The API responds to GET requests made to <span class="code">https://copyvios.toolforge.org/api.json</span>. Parameters are described in the tables below:</p>
-                <table class="parameters">
-                    <tr>
-                        <th colspan="4">Always</th>
-                    </tr>
-                    <tr>
-                        <th>Parameter</th>
-                        <th>Values</th>
-                        <th>Required?</th>
-                        <th>Description</th>
-                    </tr>
-                    <tr>
-                        <td>action</td>
-                        <td><span class="code">compare</span>, <span class="code">search</span>, <span class="code">sites</span></td>
-                        <td>Yes</td>
-                        <td>The API will do URL comparisons in <span class="code">compare</span> mode, run full copyvio checks in <span class="code">search</span> mode, and list all known site languages and projects in <span class="code">sites</span> mode.</td>
-                    </tr>
-                    <tr>
-                        <td>format</td>
-                        <td><span class="code">json</span>, <span class="code">jsonfm</span></td>
-                        <td>No&nbsp;(default:&nbsp;<span class="code">json</span>)</td>
-                        <td>The default output format is <a href="https://www.json.org/">JSON</a>. <span class="code">jsonfm</span> mode produces the same output, but renders it as a formatted HTML document for debugging.</td>
-                    </tr>
-                    <tr>
-                        <td>version</td>
-                        <td>integer</td>
-                        <td>No (default: <span class="code">1</span>)</td>
-                        <td>Currently, the API only has one version. You can skip this parameter, but it is recommended to include it for forward compatibility.</td>
-                    </tr>
-                </table>
-                <table class="parameters">
-                    <tr>
-                        <th colspan="4"><span class="code">compare</span> Mode</th>
-                    </tr>
-                    <tr>
-                        <th>Parameter</th>
-                        <th>Values</th>
-                        <th>Required?</th>
-                        <th>Description</th>
-                    </tr>
-                    <tr>
-                        <td>project</td>
-                        <td>string</td>
-                        <td>Yes</td>
-                        <td>The project code of the site the page lives on. Examples are <span class="code">wikipedia</span> and <span class="code">wiktionary</span>. A list of acceptable values can be retrieved using <span class="code">action=sites</span>.</td>
-                    </tr>
-                    <tr>
-                        <td>lang</td>
-                        <td>string</td>
-                        <td>Yes</td>
-                        <td>The language code of the site the page lives on. Examples are <span class="code">en</span> and <span class="code">de</span>. A list of acceptable values can be retrieved using <span class="code">action=sites</span>.</td>
-                    </tr>
-                    <tr>
-                        <td>title</td>
-                        <td>string</td>
-                        <td>Yes&nbsp;(either&nbsp;<span class="code">title</span>&nbsp;or&nbsp;<span class="code">oldid</span>)</td>
-                        <td>The title of the page or article to make a comparison against. Namespace must be included if the page isn't in the mainspace.</td>
-                    </tr>
-                    <tr>
-                        <td>oldid</td>
-                        <td>integer</td>
-                        <td>Yes (either <span class="code">title</span> or <span class="code">oldid</span>)</td>
-                        <td>The revision ID (also called oldid) of the page revision to make a comparison against. If both a title and oldid are given, the oldid will be used.</td>
-                    </tr>
-                    <tr>
-                        <td>url</td>
-                        <td>string</td>
-                        <td>Yes</td>
-                        <td>The URL of the suspected violation source that will be compared to the page.</td>
-                    </tr>
-                    <tr>
-                        <td>detail</td>
-                        <td>boolean</td>
-                        <td>No (default: <span class="code">false</span>)</td>
-                        <td>Whether to include the detailed HTML text comparison available in the regular interface. If not, only the similarity percentage is available.</td>
-                    </tr>
-                </table>
-                <table class="parameters">
-                    <tr>
-                        <th colspan="4"><span class="code">search</span> Mode</th>
-                    </tr>
-                    <tr>
-                        <th>Parameter</th>
-                        <th>Values</th>
-                        <th>Required?</th>
-                        <th>Description</th>
-                    </tr>
-                    <tr>
-                        <td>project</td>
-                        <td>string</td>
-                        <td>Yes</td>
-                        <td>The project code of the site the page lives on. Examples are <span class="code">wikipedia</span> and <span class="code">wiktionary</span>. A list of acceptable values can be retrieved using <span class="code">action=sites</span>.</td>
-                    </tr>
-                    <tr>
-                        <td>lang</td>
-                        <td>string</td>
-                        <td>Yes</td>
-                        <td>The language code of the site the page lives on. Examples are <span class="code">en</span> and <span class="code">de</span>. A list of acceptable values can be retrieved using <span class="code">action=sites</span>.</td>
-                    </tr>
-                    <tr>
-                        <td>title</td>
-                        <td>string</td>
-                        <td>Yes&nbsp;(either&nbsp;<span class="code">title</span>&nbsp;or&nbsp;<span class="code">oldid</span>)</td>
-                        <td>The title of the page or article to make a check against. Namespace must be included if the page isn't in the mainspace.</td>
-                    </tr>
-                    <tr>
-                        <td>oldid</td>
-                        <td>integer</td>
-                        <td>Yes (either <span class="code">title</span> or <span class="code">oldid</span>)</td>
-                        <td>The revision ID (also called oldid) of the page revision to make a check against. If both a title and oldid are given, the oldid will be used.</td>
-                    </tr>
-                    <tr>
-                        <td>use_engine</td>
-                        <td>boolean</td>
-                        <td>No (default: <span class="code">true</span>)</td>
-                        <td>Whether to use a search engine (<a href="https://developers.google.com/custom-search/">Google</a>) as a source of URLs to compare against the page.</td>
-                    </tr>
-                    <tr>
-                        <td>use_links</td>
-                        <td>boolean</td>
-                        <td>No (default: <span class="code">true</span>)</td>
-                        <td>Whether to compare the page against external links found in its wikitext.</td>
-                    </tr>
-                    <tr>
-                        <td>nocache</td>
-                        <td>boolean</td>
-                        <td>No (default: <span class="code">false</span>)</td>
-                        <td>Whether to bypass search results cached from previous checks. It is recommended that you don't pass this option unless a user specifically asks for it.</td>
-                    </tr>
-                    <tr>
-                        <td>noredirect</td>
-                        <td>boolean</td>
-                        <td>No (default: <span class="code">false</span>)</td>
-                        <td>Whether to avoid following redirects if the given page is a redirect.</td>
-                    </tr>
-                    <tr>
-                        <td>noskip</td>
-                        <td>boolean</td>
-                        <td>No (default: <span class="code">false</span>)</td>
-                        <td>If a suspected source is found during a check to have a sufficiently high similarity value, the check will end prematurely, and other pending URLs will be skipped. Passing this option will prevent this behavior, resulting in complete (but more time-consuming) checks.</td>
-                    </tr>
-                </table>
-                <h2>Responses</h2>
-                <p>The JSON response object always contains a <span class="code">status</span> key, whose value is either <span class="code">ok</span> or <span class="code">error</span>. If an error has occurred, the response will look like this:</p>
-                <pre>{
-    "status": "error",
-    "error": {
-        "code": <span class="resp-dtype">string</span> <span class="resp-desc">error code</span>,
-        "info": <span class="resp-dtype">string</span> <span class="resp-desc">human-readable description of error</span>
-    }
-}</pre>
-                <p>Valid responses for <span class="code">action=compare</span> and <span class="code">action=search</span> are formatted like this:</p>
-                <pre>{
-    "status": "ok",
-    "meta": {
-        "time":       <span class="resp-dtype">float</span> <span class="resp-desc">time to generate results, in seconds</span>,
-        "queries":    <span class="resp-dtype">int</span> <span class="resp-desc">number of search engine queries made</span>,
-        "cached":     <span class="resp-dtype">boolean</span> <span class="resp-desc">whether these results are cached from an earlier search (always false in the case of action=compare)</span>,
-        "redirected": <span class="resp-dtype">boolean</span> <span class="resp-desc">whether a redirect was followed</span>,
-        <span class="resp-cond">only if cached=true</span> "cache_time": <span class="resp-dtype">string</span> <span class="resp-desc">human-readable time of the original search that the results are cached from</span>
-    },
-    "page": {
-        "title": <span class="resp-dtype">string</span> <span class="resp-desc">the normalized title of the page checked</span>,
-        "url":   <span class="resp-dtype">string</span> <span class="resp-desc">the full URL of the page checked</span>
-    },
-    <span class="resp-cond">only if redirected=true</span> "original_page": {
-        "title": <span class="resp-dtype">string</span> <span class="resp-desc">the normalized title of the original page whose redirect was followed</span>,
-        "url":   <span class="resp-dtype">string</span> <span class="resp-desc">the full URL of the original page whose redirect was followed</span>
-    },
-    "best": {
-        "url":        <span class="resp-dtype">string</span> <span class="resp-desc">the URL of the best match found, or null if no matches were found</span>,
-        "confidence": <span class="resp-dtype">float</span> <span class="resp-desc">the similarity of a violation in the best match, or 0.0 if no matches were found</span>,
-        "violation":  <span class="resp-dtype">string</span> <span class="resp-desc">one of "suspected", "possible", or "none"</span>
-    },
-    "sources": [
-        {
-            "url":        <span class="resp-dtype">string</span> <span class="resp-desc">the URL of the source</span>,
-            "confidence": <span class="resp-dtype">float</span> <span class="resp-desc">the similarity of the source to the page checked as a ratio between 0.0 and 1.0</span>,
-            "violation":  <span class="resp-dtype">string</span> <span class="resp-desc">one of "suspected", "possible", or "none"</span>,
-            "skipped":    <span class="resp-dtype">boolean</span> <span class="resp-desc">whether the source was skipped due to the check finishing early (see note about noskip above) or an exclusion</span>,
-            "excluded":    <span class="resp-dtype">boolean</span> <span class="resp-desc">whether the source was skipped for being in the excluded URL list</span>
-        },
-        ...
-    ],
-    <span class="resp-cond">only if action=compare and detail=true</span> "detail": {
-        "article": <span class="resp-dtype">string</span> <span class="resp-desc">article text, with shared passages marked with HTML</span>,
-        "source":  <span class="resp-dtype">string</span> <span class="resp-desc">source text, with shared passages marked with HTML</span>
-    }
-}</pre>
-                <p>In the case of <span class="code">action=search</span>, <span class="code">sources</span> will contain one entry for each source checked (or skipped if the check ends early), sorted by similarity, with skipped and excluded sources at the bottom.</p>
-                <p>In the case of <span class="code">action=compare</span>, <span class="code">best</span> will always contain information about the URL that was given, so <span class="code">response["best"]["url"]</span> will never be <span class="code">null</span>. Also, <span class="code">sources</span> will always contain one entry, with the same data as <span class="code">best</span>, since only one source is checked in comparison mode.</p>
-                <p>Valid responses for <span class="code">action=sites</span> are formatted like this:</p>
-                <pre>{
-    "status": "ok",
-    "langs": [
-        [
-            <span class="resp-dtype">string</span> <span class="resp-desc">language code</span>,
-            <span class="resp-dtype">string</span> <span class="resp-desc">human-readable language name</span>
-        ],
-        ...
-    ],
-    "projects": [
-        [
-            <span class="resp-dtype">string</span> <span class="resp-desc">project code</span>,
-            <span class="resp-dtype">string</span> <span class="resp-desc">human-readable project name</span>
-        ],
-        ...
-    ]
-}</pre>
-                <h2>Etiquette</h2>
-                <p>The tool uses the same workers to handle all requests, so making concurrent API calls is only going to slow you down. Most operations are not rate-limited, but full searches with <span class="code">use_engine=True</span> are globally limited to around a thousand per day. Be respectful!</p>
-                <p>Aside from testing, you must set a reasonable <a href="https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/User-Agent">user agent</a> that identifies your bot and and gives some way to contact you. You may be blocked if using an improper user agent (for example, the default user agent set by your HTTP library), or if your bot makes requests too frequently.</p>
-                <h2>Example</h2>
-                <p><a class="no-color" href="https://copyvios.toolforge.org/api.json?version=1&amp;action=search&amp;project=wikipedia&amp;lang=en&amp;title=User:EarwigBot/Copyvios/Tests/2"><span class="code">https://copyvios.toolforge.org/api.json?<span class="param-key">version</span>=<span class="param-val">1</span>&amp;<span class="param-key">action</span>=<span class="param-val">search</span>&amp;<span class="param-key">project</span>=<span class="param-val">wikipedia</span>&amp;<span class="param-key">lang</span>=<span class="param-val">en</span>&amp;<span class="param-key">title</span>=<span class="param-val">User:EarwigBot/Copyvios/Tests/2</span></span></a></p>
-                <pre>{
-    "status": "ok",
-    "meta": {
-        "time": 2.2474379539489746,
-        "queries": 1,
-        "cached": false,
-        "redirected": false
-    },
-    "page": {
-        "title": "User:EarwigBot/Copyvios/Tests/2",
-        "url": "https://en.wikipedia.org/wiki/User:EarwigBot/Copyvios/Tests/2"
-    },
-    "best": {
-        "url": "http://www.whitehouse.gov/administration/president-obama/",
-        "confidence": 0.9886608511242603,
-        "violation": "suspected"
-    }
-    "sources": [
-        {
-            "url": "http://www.whitehouse.gov/administration/president-obama/",
-            "confidence": 0.9886608511242603,
-            "violation": "suspected",
-            "skipped": false,
-            "excluded": false
-        },
-        {
-            "url": "http://maige2009.blogspot.com/2013/07/barack-h-obama-is-44th-president-of.html",
-            "confidence": 0.9864798816568047,
-            "violation": "suspected",
-            "skipped": false,
-            "excluded": false
-        },
-        {
-            "url": "http://jeuxdemonstre-apkdownload.rhcloud.com/luo-people-of-kenya-and-tanzania---wikipedia--the-free",
-            "confidence": 0.0,
-            "violation": "none",
-            "skipped": false,
-            "excluded": false
-        },
-        {
-            "url": "http://www.whitehouse.gov/about/presidents/barackobama",
-            "confidence": 0.0,
-            "violation": "none",
-            "skipped": true,
-            "excluded": false
-        },
-        {
-            "url": "http://jeuxdemonstre-apkdownload.rhcloud.com/president-barack-obama---the-white-house",
-            "confidence": 0.0,
-            "violation": "none",
-            "skipped": true,
-            "excluded": false
-        }
-    ]
-}
-</pre>
-            </div>
-        % endif
-        % if result:
-            <div id="result">
-                <p>You are using <span class="code">jsonfm</span> output mode, which renders JSON data as a formatted HTML document. This is intended for testing and debugging only.</p>
-                <div class="json">
-                    ${walk_json(result)}
-                </div>
-            </div>
-        % endif
-    </body>
-</html>

templates/error.mako

@@ -1,7 +0,0 @@
-<%include file="/support/header.mako" args="title='Error! | Earwig\'s Copyvio Detector'"/>
-<h2>Error!</h2>
-<p>An error occurred. If it hasn't been reported (<a href="https://github.com/earwig/copyvios/issues">try to check</a>), please <a href="https://github.com/earwig/copyvios/issues/new">file an issue</a> or <a href="mailto:wikipedia.earwig@gmail.com">email me</a>. Include the following information:</p>
-<div id="info-box" class="red-box">
-    <pre>${traceback | trim,h}</pre>
-</div>
-<%include file="/support/footer.mako"/>

templates/index.mako

@@ -1,323 +0,0 @@
-<%!
-    from flask import request
-    from copyvios.attribution import get_attribution_info
-    from copyvios.checker import T_POSSIBLE, T_SUSPECT
-    from copyvios.cookies import get_cookies
-    from copyvios.misc import cache
-%>\
-<%
-    titleparts = []
-    if query.page:
-        titleparts.append(query.page.title)
-    titleparts.append("Earwig's Copyvio Detector")
-    title = " | ".join(titleparts)
-    cookies = get_cookies()
-%>\
-<%include file="/support/header.mako" args="title=title, splash=not result"/>
-<%namespace module="copyvios.highlighter" import="highlight_delta"/>\
-<%namespace module="copyvios.misc" import="httpsfix, urlstrip"/>\
-% if notice:
-    <div id="notice-box" class="gray-box">
-        ${notice}
-    </div>
-% endif
-% if query.submitted:
-    % if query.error:
-        <div id="info-box" class="red-box"><p>
-            % if query.error == "bad action":
-                Unknown action: <b><span class="mono">${query.action | h}</span></b>.
-            % elif query.error == "no search method":
-                No copyvio search methods were selected. A check can only be made using the search engine, links present in the page, Turnitin, or some combination of these.
-            % elif query.error == "bad oldid":
-                The revision ID <code>${query.oldid | h}</code> is invalid. It should be an integer.
-            % elif query.error == "no URL":
-                Compare mode requires a URL to be entered. Enter one in the text box below, or choose copyvio search mode to look for content similar to the article elsewhere on the web.
-            % elif query.error == "bad URI":
-                Unsupported URI scheme: <a href="${query.url | h}">${query.url | h}</a>.
-            % elif query.error == "no data":
-                Couldn't find any text in <a href="${query.url | h}">${query.url | h}</a>. <i>Note:</i> only HTML documents, plain text pages, and PDFs are supported, and content generated by JavaScript or found inside iframes is ignored.
-            % elif query.error == "timeout":
-                The URL <a href="${query.url | h}">${query.url | h}</a> timed out before any data could be retrieved.
-            % elif query.error == "search error":
-                An error occurred while using the search engine (${query.error.__cause__}). <i>Note:</i> there is a daily limit on the number of search queries the tool is allowed to make. You may <a href="${request.url | httpsfix, h}&amp;use_engine=0">repeat the check without using the search engine</a>.
-            % else:
-                An unknown error occurred.
-            % endif
-        </p></div>
-    % elif not query.site:
-        <div id="info-box" class="red-box">
-            <p>The given site (project=<b><span class="mono">${query.project | h}</span></b>, language=<b><span class="mono">${query.lang | h}</span></b>) doesn't seem to exist. It may also be closed or private. <a href="https://${query.lang | h}.${query.project | h}.org/">Confirm its URL.</a></p>
-        </div>
-    % elif query.oldid and not result:
-        <div id="info-box" class="red-box">
-            <p>The revision ID couldn't be found: <a href="https://${query.site.domain | h}/w/index.php?oldid=${query.oldid | h}">${query.oldid | h}</a>.</p>
-        </div>
-    % elif query.title and not result:
-        <div id="info-box" class="red-box">
-            <p>The page couldn't be found: <a href="${query.page.url}">${query.page.title | h}</a>.</p>
-        </div>
-    % endif
-%endif
-<p>This tool attempts to detect <a href="https://en.wikipedia.org/wiki/WP:COPYVIO">copyright violations</a> in articles. In <i>search mode</i>, it will check for similar content elsewhere on the web using <a href="https://developers.google.com/custom-search/">Google</a>, external links present in the text of the page, or <a href="https://en.wikipedia.org/wiki/Wikipedia:Turnitin">Turnitin</a> (via <a href="https://en.wikipedia.org/wiki/User:EranBot">EranBot</a>), depending on which options are selected. In <i>compare mode</i>, the tool will compare the article to a specific webpage without making additional searches, like the <a href="https://dupdet.toolforge.org/">Duplication Detector</a>.</p>
-<p>Running a full check can take up to a minute if other websites are slow or if the tool is under heavy use. Please be patient. If you get a timeout, wait a moment and refresh the page.</p>
-<p>Be aware that other websites can copy from Wikipedia, so check the results carefully, especially for older or well-developed articles. Specific websites can be skipped by adding them to the <a href="https://en.wikipedia.org/wiki/User:EarwigBot/Copyvios/Exclusions">excluded URL list</a>.</p>
-<form id="cv-form" action="${request.script_root}/" method="get">
-    <div class="oo-ui-layout oo-ui-horizontalLayout">
-        <label class="site oo-ui-widget oo-ui-widget-enabled oo-ui-labelElement-label oo-ui-labelElement oo-ui-labelWidget">Site</label>
-        <div class="oo-ui-widget oo-ui-widget-enabled oo-ui-inputWidget oo-ui-dropdownInputWidget oo-ui-dropdownInputWidget-php">
-            <select name="lang" required="" class="oo-ui-inputWidget-input oo-ui-indicator-down" title="Language">
-                <% selected_lang = query.orig_lang if query.orig_lang else cookies["CopyviosDefaultLang"].value if "CopyviosDefaultLang" in cookies else cache.bot.wiki.get_site().lang %>\
-                % for code, name in cache.langs:
-                    % if code == selected_lang:
-                        <option value="${code | h}" selected="selected">${name}</option>
-                    % else:
-                        <option value="${code | h}">${name}</option>
-                    % endif
-                % endfor
-            </select>
-        </div>
-        <div class="oo-ui-widget oo-ui-widget-enabled oo-ui-inputWidget oo-ui-dropdownInputWidget oo-ui-dropdownInputWidget-php">
-            <select name="project" required="" class="oo-ui-inputWidget-input oo-ui-indicator-down" title="Project">
-                <% selected_project = query.project if query.project else cookies["CopyviosDefaultProject"].value if "CopyviosDefaultProject" in cookies else cache.bot.wiki.get_site().project %>\
-                % for code, name in cache.projects:
-                    % if code == selected_project:
-                        <option value="${code | h}" selected="selected">${name}</option>
-                    % else:
-                        <option value="${code | h}">${name}</option>
-                    % endif
-                % endfor
-            </select>
-        </div>
-    </div>
-    <div class="oo-ui-layout oo-ui-horizontalLayout">
-        <label for="cv-title" class="page oo-ui-widget oo-ui-widget-enabled oo-ui-labelElement-label oo-ui-labelElement oo-ui-labelWidget">Page</label>
-        <div class="page-title oo-ui-widget oo-ui-widget-enabled oo-ui-inputWidget oo-ui-textInputWidget oo-ui-textInputWidget-type-text oo-ui-textInputWidget-php">
-            <input id="cv-title" type="text" class="oo-ui-inputWidget-input" name="title" placeholder="Title" title="Page title"
-                % if query.title:
-                    value="${query.page.title if query.page else query.title | h}"
-                % endif
-            >
-        </div>
-        <label class="oo-ui-widget oo-ui-widget-enabled oo-ui-labelElement-label oo-ui-labelElement oo-ui-labelWidget">or</label>
-        <div class="page-oldid oo-ui-widget oo-ui-widget-enabled oo-ui-inputWidget oo-ui-textInputWidget oo-ui-textInputWidget-type-text oo-ui-textInputWidget-php">
-            <input id="cv-oldid" type="text" class="oo-ui-inputWidget-input" name="oldid" placeholder="Revision ID" title="Revision ID"
-                % if query.oldid:
-                    value="${query.oldid | h}"
-                % endif
-            >
-        </div>
-    </div>
-    <div class="oo-ui-layout oo-ui-horizontalLayout">
-        <span class="oo-ui-widget oo-ui-widget-enabled">
-            <span class="oo-ui-widget oo-ui-widget-enabled oo-ui-inputWidget oo-ui-radioInputWidget">
-                <input id="action-search" class="oo-ui-inputWidget-input" type="radio" name="action" value="search" ${'checked="checked"' if (query.action == "search" or not query.action) else ""}><span></span>
-            </span>
-            <label for="action-search" class="action oo-ui-widget oo-ui-widget-enabled oo-ui-labelElement-label oo-ui-labelElement oo-ui-labelWidget">Copyvio search</label>
-        </span>
-
-        <input type="hidden" name="use_engine" value="0">
-        <span class="oo-ui-widget oo-ui-widget-enabled">
-            <span class="cv-search-oo-ui oo-ui-widget oo-ui-widget-enabled oo-ui-inputWidget oo-ui-checkboxInputWidget">
-                <input id="cv-cb-engine" class="cv-search oo-ui-inputWidget-input" type="checkbox" name="use_engine" value="1" ${'checked="checked"' if query.use_engine not in ("0", "false") else ""}>
-                <span class="oo-ui-checkboxInputWidget-checkIcon oo-ui-widget oo-ui-widget-enabled oo-ui-iconElement-icon oo-ui-icon-check oo-ui-iconElement oo-ui-labelElement-invisible oo-ui-iconWidget oo-ui-image-invert"></span>
-            </span>
-            <label for="cv-cb-engine" class="oo-ui-widget oo-ui-widget-enabled oo-ui-labelElement-label oo-ui-labelElement oo-ui-labelWidget">Use search engine</label>
-        </span>
-
-        <input type="hidden" name="use_links" value="0">
-        <span class="oo-ui-widget oo-ui-widget-enabled">
-            <span class="cv-search-oo-ui oo-ui-widget oo-ui-widget-enabled oo-ui-inputWidget oo-ui-checkboxInputWidget">
-                <input id="cv-cb-links" class="cv-search oo-ui-inputWidget-input" type="checkbox" name="use_links" value="1" ${'checked="checked"' if query.use_links not in ("0", "false") else ""}>
-                <span class="oo-ui-checkboxInputWidget-checkIcon oo-ui-widget oo-ui-widget-enabled oo-ui-iconElement-icon oo-ui-icon-check oo-ui-iconElement oo-ui-labelElement-invisible oo-ui-iconWidget oo-ui-image-invert"></span>
-            </span>
-            <label for="cv-cb-links" class="oo-ui-widget oo-ui-widget-enabled oo-ui-labelElement-label oo-ui-labelElement oo-ui-labelWidget">Use links in page</label>
-        </span>
-
-        <input type="hidden" name="turnitin" value="0">
-        <span class="oo-ui-widget oo-ui-widget-enabled">
-            <span class="cv-search-oo-ui oo-ui-widget oo-ui-widget-enabled oo-ui-inputWidget oo-ui-checkboxInputWidget">
-                <input id="cv-cb-turnitin" class="cv-search oo-ui-inputWidget-input" type="checkbox" name="turnitin" value="1" ${'checked="checked"' if query.turnitin in ("1", "true") else ""}>
-                <span class="oo-ui-checkboxInputWidget-checkIcon oo-ui-widget oo-ui-widget-enabled oo-ui-iconElement-icon oo-ui-icon-check oo-ui-iconElement oo-ui-labelElement-invisible oo-ui-iconWidget oo-ui-image-invert"></span>
-            </span>
-            <label for="cv-cb-turnitin" class="oo-ui-widget oo-ui-widget-enabled oo-ui-labelElement-label oo-ui-labelElement oo-ui-labelWidget">Use Turnitin</label>
-        </span>
-    </div>
-    <div class="oo-ui-layout oo-ui-horizontalLayout">
-        <span class="oo-ui-widget oo-ui-widget-enabled">
-            <span class="oo-ui-widget oo-ui-widget-enabled oo-ui-inputWidget oo-ui-radioInputWidget">
-                <input id="action-compare" class="oo-ui-inputWidget-input" type="radio" name="action" value="compare" ${'checked="checked"' if query.action == "compare" else ""}><span></span>
-            </span>
-            <label for="action-compare" class="action oo-ui-widget oo-ui-widget-enabled oo-ui-labelElement-label oo-ui-labelElement oo-ui-labelWidget">Copyvio compare</label>
-        </span>
-        <div class="compare-url cv-compare-oo-ui oo-ui-widget oo-ui-widget-enabled oo-ui-inputWidget oo-ui-textInputWidget oo-ui-textInputWidget-type-text oo-ui-textInputWidget-php">
-            <input type="text" class="cv-compare oo-ui-inputWidget-input" name="url" placeholder="URL" title="URL to compare"
-                % if query.url:
-                    value="${query.url | h}"
-                % endif
-            >
-        </div>
-    </div>
-    % if query.nocache or (result and result.cached):
-        <div class="oo-ui-layout oo-ui-horizontalLayout">
-            <span class="cv-search-oo-ui oo-ui-widget oo-ui-widget-enabled oo-ui-inputWidget oo-ui-checkboxInputWidget">
-                <input id="cb-nocache" class="oo-ui-inputWidget-input" type="checkbox" name="nocache" value="1" ${'checked="checked"' if query.nocache else ""}>
-                <span class="oo-ui-checkboxInputWidget-checkIcon oo-ui-widget oo-ui-widget-enabled oo-ui-iconElement-icon oo-ui-icon-check oo-ui-iconElement oo-ui-labelElement-invisible oo-ui-iconWidget oo-ui-image-invert"></span>
-            </span>
-            <label for="cb-nocache">Bypass cache</label>
-        </div>
-    % endif
-    <div class="oo-ui-layout oo-ui-horizontalLayout">
-        <span class="oo-ui-widget oo-ui-widget-enabled oo-ui-buttonElement oo-ui-buttonElement-framed oo-ui-labelElement oo-ui-flaggedElement-primary oo-ui-flaggedElement-progressive oo-ui-buttonWidget">
-            <button type="submit" class="oo-ui-inputWidget-input oo-ui-buttonElement-button">
-                <span class="oo-ui-labelElement-label">Submit</span>
-            </button>
-        </span>
-    </div>
-</form>
-
-% if result:
-    <div id="generation-time">
-        Results
-        % if result.cached:
-            <a id="cv-cached" href="#">cached<span>To save time (and money), this tool will retain the results of checks for up to 72 hours. This includes the URLs of the checked sources, but neither their content nor the content of the article. Future checks on the same page (assuming it remains unchanged) will not involve additional search queries, but a fresh comparison against the source URL will be made. If the page is modified, a new check will be run.</span></a> from <abbr title="${result.cache_time}">${result.cache_age} ago</abbr>. Originally
-        % endif
-        generated in <span class="mono">${round(result.time, 3)}</span>
-        % if query.action == "search":
-            seconds using <span class="mono">${result.queries}</span> quer${"y" if result.queries == 1 else "ies"}.
-        % else:
-            seconds.
-        % endif
-        <a href="${request.script_root | h}/?lang=${query.lang | h}&amp;project=${query.project | h}&amp;oldid=${query.oldid or query.page.lastrevid | h}&amp;action=${query.action | h}&amp;${"use_engine={0}&use_links={1}".format(int(query.use_engine not in ("0", "false")), int(query.use_links not in ("0", "false"))) if query.action == "search" else "" | h}${"url=" if query.action == "compare" else ""}${query.url if query.action == "compare" else "" | u}">Permalink.</a>
-    </div>
-
-    <div id="cv-result" class="${'red' if result.confidence >= T_SUSPECT else 'yellow' if result.confidence >= T_POSSIBLE else 'green'}-box">
-        <table id="cv-result-head-table">
-            <colgroup>
-                <col>
-                <col>
-                <col>
-            </colgroup>
-            <tr>
-                <td>
-                    <a href="${query.page.url}">${query.page.title | h}</a>
-                    % if query.oldid:
-                        @<a href="https://${query.site.domain | h}/w/index.php?oldid=${query.oldid | h}">${query.oldid | h}</a>
-                    % endif
-                    % if query.redirected_from:
-                        <br />
-                        <span id="redirected-from">Redirected from <a href="https://${query.site.domain | h}/w/index.php?title=${query.redirected_from.title | u}&amp;redirect=no">${query.redirected_from.title | h}</a>. <a href="${request.url | httpsfix, h}&amp;noredirect=1">Check original.</a></span>
-                    % endif
-                </td>
-                <td>
-                    <div>
-                        % if result.confidence >= T_SUSPECT:
-                            Violation suspected
-                        % elif result.confidence >= T_POSSIBLE:
-                            Violation possible
-                        % elif result.sources:
-                            Violation unlikely
-                        % else:
-                            No violation
-                        % endif
-                    </div>
-                    <div>${round(result.confidence * 100, 1)}%</div>
-                    <div>similarity</div>
-                </td>
-                <td>
-                    % if result.url:
-                        <a href="${result.url | h}">${result.url | urlstrip, h}</a>
-                    % else:
-                        <span id="result-head-no-sources">No matches found.</span>
-                    % endif
-                </td>
-            </tr>
-        </table>
-    </div>
-
-    <% attrib = get_attribution_info(query.site, query.page) %>
-    % if attrib:
-        <div id="attribution-warning" class="yellow-box">
-            This article contains an attribution template: <code>{{<a href="${attrib[1]}">${attrib[0] | h}</a>}}</code>. Please verify that any potential copyvios are not from properly attributed sources.
-        </div>
-    % endif
-
-    % if query.turnitin_result:
-        <div id="turnitin-container" class="${'red' if query.turnitin_result.reports else 'green'}-box">
-            <div id="turnitin-title">Turnitin Results</div>
-            % if query.turnitin_result.reports:
-                <table id="turnitin-table"><tbody>
-                % for report in turnitin_result.reports:
-                    <tr><td class="turnitin-table-cell"><a href="https://eranbot.toolforge.org/ithenticate.py?rid=${report.reportid}">Report ${report.reportid}</a> for text added at <a href="https://${query.lang}.wikipedia.org/w/index.php?title=${query.title}&amp;diff=${report.diffid}"> ${report.time_posted.strftime("%H:%M, %d %B %Y (UTC)")}</a>:
-                    <ul>
-                    % for source in report.sources:
-                          <li>${source['percent']}% of revision text (${source['words']} words) found at <a href="${source['url'] | h}">${source['url'] | h}</a></li>
-                    % endfor
-                    </ul></td></tr>
-                % endfor
-                </tbody></table>
-            % else:
-                <div id="turnitin-summary">No matching sources found.</div>
-            % endif
-        </div>
-    % endif
-
-    % if query.action == "search":
-        <% skips = False %>
-        <div id="sources-container">
-            <div id="sources-title">Checked Sources</div>
-            % if result.sources:
-                <table id="cv-result-sources">
-                    <colgroup>
-                        <col>
-                        <col>
-                        <col>
-                    </colgroup>
-                    <tr>
-                        <th>URL</th>
-                        <th>Similarity</th>
-                        <th>Compare</th>
-                    </tr>
-                    % for i, source in enumerate(result.sources):
-                        <tr ${'class="source-default-hidden"' if i >= 10 else 'id="source-row-selected"' if i == 0 else ""}>
-                            <td><a ${'id="source-selected"' if i == 0 else ""} class="source-url" href="${source.url | h}">${source.url | h}</a></td>
-                            <td>
-                                % if source.excluded:
-                                    <span class="source-excluded">Excluded</span>
-                                % elif source.skipped:
-                                    <% skips = True %>
-                                    <span class="source-skipped">Skipped</span>
-                                % else:
-                                    <span class="source-similarity ${"source-suspect" if source.confidence >= T_SUSPECT else "source-possible" if source.confidence >= T_POSSIBLE else "source-novio"}">${round(source.confidence * 100, 1)}%</span>
-                                % endif
-                            </td>
-                            <td>
-                                <a href="${request.script_root | h}/?lang=${query.lang | h}&amp;project=${query.project | h}&amp;oldid=${query.oldid or query.page.lastrevid | h}&amp;action=compare&amp;url=${source.url | u}">Compare</a>
-                            </td>
-                        </tr>
-                    % endfor
-                </table>
-            % else:
-                <div class="cv-source-footer">
-                    No sources checked.
-                </div>
-            % endif
-            % if len(result.sources) > 10:
-                <div id="cv-additional" class="cv-source-footer">
-                    ${len(result.sources) - 10} URL${"s" if len(result.sources) > 11 else ""} with lower similarity hidden. <a id="show-additional-sources" href="#">Show them.</a>
-                </div>
-            % endif
-            % if skips or result.possible_miss:
-                <div class="cv-source-footer">
-                    The search ended early because a match was found with high similarity. <a href="${request.url | httpsfix, h}&amp;noskip=1">Do a complete check.</a>
-                </div>
-            % endif
-        </div>
-    % endif
-    <table id="cv-chain-table">
-        <tr>
-            <td class="cv-chain-cell">Article: <div class="cv-chain-detail"><p>${highlight_delta(result.article_chain, result.best.chains[1] if result.best else None)}</p></div></td>
-            <td class="cv-chain-cell">Source: <div class="cv-chain-detail"><p>${highlight_delta(result.best.chains[0], result.best.chains[1]) if result.best else ""}</p></div></td>
-        </tr>
-    </table>
-% endif
-<%include file="/support/footer.mako"/>

templates/settings.mako

@@ -1,100 +0,0 @@
-<%!
-    from json import dumps, loads
-    from flask import request
-    from copyvios.cookies import get_cookies
-    from copyvios.cache import cache
-%>\
-<%
-    cookies = get_cookies()
-%>\
-<%include file="/support/header.mako" args="title='Settings | Earwig\'s Copyvio Detector', splash=True"/>
-% if status:
-    <div id="info-box" class="green-box">
-        <p>${status}</p>
-    </div>
-% endif
-<h2>Settings</h2>
-<p>This page contains some configurable options for the copyvio detector. Settings are saved as cookies.</p>
-<form action="${request.script_root}/settings" method="post">
-    <h3>Default site</h2>
-    <div class="oo-ui-layout oo-ui-labelElement oo-ui-fieldLayout oo-ui-fieldLayout-align-top">
-        <div class="oo-ui-fieldLayout-body">
-            <div class="oo-ui-fieldLayout-field">
-                <div class="oo-ui-widget oo-ui-widget-enabled">
-                    <div class="oo-ui-layout oo-ui-horizontalLayout">
-                        <div class="oo-ui-widget oo-ui-widget-enabled oo-ui-inputWidget oo-ui-dropdownInputWidget oo-ui-dropdownInputWidget-php">
-                            <select name="lang" required="" class="oo-ui-inputWidget-input oo-ui-indicator-down">
-                                <% selected_lang = cookies["CopyviosDefaultLang"].value if "CopyviosDefaultLang" in cookies else default_lang %>\
-                                % for code, name in cache.langs:
-                                    % if code == selected_lang:
-                                        <option value="${code | h}" selected="selected">${name}</option>
-                                    % else:
-                                        <option value="${code | h}">${name}</option>
-                                    % endif
-                                % endfor
-                            </select>
-                        </div>
-                        <div class="oo-ui-widget oo-ui-widget-enabled oo-ui-inputWidget oo-ui-dropdownInputWidget oo-ui-dropdownInputWidget-php">
-                            <select name="project" required="" class="oo-ui-inputWidget-input oo-ui-indicator-down">
-                                <% selected_project = cookies["CopyviosDefaultProject"].value if "CopyviosDefaultProject" in cookies else default_project %>\
-                                % for code, name in cache.projects:
-                                    % if code == selected_project:
-                                        <option value="${code | h}" selected="selected">${name}</option>
-                                    % else:
-                                        <option value="${code | h}">${name}</option>
-                                    % endif
-                                % endfor
-                            </select>
-                        </div>
-                    </div>
-                </div>
-            </div>
-        </div>
-    </div>
-
-    <h3>Background</h2>
-    <%
-        background_options = [
-            ("list", 'Randomly select from <a href="https://commons.wikimedia.org/wiki/User:The_Earwig/POTD">a subset</a> of previous <a href="https://commons.wikimedia.org/">Wikimedia Commons</a> <a href="https://commons.wikimedia.org/wiki/Commons:Picture_of_the_day">Pictures of the Day</a> that work well as widescreen backgrounds, refreshed daily (default).'),
-            ("potd", 'Use the current Commons Picture of the Day, unfiltered. Certain POTDs may be unsuitable as backgrounds due to their aspect ratio or subject matter.'),
-            ("plain", "Use a plain background."),
-        ]
-        selected = cookies["CopyviosBackground"].value if "CopyviosBackground" in cookies else "list"
-    %>\
-    <div class="oo-ui-layout oo-ui-labelElement oo-ui-fieldLayout oo-ui-fieldLayout-align-top">
-        <div class="oo-ui-fieldLayout-body">
-            <div class="oo-ui-fieldLayout-field">
-                <div class="oo-ui-widget oo-ui-widget-enabled oo-ui-inputWidget oo-ui-radioSelectInputWidget">
-                    % for value, desc in background_options:
-                        <div class="oo-ui-layout oo-ui-labelElement oo-ui-fieldLayout oo-ui-fieldLayout-align-inline">
-                            <div class="oo-ui-fieldLayout-body">
-                                <span class="oo-ui-fieldLayout-field">
-                                    <span class="oo-ui-widget oo-ui-widget-enabled oo-ui-inputWidget oo-ui-radioInputWidget">
-                                        <input id="background-${value}" class="oo-ui-inputWidget-input" type="radio" name="background" value="${value}" ${'checked="checked"' if value == selected else ''}><span></span>
-                                    </span>
-                                </span>
-                                <span class="oo-ui-fieldLayout-header">
-                                    <label for="background-${value}" class="oo-ui-labelElement-label">${desc}</label>
-                                </span>
-                            </div>
-                        </div>
-                    % endfor
-                </div>
-            </div>
-        </div>
-    </div>
-
-    <input type="hidden" name="action" value="set"/>
-    <div class="oo-ui-layout oo-ui-fieldLayout oo-ui-fieldLayout-align-left">
-        <div class="oo-ui-fieldLayout-body">
-            <span class="oo-ui-fieldLayout-field">
-                <span class="oo-ui-widget oo-ui-widget-enabled oo-ui-inputWidget oo-ui-buttonElement oo-ui-buttonElement-framed oo-ui-labelElement oo-ui-flaggedElement-primary oo-ui-flaggedElement-progressive oo-ui-labelElement oo-ui-buttonInputWidget">
-                    <button type="submit" class="oo-ui-inputWidget-input oo-ui-buttonElement-button">
-                        <span class="oo-ui-labelElement-label">Save</span>
-                    </button>
-                </span>
-            </span>
-        </div>
-    </div>
-</form>
-<%include file="/support/footer.mako"/>

templates/support/footer.mako

@@ -1,20 +0,0 @@
-<%!
-    from datetime import datetime
-    from flask import g, request
-%>\
-            </main>
-        </div>
-        <div class="padding"></div>
-    </div>
-    <footer>
-        <ul>
-            <li>Maintained by <a href="https://en.wikipedia.org/wiki/User:The_Earwig">Ben Kurtovic</a></li>
-            <li><a href="${request.script_root}/api">API</a></li>
-            <li><a href="https://github.com/earwig/copyvios">Source code</a></li>
-            % if g.descurl:
-                <li><a href="${g.descurl | h}">Background image</a></li>
-            % endif
-        </ul>
-    </footer>
-</body>
-</html>

templates/support/header.mako

@@ -1,34 +0,0 @@
-<%page args="title, splash=False"/>\
-<%!
-    from flask import request, url_for
-    from copyvios.background import get_background
-    from copyvios.cookies import get_cookies
-%>\
-<%
-    cookies = get_cookies()
-%>\
-<!DOCTYPE html>
-<html lang="en">
-<head>
-    <meta charset="utf-8">
-    <title>${title | h}</title>
-    <meta name="viewport" content="width=device-width, initial-scale=1">
-    <link rel="stylesheet" href="https://tools-static.wmflabs.org/cdnjs/ajax/libs/oojs-ui/0.41.3/oojs-ui-core-wikimediaui.min.css" integrity="sha512-xL+tTXAo7a4IAwNrNqBcOGWSqJF6ip0jg4SEda2mapAUxPzfOZQ7inazR4TvSCblHQjwtTOkUDIFtnpaSrg3xg==" crossorigin="anonymous" referrerpolicy="no-referrer"/>
-    <link rel="stylesheet" href="https://tools-static.wmflabs.org/cdnjs/ajax/libs/oojs-ui/0.41.3/oojs-ui-images-wikimediaui.min.css" integrity="sha512-A0LSCuOGH1+SyLhOs4eSKGbNgIEGXgIGh4ytb0GRj9GSUsjmmK6LFzB/E0o9ymRUvD+q7bZyv74XpboQt5qFvQ==" crossorigin="anonymous" referrerpolicy="no-referrer"/>
-    <link rel="stylesheet" href="${request.script_root}${url_for('static', file='style.min.css')}"/>
-    <script src="https://tools-static.wmflabs.org/cdnjs/ajax/libs/jquery/3.3.1/jquery.min.js"></script>
-    <script src="${request.script_root}${url_for('static', file='script.min.js')}"></script>
-</head>
-<% selected = cookies["CopyviosBackground"].value if "CopyviosBackground" in cookies else "list" %>\
-% if selected == "plain":
-    <body>
-% else:
-    <body onload="update_screen_size()" style="background-image: url('${get_background(selected) | h}');">
-% endif
-    <div id="container"${' class="splash"' if splash else ''}>
-        <div id="content">
-            <header>
-                <h1><a href="/">Earwig&apos;s <strong>Copyvio Detector</strong></a></h1>
-                <a id="settings-link" href="/settings">Settings</a>
-            </header>
-            <main>