Browse Source

Initial version of multiple source support

multi-sources
Ben Kurtovic 4 months ago
parent
commit
506183ee16
23 changed files with 960 additions and 545 deletions
  1. +24
    -0
      Makefile
  2. +1
    -1
      README.md
  3. +1
    -4
      app.py
  4. +0
    -24
      build.py
  5. +25
    -8
      copyvios/checker.py
  6. +22
    -17
      copyvios/highlighter.py
  7. +18
    -0
      copyvios/misc.py
  8. +1
    -1
      scripts/log_analyzer.py
  9. +141
    -92
      static/css/style.css
  10. +73
    -0
      static/css/tooltip.css
  11. +233
    -50
      static/script.js
  12. +1
    -1
      static/script.min.js
  13. +1
    -1
      static/style.min.css
  14. +1
    -1
      templates/api.mako
  15. +2
    -2
      templates/error.mako
  16. +2
    -2
      templates/includes/footer.mako
  17. +44
    -0
      templates/includes/form.mako
  18. +3
    -4
      templates/includes/header.mako
  19. +89
    -0
      templates/includes/ooui.mako
  20. +191
    -0
      templates/includes/result.mako
  21. +40
    -0
      templates/includes/site.mako
  22. +8
    -266
      templates/index.mako
  23. +39
    -71
      templates/settings.mako

+ 24
- 0
Makefile View File

@@ -0,0 +1,24 @@
MAKEJS := uglifyjs --compress
MAKECSS := postcss -u cssnano --no-map

.PHONY: all

.INTERMEDIATE: static/style.tmp.css

all: js css

js: static/script.min.js

css: static/style.min.css static/api.min.css

static/script.min.js: static/script.js
$(MAKEJS) -o $@ -- $^

static/style.tmp.css: static/css/*.css
cat $^ > $@

static/style.min.css: static/style.tmp.css
$(MAKECSS) -o $@ $^

static/api.min.css: static/api.css
$(MAKECSS) -o $@ $^

+ 1
- 1
README.md View File

@@ -45,6 +45,6 @@ Running
If additional arguments are needed by `oursql.connect()`, like usernames or
passwords, they should be added to the `_copyviosSQL` section.

- Run `./build.py` to minify JS and CSS files.
- Run `make` to minify JS and CSS files.

- Start the web server (on Toolforge, `webservice uwsgi-python start`).

+ 1
- 4
app.py View File

@@ -110,10 +110,7 @@ def index():
def settings():
status = process_settings() if request.method == "POST" else None
update_sites()
default = cache.bot.wiki.get_site()
kwargs = {"status": status, "default_lang": default.lang,
"default_project": default.project}
return render_template("settings.mako", **kwargs)
return render_template("settings.mako", status=status)

@app.route("/api")
@catch_errors


+ 0
- 24
build.py View File

@@ -1,24 +0,0 @@
#! /usr/bin/env python
# -*- coding: utf-8 -*-

from __future__ import print_function
import os
import subprocess

def process(*args):
print(*args)
content = subprocess.check_output(args)

def main():
root = os.path.join(os.path.dirname(__file__), "static")
for dirpath, dirnames, filenames in os.walk(root):
for filename in filenames:
name = os.path.relpath(os.path.join(dirpath, filename))
if filename.endswith(".js") and ".min." not in filename:
process("uglifyjs", "--compress", "-o", name.replace(".js", ".min.js"), "--", name)
if filename.endswith(".css") and ".min." not in filename:
process("postcss", "-u", "cssnano", "--no-map", name, "-o",
name.replace(".css", ".min.css"))

if __name__ == "__main__":
main()

+ 25
- 8
copyvios/checker.py View File

@@ -37,6 +37,21 @@ def do_check(query=None):
if query.oldid:
query.oldid = query.oldid.strip().lstrip("0")

urls = {}
for key, value in query.query.items():
if not value:
continue
if key == "url":
urls[0] = value
elif key.startswith("url"):
try:
num = int(key[3:])
except ValueError:
continue
urls[num] = value
query.urls = [url for _, url in sorted(urls.items())]
query.url = query.urls[0] if query.urls else None

query.submitted = query.project and query.lang and (query.title or query.oldid)
if query.submitted:
query.site = get_site(query)
@@ -69,7 +84,7 @@ def _get_results(query, follow=True):
return

if not query.action:
query.action = "compare" if query.url else "search"
query.action = "compare" if query.urls else "search"
if query.action == "search":
use_engine = 0 if query.use_engine in ("0", "false") else 1
use_links = 0 if query.use_links in ("0", "false") else 1
@@ -85,15 +100,17 @@ def _get_results(query, follow=True):
# Handle the copyvio check
_perform_check(query, page, use_engine, use_links)
elif query.action == "compare":
if not query.url:
if not query.urls:
query.error = "no URL"
return
scheme = urlparse(query.url).scheme
if not scheme and query.url[0] not in ":/":
query.url = "http://" + query.url
elif scheme not in ["http", "https"]:
query.error = "bad URI"
return
for i, url in enumerate(query.urls):
scheme = urlparse(url).scheme
if not scheme and url[0] not in ":/":
query.urls[i] = "http://" + url
elif scheme not in ["http", "https"]:
query.error = "bad URI"
query.bad_uri = url
return
degree = 5
if query.degree:
try:


+ 22
- 17
copyvios/highlighter.py View File

@@ -8,20 +8,24 @@ from markupsafe import escape

__all__ = ["highlight_delta"]

def highlight_delta(context, chain, delta):
def highlight_delta(context, chain, deltas, index=1):
degree = chain.degree - 1
highlights = [False] * degree
highlights = [None] * degree
block = deque([chain.START] * degree)
if not delta:
delta = EMPTY_INTERSECTION
if deltas is None:
deltas = [EMPTY_INTERSECTION]
if not isinstance(deltas, list):
deltas = [deltas]
for word in chain.text.split() + ([chain.END] * degree):
word = _strip_word(chain, word)
block.append(word)
if tuple(block) in delta.chain:
highlights[-1 * degree:] = [True] * degree
highlights.append(True)
for i, delta in enumerate(deltas, index):
if tuple(block) in delta.chain:
highlights[-1 * degree:] = [i] * degree
highlights.append(i)
break
else:
highlights.append(False)
highlights.append(None)
block.popleft()

i = degree
@@ -36,7 +40,7 @@ def highlight_delta(context, chain, delta):
after = highlights[i + 1]
first = i == degree
last = i - degree + 1 == numwords
words.append(_highlight_word(word, before, after, first, last))
words.append(_highlight_word(word, highlights[i], before, after, first, last))
else:
words.append(unicode(escape(word)))
result.append(u" ".join(words))
@@ -58,24 +62,25 @@ def _get_next(paragraphs):
break
return body

def _highlight_word(word, before, after, first, last):
if before and after:
def _highlight_word(word, this, before, after, first, last):
open_span = u'<span class="cv-hl cv-hl-%s">' % this
if this == before and this == after:
# Word is in the middle of a highlighted block:
res = unicode(escape(word))
if first:
res = u'<span class="cv-hl">' + res
res = open_span + res
if last:
res += u'</span>'
elif after:
elif this == after:
# Word is the first in a highlighted block:
res = u'<span class="cv-hl">' + _fade_word(word, u"in")
res = open_span + _fade_word(word, u"in")
if last:
res += u"</span>"
elif before:
elif this == before:
# Word is the last in a highlighted block:
res = _fade_word(word, u"out") + u"</span>"
if first:
res = u'<span class="cv-hl">' + res
res = open_span + res
else:
res = unicode(escape(word))
return res
@@ -96,4 +101,4 @@ def _fade_word(word, dir):
def _strip_word(chain, word):
if word == chain.START or word == chain.END:
return word
return sub("[^\w\s-]", "", word.lower(), flags=UNICODE)
return sub(r"[^\w\s-]", "", word.lower(), flags=UNICODE)

+ 18
- 0
copyvios/misc.py View File

@@ -1,8 +1,10 @@
# -*- coding: utf-8 -*-

from contextlib import contextmanager
from collections import OrderedDict
import datetime
from os.path import expanduser, join
import urllib

import apsw
from flask import g, request
@@ -116,3 +118,19 @@ def urlstrip(context, url):
if url.endswith("/"):
url = url[:-1]
return url

def get_permalink(context, query):
params = OrderedDict()
params["lang"] = query.lang
params["project"] = query.project
params["oldid"] = query.oldid or query.page.lastrevid
params["action"] = query.action
if query.action == "search":
params["use_engine"] = int(query.use_engine not in ("0", "false"))
params["use_links"] = int(query.use_links not in ("0", "false"))
params["turnitin"] = int(query.turnitin in ("1", "true"))
elif query.action == "compare":
params["url"] = query.url
for i, url in enumerate(query.urls[1:], 2):
params["url%d" % i] = url
return "%s/?%s" % (request.script_root, urllib.urlencode(params))

+ 1
- 1
scripts/log_analyzer.py View File

@@ -11,7 +11,7 @@ REGEX = re.compile(
r'{(?P<vars>\d+) vars in (?P<var_bytes>\d+) bytes} '
r'\[(?P<date>[0-9A-Za-z: ]+)\] (?P<method>\w+) (?P<url>.*?) => '
r'generated (?P<resp_bytes>\d+) bytes in (?P<msecs>\d+) msecs '
r'\((- http://hasty.ai)?(?P<proto>[A-Z0-9/.]+) (?P<status>\d+)\) '
r'\((- http://hasty.ai ?)?(?P<proto>[A-Z0-9/.]+) (?P<status>\d+)\) '
r'(?P<headers>\d+) headers in (?P<header_bytes>\d+) bytes '
r'\((?P<switches>\d+) switches on core (?P<core>\d+)\) '
r'(?P<agent>.*?)'


static/style.css → static/css/style.css View File

@@ -46,8 +46,8 @@ body {

#content {
background-color: #fff;
border: 1px solid #c8ccd1;
filter: drop-shadow(0 0 10px rgba(0, 0, 0, 0.25));
border: 1px solid #a2a9b1;
filter: drop-shadow(0 0 1em rgba(0, 0, 0, 0.25));
margin: 1.5em 3em;
padding: 1em;
}
@@ -102,22 +102,10 @@ footer {
font-size: 0.9em;
text-align: center;
line-height: 1.5;
border-top: 1px solid #c8ccd1;
border-top: 1px solid #a2a9b1;
background: #fff;
}

footer ul {
margin: 0;
}

footer li {
display: inline;
}

footer li:not(:last-child)::after {
content: ' \00b7';
}

footer a {
white-space: nowrap;
}
@@ -158,8 +146,7 @@ h2 {
#sources-container {
padding: 0.5em 1em 1em;
margin: 1em 0;
background-color: #eee;
border: 1px solid #bbb;
border: 1px solid #a2a9b1;
}

#turnitin-title, #sources-title {
@@ -183,29 +170,34 @@ h2 {
}
}

#heading {
width: 100%;
}

#cv-result-sources {
width: 100%;
border-spacing: 0 0.4em;
table-layout: fixed;
}

#cv-result-sources col:nth-child(1) { width: 80%; }
#cv-result-sources col:nth-child(2) { width: 10%; }
#cv-result-sources col:nth-child(3) { width: 10%; }
#cv-result-sources tr:first-child th:nth-child(1) { width: 4em; }
#cv-result-sources tr:first-child th:nth-child(3) { width: 7em; }
#cv-result-sources tr:first-child th:nth-child(4) { width: 10em; }

@media only screen and (max-width: 1000px) {
#cv-result-sources tr:first-child th:nth-child(1) { width: 3em; }
#cv-result-sources tr:first-child th:nth-child(3) { width: 5em; }
#cv-result-sources tr:first-child th:nth-child(4) { width: 8em; }
}

#cv-result-sources th {
text-align: left;
}

#cv-result-sources tr:nth-child(even) {
background-color: #e0e0e0;
background-color: #eaecf0;
}

#cv-result-sources th:first-child, #cv-result-sources td:first-child {
text-align: center;
}

#cv-result-sources td:first-child {
#cv-result-sources td:nth-child(2) {
overflow: hidden;
word-wrap: break-word;
}
@@ -247,12 +239,6 @@ h2 {
font-size: 0.8em;
}

#cv-chain-table {
width: 100%;
border-spacing: 0;
table-layout: fixed;
}

#turnitin-table {
table-layout: fixed;
width: 100%;
@@ -260,12 +246,20 @@ h2 {
border-spacing: 0;
}

#source-row-selected {
background-color: #cfcfcf !important;
#cv-result-sources tr.source-row-selected {
background-color: #d0d2d5;
}

#head-settings {
text-align: right;
.source-row-selected .source-url {
font-weight: bold;
}

.source-row-selected .source-compare {
display: none;
}

.source-row:not(.source-row-selected) .source-compare-selected {
display: none;
}

#cv-result-header {
@@ -282,47 +276,37 @@ h2 {
font-style: italic;
}

#source-selected {
font-weight: bold;
.hlist {
margin: 0;
padding: 0;
}

#cv-cached {
position: relative;
.hlist li {
display: inline;
}

#cv-cached span {
display: none;
position: absolute;
top: 1.5em;
left: -5em;
width: 30em;
padding: 1em;
z-index: 1;
background: #f3f3f3;
border: 1px solid #aaa;
color: black;
font-style: normal;
text-align: left;
.hlist li:not(:last-child)::after {
content: ' \00b7';
}

.green-box {
background-color: #efe;
border: 1px solid #7f7;
background-color: #e0fdf4;
border: 1px solid #54a66d;
}

.yellow-box {
background-color: #ffd;
border: 1px solid #ee5;
background-color: #fef6e7;
border: 1px solid #fc3;
}

.red-box {
background-color: #fee;
border: 1px solid #f77;
background-color: #fee7e6;
border: 1px solid #d33;
}

.gray-box {
background-color: #eee;
border: 1px solid #aaa;
background-color: #eaecf0;
border: 1px solid #a2a9b1;
}

.indentable {
@@ -338,25 +322,30 @@ h2 {
font-style: normal;
}

.cv-chain-detail {
padding: 0 1em;
background-color: #fff;
border: 1px solid #bbb;
.cv-chain-table {
width: 100%;
border-spacing: 0;
table-layout: fixed;
}

.cv-chain-cell {
.cv-chain-table td {
vertical-align: top;
word-wrap: break-word;
}

.cv-chain-cell:first-child {
.cv-chain-table td:first-child {
padding-right: 0.5em;
}

.cv-chain-cell:last-child {
.cv-chain-table td:not(:first-child) {
padding-left: 0.5em;
}

.cv-chain-cell > div {
padding: 0 1em;
border: 1px solid #a2a9b1;
}

.turnitin-table-cell {
padding: 0.5em 0 0.3em 0;
}
@@ -366,18 +355,10 @@ h2 {
line-height: 1.4;
}

.cv-hl {
background: #faa;
}

.cv-hl-in {
background: #fcc;
background: linear-gradient(to left, #faa, #fff);
}

.cv-hl-out {
background: #fcc;
background: linear-gradient(to right, #faa, #fff);
.highlight-demo {
display: inline-block;
border: 1px solid #777;
padding: 0.25em 0.5em;
}

.mono { font-family: monospace; }
@@ -414,14 +395,48 @@ header a:active {
color: #333;
}

#cv-cached:active { color: #040; text-decoration: none; }
#cv-cached:hover { text-decoration: none; }
#cv-cached:hover span { display: block; }
.source-num {
display: inline-block;
min-width: 1.5em;
text-align: center;
}

.source-num-included {
border: 1px solid #777;
}

a.source-num-included {
color: #000;
}

.source-num-included:hover, .source-num-included:active {
text-decoration: none;
}

.hidden {
display: none;
}

#source-tooltips {
position: relative;
}

.source-tooltip-selected a.selector {
display: none;
}

.source-tooltip:not(.source-tooltip-selected) strong.selector {
display: none;
}

.source-tooltip .selector, .source-tooltip .domain, .source-tooltip .wordcount {
display: inline-block;
}

.source-url:link { color: #357; }
.source-url:visited { color: #357; }
.source-url:hover { color: #035; }
.source-url:active { color: #404; }
.cv-chain-cell .cv-hl:hover, .cv-selected {
outline: 1px dashed #333;
cursor: pointer;
}

.oo-ui-horizontalLayout > .oo-ui-textInputWidget,
.oo-ui-horizontalLayout > .oo-ui-dropdownInputWidget {
@@ -454,12 +469,46 @@ label.site, label.page {
min-width: 4em;
}

label.action {
min-width: 10em;
}

@media only screen and (max-width: 720px) {
.oo-ui-horizontalLayout > .oo-ui-widget {
width: 100%;
}
}

/* https://colorbrewer2.org/?type=qualitative&scheme=Pastel1&n=8 */

.cv-hl { background: #dddddd; }
.cv-hl .cv-hl-in { background: linear-gradient(to left, #dddddd, #fff); }
.cv-hl .cv-hl-out { background: linear-gradient(to right, #dddddd, #fff); }

.cv-hl-1 { background: #fed9a6; }
.cv-hl-1 .cv-hl-in { background: linear-gradient(to left, #fed9a6, #fff); }
.cv-hl-1 .cv-hl-out { background: linear-gradient(to right, #fed9a6, #fff); }

.cv-hl-2 { background: #b3cde3; }
.cv-hl-2 .cv-hl-in { background: linear-gradient(to left, #b3cde3, #fff); }
.cv-hl-2 .cv-hl-out { background: linear-gradient(to right, #b3cde3, #fff); }

.cv-hl-3 { background: #ccebc5; }
.cv-hl-3 .cv-hl-in { background: linear-gradient(to left, #ccebc5, #fff); }
.cv-hl-3 .cv-hl-out { background: linear-gradient(to right, #ccebc5, #fff); }

.cv-hl-4 { background: #fbb4ae; }
.cv-hl-4 .cv-hl-in { background: linear-gradient(to left, #fbb4ae, #fff); }
.cv-hl-4 .cv-hl-out { background: linear-gradient(to right, #fbb4ae, #fff); }

.cv-hl-5 { background: #decbe4; }
.cv-hl-5 .cv-hl-in { background: linear-gradient(to left, #decbe4, #fff); }
.cv-hl-5 .cv-hl-out { background: linear-gradient(to right, #decbe4, #fff); }

.cv-hl-6 { background: #ffffcc; }
.cv-hl-6 .cv-hl-in { background: linear-gradient(to left, #ffffcc, #fff); }
.cv-hl-6 .cv-hl-out { background: linear-gradient(to right, #ffffcc, #fff); }

.cv-hl-7 { background: #e5d8bd; }
.cv-hl-7 .cv-hl-in { background: linear-gradient(to left, #e5d8bd, #fff); }
.cv-hl-7 .cv-hl-out { background: linear-gradient(to right, #e5d8bd, #fff); }

.cv-hl-8 { background: #fddaec; }
.cv-hl-8 .cv-hl-in { background: linear-gradient(to left, #fddaec, #fff); }
.cv-hl-8 .cv-hl-out { background: linear-gradient(to right, #fddaec, #fff); }

+ 73
- 0
static/css/tooltip.css View File

@@ -0,0 +1,73 @@
.tooltip {
display: block;
position: absolute;
bottom: 100%;
width: 30em;
padding-bottom: 0.5em;
font-style: normal;
text-align: left;
z-index: 1;
}

.tooltip > span {
display: block;
background-color: white;
border: 1px solid #c8ccd1;
filter: drop-shadow(0 0 0.5em rgba(0, 0, 0, 0.25));
padding: 0.5em 1em;
}

.tooltip > span::after {
content: ' ';
position: absolute;
top: 100%;
border: 0.5em solid transparent;
border-top-color: white;
}

abbr, .tooltip-anchor-inline {
border-width: 0;
text-decoration: underline dotted;
}

.tooltip-anchor-inline {
position: relative;
}

.tooltip-anchor-inline > .tooltip {
display: none;
}

.tooltip-anchor-inline:hover > .tooltip {
display: block;
}

.tooltip-anchor-fixed {
position: absolute;
}

.tooltip-align-center {
left: 50%;
margin-left: -15em;
}

.tooltip-align-center > span::after {
left: 50%;
margin-left: -0.5em;
}

.tooltip-align-left {
left: -2.5em;
}

.tooltip-align-left > span::after {
left: 2em;
}

.tooltip-align-right {
right: -2.5em;
}

.tooltip-align-right > span::after {
right: 2em;
}

+ 233
- 50
static/script.js View File

@@ -1,16 +1,16 @@
function update_screen_size() {
var cache = cache_cookie();
function updateScreenSize() {
var cache = cacheCookie();
var data = {
"width": window.screen.availWidth,
"height": window.screen.availHeight
}
if (!cache || cache["width"] != data["width"] || cache["height"] != data["height"]) {
set_cookie("CopyviosScreenCache", JSON.stringify(data), 1095);
setCookie("CopyviosScreenCache", JSON.stringify(data), 1095);
}
}

function cache_cookie() {
var cookie = get_cookie("CopyviosScreenCache");
function cacheCookie() {
var cookie = getCookie("CopyviosScreenCache");
if (cookie) {
try {
data = JSON.parse(cookie);
@@ -27,7 +27,7 @@ function cache_cookie() {

// Cookie code partially based on http://www.quirksmode.org/js/cookies.html

function get_cookie(name) {
function getCookie(name) {
var nameEQ = name + "=";
var ca = document.cookie.split(";");
for (var i = 0; i < ca.length; i++) {
@@ -45,34 +45,61 @@ function get_cookie(name) {
return null;
}

function set_cookie_with_date(name, value, date) {
function setCookieWithDate(name, value, date) {
value = window.btoa("--cpv2" + value);
var path = window.location.pathname.split("/", 2)[1];
if (date) {
var expires = "; expires=" + date.toUTCString();
var expires = ";expires=" + date.toUTCString();
}
else {
var expires = "";
}
document.cookie = name + "=" + value + expires + "; path=/" + path;
document.cookie = name + "=" + value + expires + ";path=/;samesite=lax";
}

function set_cookie(name, value, days) {
function setCookie(name, value, days) {
if (days) {
var date = new Date();
date.setTime(date.getTime() + (days * 24 * 60 * 60 * 1000));
set_cookie_with_date(name, value, date);
setCookieWithDate(name, value, date);
}
else {
set_cookie_with_date(name, value);
setCookieWithDate(name, value);
}
}

function delete_cookie(name) {
set_cookie(name, "", -1);
function selectTab(e) {
var tab = $(e.target);
if (tab.hasClass("oo-ui-optionWidget-selected")) {
return false;
}
var name = tab.data("name");
var menu = tab.closest(".oo-ui-menuLayout");
menu.find(".oo-ui-optionWidget-selected")
.removeClass("oo-ui-optionWidget-selected")
.attr("aria-selected", "false");
tab.addClass("oo-ui-optionWidget-selected")
.attr("aria-selected", "true");
menu.find(".oo-ui-tabPanelLayout-active")
.removeClass("oo-ui-tabPanelLayout-active")
.addClass("oo-ui-element-hidden")
.attr("aria-hidden", "true");
menu.find('.oo-ui-tabPanelLayout[data-name="' + name + '"]')
.addClass("oo-ui-tabPanelLayout-active")
.removeClass("oo-ui-element-hidden")
.removeAttr("aria-hidden");
return false;
}

function submitForm() {
$("#cv-form button[type='submit']")
.prop("disabled", true)
.css("cursor", "progress")
.parent()
.addClass("oo-ui-widget-disabled")
.removeClass("oo-ui-widget-enabled");
}

function toggle_notice() {
function toggleNotice() {
var details = $("#notice-collapse-box"),
trigger = $("#notice-collapse-trigger");
if (details.is(":hidden")) {
@@ -85,16 +112,16 @@ function toggle_notice() {
}
}

function install_notice() {
function setNotice() {
var details = $("#notice-collapse-box"),
trigger = $("#notice-collapse-trigger");
if (details.length >= 0 && trigger.length >= 0) {
trigger.replaceWith($("<a/>", {
trigger.replaceWith($("<a>", {
id: "notice-collapse-trigger",
href: "#",
text: "[show]",
click: function() {
toggle_notice();
toggleNotice();
return false;
}
}));
@@ -102,42 +129,173 @@ function install_notice() {
}
}

$(document).ready(function() {
$("#action-search").change(function() {
$(".cv-search").prop("disabled", false);
$(".cv-compare").prop("disabled", true);
$(".cv-search-oo-ui").addClass("oo-ui-widget-enabled").removeClass("oo-ui-widget-disabled");
$(".cv-compare-oo-ui").addClass("oo-ui-widget-disabled").removeClass("oo-ui-widget-enabled");
function addUrl() {
var template = $("#compare-new-url");
var widget = template[0].content.cloneNode(true);
$(widget).find("input").prop("name", "url" + ($(".compare-url").length + 1));
$(widget).find(".compare-remove-url").click(removeUrl);
template.before(widget);
return false;
}

function removeUrl(e) {
$(e.target).closest(".oo-ui-layout").remove();
$(".compare-url:not(.compare-url-first)").each(function(i, e) {
$(e).find("input").prop("name", "url" + (i + 2));
});
$("#action-compare").change(function() {
$(".cv-search").prop("disabled", true);
$(".cv-compare").prop("disabled", false);
$(".cv-search-oo-ui").addClass("oo-ui-widget-disabled").removeClass("oo-ui-widget-enabled");
$(".cv-compare-oo-ui").addClass("oo-ui-widget-enabled").removeClass("oo-ui-widget-disabled");
return false;
}

function pasteText() {
// TODO
return false;
}

function uploadFile() {
// TODO
return false;
}

function selectResult(n) {
var select = $(".cv-chain-source-" + n);
if (select.length === 0) {
return;
}
$(".cv-chain-source").addClass("hidden");
select.removeClass("hidden");
$(".source-row-selected").removeClass("source-row-selected");
$($(".source-row")[n - 1]).addClass("source-row-selected");
$(".source-tooltip > .tooltip-align-right").remove();
$(".source-tooltip-selected").removeClass("source-tooltip-selected");
$(".source-tooltip").filter(function(i, elem) {
return elem.dataset.id === n.toString();
}).addClass("source-tooltip-selected");
}

function setResultSelectionHandlers() {
$(".source-compare").click(function(e) {
selectResult($(e.target).data("id"));
return false;
});

if ($("#action-search" ).is(":checked")) $("#action-search" ).change();
if ($("#action-compare").is(":checked")) $("#action-compare").change();

$("#cv-form").submit(function() {
if ($("#action-search").is(":checked")) {
var hidden = [
["engine", "use_engine"], ["links", "use_links"],
["turnitin", "turnitin"]];
$.each(hidden, function(i, val) {
if ($("#cv-cb-" + val[0]).is(":checked"))
$("#cv-form input[type='hidden'][name='" + val[1] + "']")
.prop("disabled", true);
});
$("#cv-result-sources tr:not(:first-child)").click(function(e) {
if (e.target.tagName === "TD") {
selectResult($(e.target).parent().data("id"));
return false;
}
$("#cv-form button[type='submit']")
.prop("disabled", true)
.css("cursor", "progress")
.parent()
.addClass("oo-ui-widget-disabled")
.removeClass("oo-ui-widget-enabled");
});
}

function toggleSource(e) {
var el = $(e.target),
id = el.data("id");
if (el.hasClass("cv-hl")) {
$(".cv-hl-" + id)
.addClass("cv-hl-disabled-" + id)
.removeClass(["cv-hl-" + id, "cv-hl"]);
} else {
$(".cv-hl-disabled-" + id)
.addClass(["cv-hl-" + id, "cv-hl"])
.removeClass("cv-hl-disabled-" + id);
}
return false;
}

function unselectRegions() {
if ($(".source-tooltip, .cv-selected").length > 0) {
$(".source-tooltip").remove();
$(".cv-selected").removeClass("cv-selected");
return false;
}
}

function selectRegion(e) {
unselectRegions();
var target = $(e.target).closest(".cv-hl");
if (target.length === 0) {
return;
}
var hls = [].slice.apply(target[0].classList).filter(function(c) {
return c.startsWith("cv-hl-");
});
if (hls.length === 0) {
return;
}
var num = parseInt(hls[0].substr(6));
var url = null, selected = true;
if ($("#cv-result-sources").length > 0) {
url = $(".source-url-" + num);
if (url.length === 0) {
return;
}
selected = $(".source-row-selected").data("id") === num;
}
var wordcount = target.text().split(/\s+/).filter(function(s) { return s != '' }).length;
var width;

var contents = $("<span>");
if (url !== null) {
var domain = url.data("domain") || url.text();
contents.append(
$("<a>", {
class: "selector",
href: "#",
title: "Select source",
})
.text("Source " + num)
.click(function() {
selectResult(num);
return false;
})
).append(
$("<strong>", {class: "selector"})
.text("Source " + num)
).append(" ")
.append(
$("<span>", {class: "domain"})
.text("(" + domain + "):")
).append(" ");
width = Math.min(15 + domain.length / 2, 30);
} else {
width = 8;
}
contents
.append(
$("<span>", {class: "wordcount"})
.text(wordcount.toString() + " words")
).click(function() {
if ($(".source-row-selected").data("id") === num) {
unselectRegions();
} else {
selectResult(num);
}
return false;
});

var container = $("#source-tooltips");
var containerOffset = container.offset();
var chain = target.closest(".cv-chain-cell");
var tooltipDirection = chain.hasClass("cv-chain-source") ? "right" : "left";
var tooltip = $("<div>", {class: "source-tooltip tooltip-anchor-fixed"})
.css({
top: (e.pageY - containerOffset.top) + "px",
left: (e.pageX - containerOffset.left) + "px",
})
.append(
$("<span>", {class: "tooltip tooltip-align-" + tooltipDirection})
.css({
width: width + "em",
}).append(contents)
).appendTo(container)
.attr("data-id", num.toString());
if (selected) {
tooltip.addClass("source-tooltip-selected");
}
target.addClass("cv-selected");
return false;
}

function hideAdditionalSources() {
if ($("#cv-additional").length >= 0) {
$("#cv-additional").css("display", "block");
$(".source-default-hidden").css("display", "none");
@@ -147,6 +305,31 @@ $(document).ready(function() {
return false;
});
}
}

$(document).ready(function() {
$(".oo-ui-optionWidget").click(selectTab);
$("#compare-add-url").click(addUrl);
$("#compare-paste").click(pasteText);
$("#compare-upload").click(uploadFile);
$(".compare-remove-url").click(removeUrl);

setResultSelectionHandlers();

$(".source-num-included").click(toggleSource);

$(".cv-chain-cell .cv-hl").click(selectRegion);
$("body").click(unselectRegions);

$(document).keyup(function(e) {
if (e.key === "Escape") {
return unselectRegions();
}
});

$("#cv-form").submit(submitForm);

hideAdditionalSources();

install_notice();
setNotice();
});

+ 1
- 1
static/script.min.js
File diff suppressed because it is too large
View File


+ 1
- 1
static/style.min.css
File diff suppressed because it is too large
View File


+ 1
- 1
templates/api.mako View File

@@ -33,7 +33,7 @@
<html lang="en">
<head>
<meta charset="utf-8">
<title>API | Earwig's Copyvio Detector</title>
<title>API - Earwig's Copyvio Detector</title>
<link rel="stylesheet" href="${request.script_root}${url_for('static', file='api.min.css')}" type="text/css" />
</head>
<body>


+ 2
- 2
templates/error.mako View File

@@ -1,7 +1,7 @@
<%include file="/support/header.mako" args="title='Error! | Earwig\'s Copyvio Detector'"/>
<%include file="/includes/header.mako" args="title='Error! - Earwig\'s Copyvio Detector'"/>
<h2>Error!</h2>
<p>An error occurred. If it hasn't been reported (<a href="https://github.com/earwig/copyvios/issues">try to check</a>), please <a href="https://github.com/earwig/copyvios/issues/new">file an issue</a> or <a href="mailto:wikipedia.earwig@gmail.com">email me</a>. Include the following information:</p>
<div id="info-box" class="red-box">
<pre>${traceback | trim,h}</pre>
</div>
<%include file="/support/footer.mako"/>
<%include file="/includes/footer.mako"/>

templates/support/footer.mako → templates/includes/footer.mako View File

@@ -1,13 +1,13 @@
<%!
from datetime import datetime
from flask import g, request
%>\
%>
</main>
</div>
<div class="padding"></div>
</div>
<footer>
<ul>
<ul class="hlist">
<li>Maintained by <a href="https://en.wikipedia.org/wiki/User:The_Earwig">Ben Kurtovic</a></li>
<li><a href="${request.script_root}/api">API</a></li>
<li><a href="https://github.com/earwig/copyvios">Source code</a></li>

+ 44
- 0
templates/includes/form.mako View File

@@ -0,0 +1,44 @@
<%!
from flask import g, request
from copyvios.misc import cache
%>
<%namespace name="ooui" file="/includes/ooui.mako"/>
<form id="cv-form" action="${request.script_root}/" method="get">
<%ooui:horizontal_layout>
<label class="site oo-ui-widget oo-ui-widget-enabled oo-ui-labelElement-label oo-ui-labelElement oo-ui-labelWidget">Site</label>
<%include file="/includes/site.mako" args="selected_lang=query.orig_lang, selected_project=query.project"/>
</%ooui:horizontal_layout>
<%ooui:horizontal_layout>
<label for="cv-title" class="page oo-ui-widget oo-ui-widget-enabled oo-ui-labelElement-label oo-ui-labelElement oo-ui-labelWidget">Page</label>
<%ooui:text classes="page-title">
<input id="cv-title" type="text" class="oo-ui-inputWidget-input" name="title" placeholder="Title" title="Page title"
% if query.title:
value="${query.page.title if query.page else query.title | h}"
% endif
>
</%ooui:text>
<label class="oo-ui-widget oo-ui-widget-enabled oo-ui-labelElement-label oo-ui-labelElement oo-ui-labelWidget">or</label>
<%ooui:text classes="page-oldid">
<input id="cv-oldid" type="text" class="oo-ui-inputWidget-input" name="oldid" placeholder="Revision ID" title="Revision ID"
% if query.oldid:
value="${query.oldid | h}"
% endif
>
</%ooui:text>
</%ooui:horizontal_layout>

<%ooui:menu_layout>
<%ooui:menu_layout_menu>
${ooui.menu_layout_tab('search', 'Copyvio search', selected=query.action == "search" or not query.action)}
${ooui.menu_layout_tab('compare', 'Copyvio compare', selected=query.action == "compare")}
</%ooui:menu_layout_menu>
<%ooui:menu_layout_content>
<%ooui:menu_layout_panel name="search" active="${query.action == 'search' or not query.action}">
Search!
</%ooui:menu_layout_panel>
<%ooui:menu_layout_panel name="compare" active="${query.action == 'compare'}">
Compare!
</%ooui:menu_layout_panel>
</%ooui:menu_layout_content>
</%ooui:menu_layout>
</form>

templates/support/header.mako → templates/includes/header.mako View File

@@ -9,17 +9,16 @@
<meta charset="utf-8">
<title>${title | h}</title>
<meta name="viewport" content="width=device-width, initial-scale=1">
<link rel="stylesheet" href="https://tools-static.wmflabs.org/cdnjs/ajax/libs/oojs-ui/0.41.3/oojs-ui-core-wikimediaui.min.css" integrity="sha512-xL+tTXAo7a4IAwNrNqBcOGWSqJF6ip0jg4SEda2mapAUxPzfOZQ7inazR4TvSCblHQjwtTOkUDIFtnpaSrg3xg==" crossorigin="anonymous" referrerpolicy="no-referrer"/>
<link rel="stylesheet" href="https://tools-static.wmflabs.org/cdnjs/ajax/libs/oojs-ui/0.41.3/oojs-ui-images-wikimediaui.min.css" integrity="sha512-A0LSCuOGH1+SyLhOs4eSKGbNgIEGXgIGh4ytb0GRj9GSUsjmmK6LFzB/E0o9ymRUvD+q7bZyv74XpboQt5qFvQ==" crossorigin="anonymous" referrerpolicy="no-referrer"/>
<link rel="stylesheet" href="https://tools-static.wmflabs.org/cdnjs/ajax/libs/oojs-ui/0.41.3/oojs-ui-wikimediaui.min.css" integrity="sha512-NfHDuNXQxgngdmLBodQLDR2DAkT+hFpALuQv4TvRXC2AiDklxQHji6+KCFMrR/EOrUpaq30yc4CMP+aQ39kwXA==" crossorigin="anonymous" referrerpolicy="no-referrer" />
<link rel="stylesheet" href="${request.script_root}${url_for('static', file='style.min.css')}"/>
<script src="https://tools-static.wmflabs.org/cdnjs/ajax/libs/jquery/3.3.1/jquery.min.js"></script>
<script src="${request.script_root}${url_for('static', file='script.min.js')}"></script>
</head>
<% selected = g.cookies["CopyviosBackground"].value if "CopyviosBackground" in g.cookies else "list" %>\
<% selected = g.cookies["CopyviosBackground"].value if "CopyviosBackground" in g.cookies else "list" %>
% if selected == "plain":
<body>
% else:
<body onload="update_screen_size()" style="background-image: url('${set_background(selected) | h}');">
<body onload="updateScreenSize()" style="background-image: url('${set_background(selected) | h}');">
% endif
<div id="container"${' class="splash"' if splash else ''}>
<div id="content">

+ 89
- 0
templates/includes/ooui.mako View File

@@ -0,0 +1,89 @@
<%def name="widget(classes='', raw_classes='')">
<div class="oo-ui-widget oo-ui-widget-enabled ${' '.join('oo-ui-%s' % cls for cls in classes.strip().split())} ${' '.join(raw_classes.strip().split())}">
${caller.body()}
</div>
</%def>

<%def name="field_layout(align='top')">
<div class="oo-ui-layout oo-ui-fieldLayout oo-ui-fieldLayout-align-${align} oo-ui-labelElement">
<div class="oo-ui-fieldLayout-body">
<div class="oo-ui-fieldLayout-field">
${caller.body()}
</div>
</div>
</div>
</%def>

<%def name="field_layout_header()">
</div>
<div class="oo-ui-fieldLayout-header">
${caller.body()}
</%def>

<%def name="horizontal_layout()">
<div class="oo-ui-layout oo-ui-horizontalLayout">
${caller.body()}
</div>
</%def>

<%def name="menu_layout()">
<div class="oo-ui-layout oo-ui-menuLayout oo-ui-menuLayout-static oo-ui-menuLayout-top oo-ui-menuLayout-showMenu oo-ui-indexLayout">
${caller.body()}
</div>
</%def>

<%def name="menu_layout_menu(frame_style='frameless')">
<div class="oo-ui-menuLayout-menu">
<div class="oo-ui-layout oo-ui-panelLayout oo-ui-indexLayout-tabPanel">
<div role="tablist" tabindex="0" class="oo-ui-selectWidget oo-ui-selectWidget-unpressed oo-ui-widget oo-ui-widget-enabled oo-ui-tabSelectWidget oo-ui-tabSelectWidget-${frame_style}">
${caller.body()}
</div>
</div>
</div>
</%def>

<%def name="menu_layout_tab(name, label, selected=False)">
<div aria-selected="${'true' if selected else 'false'}" role="tab" class="oo-ui-widget oo-ui-widget-enabled oo-ui-optionWidget oo-ui-tabOptionWidget oo-ui-labelElement${' oo-ui-optionWidget-selected' if selected else ''}" data-name="${name}">
<span class="oo-ui-labelElement-label">${label}</span>
</div>
</%def>

<%def name="menu_layout_content()">
<div class="oo-ui-menuLayout-content">
<div class="oo-ui-layout oo-ui-panelLayout oo-ui-stackLayout oo-ui-indexLayout-stackLayout">
${caller.body()}
</div>
</div>
</%def>

<%def name="menu_layout_panel(name, active='false')">
<div role="tabpanel"${'' if active else ' aria-hidden="true"'} class="oo-ui-layout oo-ui-panelLayout oo-ui-panelLayout-scrollable oo-ui-tabPanelLayout ${'oo-ui-tabPanelLayout-active' if active else 'oo-ui-element-hidden'}" data-name="${name}">
${caller.body()}
</div>
</%def>

<%def name="radio_select()">
<%self:widget classes="inputWidget radioSelectInputWidget">
${caller.body()}
</%self:widget>
</%def>

<%def name="radio()">
<%self:widget classes="inputWidget radioInputWidget">
${caller.body()}
</%self:widget>
</%def>

<%def name="text(classes='')">
<%self:widget classes="inputWidget textInputWidget textInputWidget-type-text textInputWidget-php" raw_classes="${classes}">
${caller.body()}
</%self:widget>
</%def>

<%def name="submit_button(label)">
<%self:widget classes="inputWidget buttonElement buttonElement-framed labelElement flaggedElement-primary flaggedElement-progressive buttonInputWidget">
<button type="submit" class="oo-ui-inputWidget-input oo-ui-buttonElement-button">
<span class="oo-ui-labelElement-label">${label}</span>
</button>
</%self:widget>
</%def>

+ 191
- 0
templates/includes/result.mako View File

@@ -0,0 +1,191 @@
<%!
from flask import request
from copyvios.attribution import get_attribution_info
from copyvios.checker import T_POSSIBLE, T_SUSPECT
%>
<%namespace module="copyvios.highlighter" import="highlight_delta"/>
<%namespace module="copyvios.misc" import="get_permalink, httpsfix, urlstrip"/>
<div id="generation-time">
Results
% if result.cached:
<span class="tooltip-anchor-inline">cached<span class="tooltip tooltip-align-center"><span>To save time (and money), this tool will retain the results of checks for up to 72 hours. This includes the URLs of the checked sources, but neither their content nor the content of the article. Future checks on the same page (assuming it remains unchanged) will not involve additional search queries, but a fresh comparison against the source URL will be made. If the page is modified, a new check will be run.</span></span></span> from <abbr title="${result.cache_time}">${result.cache_age} ago</abbr>. Originally
% endif
generated in <span class="mono">${round(result.time, 3)}</span>
% if query.action == "search":
seconds using <span class="mono">${result.queries}</span> quer${"y" if result.queries == 1 else "ies"}.
% else:
seconds.
% endif
<a href="${ get_permalink(query) | h}">Permalink.</a>
</div>

<div id="cv-result" class="${'red' if result.confidence >= T_SUSPECT else 'yellow' if result.confidence >= T_POSSIBLE else 'green'}-box">
<table id="cv-result-head-table">
<colgroup>
<col>
<col>
<col>
</colgroup>
<tr>
<td>
<a href="${query.page.url}">${query.page.title | h}</a>
% if query.oldid:
@<a href="https://${query.site.domain | h}/w/index.php?oldid=${query.oldid | h}">${query.oldid | h}</a>
% endif
% if query.redirected_from:
<br />
<span id="redirected-from">Redirected from <a href="https://${query.site.domain | h}/w/index.php?title=${query.redirected_from.title | u}&amp;redirect=no">${query.redirected_from.title | h}</a>. <a href="${request.url | httpsfix, h}&amp;noredirect=1">Check original.</a></span>
% endif
</td>
<td>
<div>
% if result.confidence >= T_SUSPECT:
Violation suspected
% elif result.confidence >= T_POSSIBLE:
Violation possible
% elif result.sources:
Violation unlikely
% else:
No violation
% endif
</div>
<div>${int(round(result.confidence * 100))}%</div>
<div>similarity</div>
</td>
<td>
% if result.url:
<a href="${result.url | h}">${result.url | urlstrip, h}</a>
% if len(result.included_sources) == 2:
<br>and <a href="${result.included_sources[1].url | h}">${result.included_sources[1].url | urlstrip, h}</a>
% elif len(result.included_sources) > 2:
<br>and ${len(result.included_sources) - 1} other sources
% endif
% else:
<span id="result-head-no-sources">No matches found.</span>
% endif
</td>
</tr>
</table>
</div>

<% attrib = get_attribution_info(query.site, query.page) %>
% if attrib:
<div id="attribution-warning" class="yellow-box">
This article contains an attribution template: <code>{{<a href="${attrib[1]}">${attrib[0] | h}</a>}}</code>. Please verify that any potential copyvios are not from properly attributed sources.
</div>
% endif

% if query.turnitin_result:
<div id="turnitin-container" class="${'red' if query.turnitin_result.reports else 'green'}-box">
<div id="turnitin-title">Turnitin Results</div>
% if query.turnitin_result.reports:
<table id="turnitin-table"><tbody>
% for report in turnitin_result.reports:
<tr><td class="turnitin-table-cell"><a href="https://eranbot.toolforge.org/ithenticate.py?rid=${report.reportid}">Report ${report.reportid}</a> for text added at <a href="https://${query.lang}.wikipedia.org/w/index.php?title=${query.title}&amp;diff=${report.diffid}"> ${report.time_posted.strftime("%H:%M, %d %B %Y (UTC)")}</a>:
<ul>
% for source in report.sources:
<li>${source['percent']}% of revision text (${source['words']} words) found at <a href="${source['url'] | h}">${source['url'] | h}</a></li>
% endfor
</ul></td></tr>
% endfor
</tbody></table>
% else:
<div id="turnitin-summary">No matching sources found.</div>
% endif
</div>
% endif

% if query.action == "search" or len(result.sources) > 1:
<% skips = False %>
<div id="sources-container">
<div id="sources-title">Checked sources</div>
% if result.sources:
<table id="cv-result-sources">
</colgroup>
<tr>
<th>#</th>
<th>URL</th>
<th>Similarity</th>
<th>Actions</th>
</tr>
% for i, source in enumerate(result.sources):
<tr class="source-row ${"source-default-hidden" if i >= 10 else "source-row-selected" if i == 0 else ""}" data-id="${i + 1}">
<td>
% if i < len(result.included_sources):
<a class="source-num source-num-included cv-hl cv-hl-${i + 1}" href="#" title="Toggle highlighting" data-id="${i + 1}">${i + 1}</a>
% else:
<span class="source-num">${i + 1}</span>
% endif
</td>
<td>
<a class="source-url source-url-${i + 1}" href="${source.url | h}" data-domain="${source.domain or '' | h}">${source.url | h}</a>
</td>
<td>
% if source.excluded:
<span class="source-excluded">Excluded</span>
% elif source.skipped:
<% skips = True %>
<span class="source-skipped">Skipped</span>
% else:
<span class="source-similarity ${"source-suspect" if source.confidence >= T_SUSPECT else "source-possible" if source.confidence >= T_POSSIBLE else "source-novio"}">${int(round(source.confidence * 100))}%</span>
% endif
</td>
<td>
<ul class="hlist">
% if i < len(result.included_sources):
<li>
<a class="source-compare" href="#" title="View this source" data-id="${i + 1}">Select</a>
<strong class="source-compare-selected">Select</strong>
</li>
% endif
<li>
<a href="${request.script_root | h}/?lang=${query.lang | h}&amp;project=${query.project | h}&amp;oldid=${query.oldid or query.page.lastrevid | h}&amp;action=compare&amp;url=${source.url | u}" title="Open a direct comparison to this source">Open</a>
</li>
</ul>
</td>
</tr>
% endfor
</table>
% else:
<div class="cv-source-footer">
No sources checked.
</div>
% endif
% if len(result.sources) > 10:
<div id="cv-additional" class="cv-source-footer">
${len(result.sources) - 10} URL${"s" if len(result.sources) > 11 else ""} with lower similarity hidden. <a id="show-additional-sources" href="#">Show them.</a>
</div>
% endif
% if skips or result.possible_miss:
<div class="cv-source-footer">
The search ended early because a match was found with high similarity. <a href="${request.url | httpsfix, h}&amp;noskip=1">Do a complete check.</a>
</div>
% endif
</div>
% endif
<div id="source-tooltips"></div>
<table class="cv-chain-table">
<tr>
<td class="cv-chain-article">
Article:
</td>
% for i, source in enumerate(result.included_sources, 1):
<td class="cv-chain-source cv-chain-source-${i} ${"hidden" if i > 1 else ""}">
Source${" %s (%s)" % (i, source.domain or source.url) if len(result.included_sources) > 1 else "" | h}:
</td>
% endfor
</tr>
<tr>
<td class="cv-chain-cell cv-chain-article">
<div><p>${highlight_delta(result.article_chain, [source.chains[1] for source in result.included_sources])}</p></div>
</td>
<td class="cv-chain-cell cv-chain-source cv-chain-source-1">
<div><p>${highlight_delta(result.best.chains[0], result.best.chains[1]) if result.best else ""}</p></div>
</td>
% for i, source in enumerate(result.included_sources[1:], 2):
<td class="cv-chain-cell cv-chain-source cv-chain-source-${i} hidden">
<div><p>${highlight_delta(source.chains[0], source.chains[1], index=i)}</p></div>
</td>
% endfor
</tr>
</table>

+ 40
- 0
templates/includes/site.mako View File

@@ -0,0 +1,40 @@
<%page args="selected_lang=None, selected_project=None"/>\
<%!
from flask import g
from copyvios.misc import cache
%>\
<%
if selected_lang is None:
if "CopyviosDefaultLang" in g.cookies:
selected_lang = g.cookies["CopyviosDefaultLang"].value
else:
selected_lang = cache.bot.wiki.get_site().lang

if selected_project is None:
if "CopyviosDefaultProject" in g.cookies:
selected_project = g.cookies["CopyviosDefaultProject"].value
else:
selected_project = cache.bot.wiki.get_site().project
%>
<div class="oo-ui-widget oo-ui-widget-enabled oo-ui-inputWidget oo-ui-dropdownInputWidget oo-ui-dropdownInputWidget-php">
<select name="lang" required="" class="oo-ui-inputWidget-input oo-ui-indicator-down">
% for code, name in cache.langs:
% if code == selected_lang:
<option value="${code | h}" selected="selected">${name}</option>
% else:
<option value="${code | h}">${name}</option>
% endif
% endfor
</select>
</div>
<div class="oo-ui-widget oo-ui-widget-enabled oo-ui-inputWidget oo-ui-dropdownInputWidget oo-ui-dropdownInputWidget-php">
<select name="project" required="" class="oo-ui-inputWidget-input oo-ui-indicator-down">
% for code, name in cache.projects:
% if code == selected_project:
<option value="${code | h}" selected="selected">${name}</option>
% else:
<option value="${code | h}">${name}</option>
% endif
% endfor
</select>
</div>

+ 8
- 266
templates/index.mako View File

@@ -1,19 +1,15 @@
<%!
from flask import g, request
from copyvios.attribution import get_attribution_info
from copyvios.checker import T_POSSIBLE, T_SUSPECT
from copyvios.misc import cache
from flask import request
%>\
<%
titleparts = []
if query.page:
titleparts.append(query.page.title)
titleparts.append("Earwig's Copyvio Detector")
title = " | ".join(titleparts)
title = " - ".join(titleparts)
%>\
<%include file="/support/header.mako" args="title=title, splash=not result"/>
<%namespace module="copyvios.highlighter" import="highlight_delta"/>\
<%namespace module="copyvios.misc" import="httpsfix, urlstrip"/>\
<%include file="/includes/header.mako" args="title=title, splash=not result"/>
<%namespace module="copyvios.misc" import="httpsfix"/>
% if notice:
<div id="notice-box" class="gray-box">
${notice}
@@ -31,7 +27,7 @@
% elif query.error == "no URL":
Compare mode requires a URL to be entered. Enter one in the text box below, or choose copyvio search mode to look for content similar to the article elsewhere on the web.
% elif query.error == "bad URI":
Unsupported URI scheme: <a href="${query.url | h}">${query.url | h}</a>.
Unsupported URI scheme: <a href="${query.bad_uri | h}">${query.bad_uri | h}</a>.
% elif query.error == "no data":
Couldn't find any text in <a href="${query.url | h}">${query.url | h}</a>. <i>Note:</i> only HTML documents, plain text pages, and PDFs are supported, and content generated by JavaScript or found inside iframes is ignored.
% elif query.error == "timeout":
@@ -59,263 +55,9 @@
<p>This tool attempts to detect <a href="https://en.wikipedia.org/wiki/WP:COPYVIO">copyright violations</a> in articles. In <i>search mode</i>, it will check for similar content elsewhere on the web using <a href="https://developers.google.com/custom-search/">Google</a>, external links present in the text of the page, or <a href="https://en.wikipedia.org/wiki/Wikipedia:Turnitin">Turnitin</a> (via <a href="https://en.wikipedia.org/wiki/User:EranBot">EranBot</a>), depending on which options are selected. In <i>compare mode</i>, the tool will compare the article to a specific webpage without making additional searches, like the <a href="https://dupdet.toolforge.org/">Duplication Detector</a>.</p>
<p>Running a full check can take up to a minute if other websites are slow or if the tool is under heavy use. Please be patient. If you get a timeout, wait a moment and refresh the page.</p>
<p>Be aware that other websites can copy from Wikipedia, so check the results carefully, especially for older or well-developed articles. Specific websites can be skipped by adding them to the <a href="https://en.wikipedia.org/wiki/User:EarwigBot/Copyvios/Exclusions">excluded URL list</a>.</p>
<form id="cv-form" action="${request.script_root}/" method="get">
<div class="oo-ui-layout oo-ui-horizontalLayout">
<label class="site oo-ui-widget oo-ui-widget-enabled oo-ui-labelElement-label oo-ui-labelElement oo-ui-labelWidget">Site</label>
<div class="oo-ui-widget oo-ui-widget-enabled oo-ui-inputWidget oo-ui-dropdownInputWidget oo-ui-dropdownInputWidget-php">
<select name="lang" required="" class="oo-ui-inputWidget-input oo-ui-indicator-down" title="Language">
<% selected_lang = query.orig_lang if query.orig_lang else g.cookies["CopyviosDefaultLang"].value if "CopyviosDefaultLang" in g.cookies else cache.bot.wiki.get_site().lang %>\
% for code, name in cache.langs:
% if code == selected_lang:
<option value="${code | h}" selected="selected">${name}</option>
% else:
<option value="${code | h}">${name}</option>
% endif
% endfor
</select>
</div>
<div class="oo-ui-widget oo-ui-widget-enabled oo-ui-inputWidget oo-ui-dropdownInputWidget oo-ui-dropdownInputWidget-php">
<select name="project" required="" class="oo-ui-inputWidget-input oo-ui-indicator-down" title="Project">
<% selected_project = query.project if query.project else g.cookies["CopyviosDefaultProject"].value if "CopyviosDefaultProject" in g.cookies else cache.bot.wiki.get_site().project %>\
% for code, name in cache.projects:
% if code == selected_project:
<option value="${code | h}" selected="selected">${name}</option>
% else:
<option value="${code | h}">${name}</option>
% endif
% endfor
</select>
</div>
</div>
<div class="oo-ui-layout oo-ui-horizontalLayout">
<label for="cv-title" class="page oo-ui-widget oo-ui-widget-enabled oo-ui-labelElement-label oo-ui-labelElement oo-ui-labelWidget">Page</label>
<div class="page-title oo-ui-widget oo-ui-widget-enabled oo-ui-inputWidget oo-ui-textInputWidget oo-ui-textInputWidget-type-text oo-ui-textInputWidget-php">
<input id="cv-title" type="text" class="oo-ui-inputWidget-input" name="title" placeholder="Title" title="Page title"
% if query.title:
value="${query.page.title if query.page else query.title | h}"
% endif
>
</div>
<label class="oo-ui-widget oo-ui-widget-enabled oo-ui-labelElement-label oo-ui-labelElement oo-ui-labelWidget">or</label>
<div class="page-oldid oo-ui-widget oo-ui-widget-enabled oo-ui-inputWidget oo-ui-textInputWidget oo-ui-textInputWidget-type-text oo-ui-textInputWidget-php">
<input id="cv-oldid" type="text" class="oo-ui-inputWidget-input" name="oldid" placeholder="Revision ID" title="Revision ID"
% if query.oldid:
value="${query.oldid | h}"
% endif
>
</div>
</div>
<div class="oo-ui-layout oo-ui-horizontalLayout">
<span class="oo-ui-widget oo-ui-widget-enabled">
<span class="oo-ui-widget oo-ui-widget-enabled oo-ui-inputWidget oo-ui-radioInputWidget">
<input id="action-search" class="oo-ui-inputWidget-input" type="radio" name="action" value="search" ${'checked="checked"' if (query.action == "search" or not query.action) else ""}><span></span>
</span>
<label for="action-search" class="action oo-ui-widget oo-ui-widget-enabled oo-ui-labelElement-label oo-ui-labelElement oo-ui-labelWidget">Copyvio search</label>
</span>

<input type="hidden" name="use_engine" value="0">
<span class="oo-ui-widget oo-ui-widget-enabled">
<span class="cv-search-oo-ui oo-ui-widget oo-ui-widget-enabled oo-ui-inputWidget oo-ui-checkboxInputWidget">
<input id="cv-cb-engine" class="cv-search oo-ui-inputWidget-input" type="checkbox" name="use_engine" value="1" ${'checked="checked"' if query.use_engine not in ("0", "false") else ""}>
<span class="oo-ui-checkboxInputWidget-checkIcon oo-ui-widget oo-ui-widget-enabled oo-ui-iconElement-icon oo-ui-icon-check oo-ui-iconElement oo-ui-labelElement-invisible oo-ui-iconWidget oo-ui-image-invert"></span>
</span>
<label for="cv-cb-engine" class="oo-ui-widget oo-ui-widget-enabled oo-ui-labelElement-label oo-ui-labelElement oo-ui-labelWidget">Use search engine</label>
</span>

<input type="hidden" name="use_links" value="0">
<span class="oo-ui-widget oo-ui-widget-enabled">
<span class="cv-search-oo-ui oo-ui-widget oo-ui-widget-enabled oo-ui-inputWidget oo-ui-checkboxInputWidget">
<input id="cv-cb-links" class="cv-search oo-ui-inputWidget-input" type="checkbox" name="use_links" value="1" ${'checked="checked"' if query.use_links not in ("0", "false") else ""}>
<span class="oo-ui-checkboxInputWidget-checkIcon oo-ui-widget oo-ui-widget-enabled oo-ui-iconElement-icon oo-ui-icon-check oo-ui-iconElement oo-ui-labelElement-invisible oo-ui-iconWidget oo-ui-image-invert"></span>
</span>
<label for="cv-cb-links" class="oo-ui-widget oo-ui-widget-enabled oo-ui-labelElement-label oo-ui-labelElement oo-ui-labelWidget">Use links in page</label>
</span>

<input type="hidden" name="turnitin" value="0">
<span class="oo-ui-widget oo-ui-widget-enabled">
<span class="cv-search-oo-ui oo-ui-widget oo-ui-widget-enabled oo-ui-inputWidget oo-ui-checkboxInputWidget">
<input id="cv-cb-turnitin" class="cv-search oo-ui-inputWidget-input" type="checkbox" name="turnitin" value="1" ${'checked="checked"' if query.turnitin in ("1", "true") else ""}>
<span class="oo-ui-checkboxInputWidget-checkIcon oo-ui-widget oo-ui-widget-enabled oo-ui-iconElement-icon oo-ui-icon-check oo-ui-iconElement oo-ui-labelElement-invisible oo-ui-iconWidget oo-ui-image-invert"></span>
</span>
<label for="cv-cb-turnitin" class="oo-ui-widget oo-ui-widget-enabled oo-ui-labelElement-label oo-ui-labelElement oo-ui-labelWidget">Use Turnitin</label>
</span>
</div>
<div class="oo-ui-layout oo-ui-horizontalLayout">
<span class="oo-ui-widget oo-ui-widget-enabled">
<span class="oo-ui-widget oo-ui-widget-enabled oo-ui-inputWidget oo-ui-radioInputWidget">
<input id="action-compare" class="oo-ui-inputWidget-input" type="radio" name="action" value="compare" ${'checked="checked"' if query.action == "compare" else ""}><span></span>
</span>
<label for="action-compare" class="action oo-ui-widget oo-ui-widget-enabled oo-ui-labelElement-label oo-ui-labelElement oo-ui-labelWidget">Copyvio compare</label>
</span>
<div class="compare-url cv-compare-oo-ui oo-ui-widget oo-ui-widget-enabled oo-ui-inputWidget oo-ui-textInputWidget oo-ui-textInputWidget-type-text oo-ui-textInputWidget-php">
<input type="text" class="cv-compare oo-ui-inputWidget-input" name="url" placeholder="URL" title="URL to compare"
% if query.url:
value="${query.url | h}"
% endif
>
</div>
</div>
% if query.nocache or (result and result.cached):
<div class="oo-ui-layout oo-ui-horizontalLayout">
<span class="cv-search-oo-ui oo-ui-widget oo-ui-widget-enabled oo-ui-inputWidget oo-ui-checkboxInputWidget">
<input id="cb-nocache" class="oo-ui-inputWidget-input" type="checkbox" name="nocache" value="1" ${'checked="checked"' if query.nocache else ""}>
<span class="oo-ui-checkboxInputWidget-checkIcon oo-ui-widget oo-ui-widget-enabled oo-ui-iconElement-icon oo-ui-icon-check oo-ui-iconElement oo-ui-labelElement-invisible oo-ui-iconWidget oo-ui-image-invert"></span>
</span>
<label for="cb-nocache">Bypass cache</label>
</div>
% endif
<div class="oo-ui-layout oo-ui-horizontalLayout">
<span class="oo-ui-widget oo-ui-widget-enabled oo-ui-buttonElement oo-ui-buttonElement-framed oo-ui-labelElement oo-ui-flaggedElement-primary oo-ui-flaggedElement-progressive oo-ui-buttonWidget">
<button type="submit" class="oo-ui-inputWidget-input oo-ui-buttonElement-button">
<span class="oo-ui-labelElement-label">Submit</span>
</button>
</span>
</div>
</form>

<%include file="/includes/form.mako" args="query=query"/>
% if result:
<div id="generation-time">
Results
% if result.cached:
<a id="cv-cached" href="#">cached<span>To save time (and money), this tool will retain the results of checks for up to 72 hours. This includes the URLs of the checked sources, but neither their content nor the content of the article. Future checks on the same page (assuming it remains unchanged) will not involve additional search queries, but a fresh comparison against the source URL will be made. If the page is modified, a new check will be run.</span></a> from <abbr title="${result.cache_time}">${result.cache_age} ago</abbr>. Originally
% endif
generated in <span class="mono">${round(result.time, 3)}</span>
% if query.action == "search":
seconds using <span class="mono">${result.queries}</span> quer${"y" if result.queries == 1 else "ies"}.
% else:
seconds.
% endif
<a href="${request.script_root | h}/?lang=${query.lang | h}&amp;project=${query.project | h}&amp;oldid=${query.oldid or query.page.lastrevid | h}&amp;action=${query.action | h}&amp;${"use_engine={0}&use_links={1}".format(int(query.use_engine not in ("0", "false")), int(query.use_links not in ("0", "false"))) if query.action == "search" else "" | h}${"url=" if query.action == "compare" else ""}${query.url if query.action == "compare" else "" | u}">Permalink.</a>
</div>

<div id="cv-result" class="${'red' if result.confidence >= T_SUSPECT else 'yellow' if result.confidence >= T_POSSIBLE else 'green'}-box">
<table id="cv-result-head-table">
<colgroup>
<col>
<col>
<col>
</colgroup>
<tr>
<td>
<a href="${query.page.url}">${query.page.title | h}</a>
% if query.oldid:
@<a href="https://${query.site.domain | h}/w/index.php?oldid=${query.oldid | h}">${query.oldid | h}</a>
% endif
% if query.redirected_from:
<br />
<span id="redirected-from">Redirected from <a href="https://${query.site.domain | h}/w/index.php?title=${query.redirected_from.title | u}&amp;redirect=no">${query.redirected_from.title | h}</a>. <a href="${request.url | httpsfix, h}&amp;noredirect=1">Check original.</a></span>
% endif
</td>
<td>
<div>
% if result.confidence >= T_SUSPECT:
Violation suspected
% elif result.confidence >= T_POSSIBLE:
Violation possible
% elif result.sources:
Violation unlikely
% else:
No violation
% endif
</div>
<div>${round(result.confidence * 100, 1)}%</div>
<div>similarity</div>
</td>
<td>
% if result.url:
<a href="${result.url | h}">${result.url | urlstrip, h}</a>
% else:
<span id="result-head-no-sources">No matches found.</span>
% endif
</td>
</tr>
</table>
</div>

<% attrib = get_attribution_info(query.site, query.page) %>
% if attrib:
<div id="attribution-warning" class="yellow-box">
This article contains an attribution template: <code>{{<a href="${attrib[1]}">${attrib[0] | h}</a>}}</code>. Please verify that any potential copyvios are not from properly attributed sources.
</div>
% endif

% if query.turnitin_result:
<div id="turnitin-container" class="${'red' if query.turnitin_result.reports else 'green'}-box">
<div id="turnitin-title">Turnitin Results</div>
% if query.turnitin_result.reports:
<table id="turnitin-table"><tbody>
% for report in turnitin_result.reports:
<tr><td class="turnitin-table-cell"><a href="https://eranbot.toolforge.org/ithenticate.py?rid=${report.reportid}">Report ${report.reportid}</a> for text added at <a href="https://${query.lang}.wikipedia.org/w/index.php?title=${query.title}&amp;diff=${report.diffid}"> ${report.time_posted.strftime("%H:%M, %d %B %Y (UTC)")}</a>:
<ul>
% for source in report.sources:
<li>${source['percent']}% of revision text (${source['words']} words) found at <a href="${source['url'] | h}">${source['url'] | h}</a></li>
% endfor
</ul></td></tr>
% endfor
</tbody></table>
% else:
<div id="turnitin-summary">No matching sources found.</div>
% endif
</div>
% endif

% if query.action == "search":
<% skips = False %>
<div id="sources-container">
<div id="sources-title">Checked Sources</div>
% if result.sources:
<table id="cv-result-sources">
<colgroup>
<col>
<col>
<col>
</colgroup>
<tr>
<th>URL</th>
<th>Similarity</th>
<th>Compare</th>
</tr>
% for i, source in enumerate(result.sources):
<tr ${'class="source-default-hidden"' if i >= 10 else 'id="source-row-selected"' if i == 0 else ""}>
<td><a ${'id="source-selected"' if i == 0 else ""} class="source-url" href="${source.url | h}">${source.url | h}</a></td>
<td>
% if source.excluded:
<span class="source-excluded">Excluded</span>
% elif source.skipped:
<% skips = True %>
<span class="source-skipped">Skipped</span>
% else:
<span class="source-similarity ${"source-suspect" if source.confidence >= T_SUSPECT else "source-possible" if source.confidence >= T_POSSIBLE else "source-novio"}">${round(source.confidence * 100, 1)}%</span>
% endif
</td>
<td>
<a href="${request.script_root | h}/?lang=${query.lang | h}&amp;project=${query.project | h}&amp;oldid=${query.oldid or query.page.lastrevid | h}&amp;action=compare&amp;url=${source.url | u}">Compare</a>
</td>
</tr>
% endfor
</table>
% else:
<div class="cv-source-footer">
No sources checked.
</div>
% endif
% if len(result.sources) > 10:
<div id="cv-additional" class="cv-source-footer">
${len(result.sources) - 10} URL${"s" if len(result.sources) > 11 else ""} with lower similarity hidden. <a id="show-additional-sources" href="#">Show them.</a>
</div>
% endif
% if skips or result.possible_miss:
<div class="cv-source-footer">
The search ended early because a match was found with high similarity. <a href="${request.url | httpsfix, h}&amp;noskip=1">Do a complete check.</a>
</div>
% endif
</div>
% endif
<table id="cv-chain-table">
<tr>
<td class="cv-chain-cell">Article: <div class="cv-chain-detail"><p>${highlight_delta(result.article_chain, result.best.chains[1] if result.best else None)}</p></div></td>
<td class="cv-chain-cell">Source: <div class="cv-chain-detail"><p>${highlight_delta(result.best.chains[0], result.best.chains[1]) if result.best else ""}</p></div></td>
</tr>
</table>
<%include file="/includes/result.mako" args="query=query, result=result, turnitin_result=turnitin_result"/>
% endif
<%include file="/support/footer.mako"/>
<%include file="/includes/footer.mako"/>

+ 39
- 71
templates/settings.mako View File

@@ -3,7 +3,8 @@
from flask import g, request
from copyvios.misc import cache
%>\
<%include file="/support/header.mako" args="title='Settings | Earwig\'s Copyvio Detector', splash=True"/>
<%include file="/includes/header.mako" args="title='Settings - Earwig\'s Copyvio Detector', splash=True"/>
<%namespace name="ooui" file="/includes/ooui.mako"/>
% if status:
<div id="info-box" class="green-box">
<p>${status}</p>
@@ -12,43 +13,16 @@
<h2>Settings</h2>
<p>This page contains some configurable options for the copyvio detector. Settings are saved as cookies.</p>
<form action="${request.script_root}/settings" method="post">
<h3>Default site</h2>
<div class="oo-ui-layout oo-ui-labelElement oo-ui-fieldLayout oo-ui-fieldLayout-align-top">
<div class="oo-ui-fieldLayout-body">
<div class="oo-ui-fieldLayout-field">
<div class="oo-ui-widget oo-ui-widget-enabled">
<div class="oo-ui-layout oo-ui-horizontalLayout">
<div class="oo-ui-widget oo-ui-widget-enabled oo-ui-inputWidget oo-ui-dropdownInputWidget oo-ui-dropdownInputWidget-php">
<select name="lang" required="" class="oo-ui-inputWidget-input oo-ui-indicator-down">
<% selected_lang = g.cookies["CopyviosDefaultLang"].value if "CopyviosDefaultLang" in g.cookies else default_lang %>\
% for code, name in cache.langs:
% if code == selected_lang:
<option value="${code | h}" selected="selected">${name}</option>
% else:
<option value="${code | h}">${name}</option>
% endif
% endfor
</select>
</div>
<div class="oo-ui-widget oo-ui-widget-enabled oo-ui-inputWidget oo-ui-dropdownInputWidget oo-ui-dropdownInputWidget-php">
<select name="project" required="" class="oo-ui-inputWidget-input oo-ui-indicator-down">
<% selected_project = g.cookies["CopyviosDefaultProject"].value if "CopyviosDefaultProject" in g.cookies else default_project %>\
% for code, name in cache.projects:
% if code == selected_project:
<option value="${code | h}" selected="selected">${name}</option>
% else:
<option value="${code | h}">${name}</option>
% endif
% endfor
</select>
</div>
</div>
</div>
</div>
</div>
</div>
<h3>Default site</h3>
<%ooui:field_layout>
<%ooui:widget>
<%ooui:horizontal_layout>
<%include file="/includes/site.mako"/>
</%ooui:horizontal_layout>
</%ooui:widget>
</%ooui:field_layout>

<h3>Background</h2>
<h3>Background</h3>
<%
background_options = [
("list", 'Randomly select from <a href="https://commons.wikimedia.org/wiki/User:The_Earwig/POTD">a subset</a> of previous <a href="https://commons.wikimedia.org/">Wikimedia Commons</a> <a href="https://commons.wikimedia.org/wiki/Commons:Picture_of_the_day">Pictures of the Day</a> that work well as widescreen backgrounds, refreshed daily (default).'),
@@ -56,41 +30,35 @@
("plain", "Use a plain background."),
]
selected = g.cookies["CopyviosBackground"].value if "CopyviosBackground" in g.cookies else "list"
%>\
<div class="oo-ui-layout oo-ui-labelElement oo-ui-fieldLayout oo-ui-fieldLayout-align-top">
<div class="oo-ui-fieldLayout-body">
<div class="oo-ui-fieldLayout-field">
<div class="oo-ui-widget oo-ui-widget-enabled oo-ui-inputWidget oo-ui-radioSelectInputWidget">
% for value, desc in background_options:
<div class="oo-ui-layout oo-ui-labelElement oo-ui-fieldLayout oo-ui-fieldLayout-align-inline">
<div class="oo-ui-fieldLayout-body">
<span class="oo-ui-fieldLayout-field">
<span class="oo-ui-widget oo-ui-widget-enabled oo-ui-inputWidget oo-ui-radioInputWidget">
<input id="background-${value}" class="oo-ui-inputWidget-input" type="radio" name="background" value="${value}" ${'checked="checked"' if value == selected else ''}><span></span>
</span>
</span>
<span class="oo-ui-fieldLayout-header">
<label for="background-${value}" class="oo-ui-labelElement-label">${desc}</label>
</span>
</div>
</div>
% endfor
</div>
</div>
</div>
%>
<%ooui:field_layout>
<%ooui:radio_select>
% for value, desc in background_options:
<%ooui:field_layout align="inline">
<%ooui:radio>
<input id="background-${value}" class="oo-ui-inputWidget-input" type="radio" name="background" value="${value}" ${'checked="checked"' if value == selected else ''}><span></span>
</%ooui:radio>
<%ooui:field_layout_header>
<label for="background-${value}" class="oo-ui-labelElement-label">${desc}</label>
</%ooui:field_layout_header>
</%ooui:field_layout>
% endfor
</%ooui:radio_select>
</%ooui:field_layout>

<h3>Highlight colors</h3>
<p><em>This is not currently configurable, but it will be soon.</em></p>
<div>
Default:
% for i in range(1, 9):
<span class="highlight-demo cv-hl cv-hl-${i}">${i}</span>
% endfor
<span class="highlight-demo cv-hl">9+</span>
</div>

<input type="hidden" name="action" value="set"/>
<div class="oo-ui-layout oo-ui-fieldLayout oo-ui-fieldLayout-align-left">
<div class="oo-ui-fieldLayout-body">
<span class="oo-ui-fieldLayout-field">
<span class="oo-ui-widget oo-ui-widget-enabled oo-ui-inputWidget oo-ui-buttonElement oo-ui-buttonElement-framed oo-ui-labelElement oo-ui-flaggedElement-primary oo-ui-flaggedElement-progressive oo-ui-labelElement oo-ui-buttonInputWidget">
<button type="submit" class="oo-ui-inputWidget-input oo-ui-buttonElement-button">
<span class="oo-ui-labelElement-label">Save</span>
</button>
</span>
</span>
</div>
</div>
<%ooui:field_layout align="left">
${ooui.submit_button(label="Save")}
</%ooui:field_layout>
</form>
<%include file="/support/footer.mako"/>
<%include file="/includes/footer.mako"/>

Loading…
Cancel
Save