Update/fix index.mako.

app.fcgi

@@ -11,6 +11,7 @@ from flask import Flask, g, request
 from flask.ext.mako import MakoTemplates, render_template, TemplateError
 from flup.server.fcgi import WSGIServer
 
+from copyvios.checker import do_check
 from copyvios.cookies import parse_cookies
 from copyvios.misc import get_bot
 from copyvios.settings import process_settings
@@ -56,7 +57,8 @@ def write_access_log(response):
 @app.route("/")
 @catch_errors
 def index():
-    return render_template("index.mako")
+    query = do_check()
+    return render_template("index.mako", query=query, result=query.result)
 
 @app.route("/settings", methods=["GET", "POST"])
 @catch_errors

copyvios/__init__.py

@@ -1,23 +1 @@
 # -*- coding: utf-8  -*-
-
-from .checker import get_results
-from .highlighter import highlight_delta
-from .misc import get_bot, Query
-from .sites import get_site, get_sites
-
-def main(context, environ):
-    query = Query(environ)
-    if query.lang:
-        query.lang = query.orig_lang = query.lang.lower()
-        if "::" in query.lang:
-            query.lang, query.name = query.lang.split("::", 1)
-    if query.project:
-        query.project = query.project.lower()
-
-    query.bot = get_bot()
-    query.all_langs, query.all_projects = get_sites(query.bot)
-    if query.lang and query.project and query.title:
-        query.site = get_site(query)
-        if query.site:
-            get_results(query)
-    return query

copyvios/checker.py

@@ -6,9 +6,27 @@ from urlparse import urlparse
 
 from earwigbot import exceptions
 
-from .misc import open_sql_connection
+from .misc import get_bot, Query, open_sql_connection
+from .sites import get_site, get_sites
 
-def get_results(query):
+def do_check():
+    query = Query()
+    if query.lang:
+        query.lang = query.orig_lang = query.lang.lower()
+        if "::" in query.lang:
+            query.lang, query.name = query.lang.split("::", 1)
+    if query.project:
+        query.project = query.project.lower()
+
+    query.bot = get_bot()
+    query.all_langs, query.all_projects = get_sites(query.bot)
+    if query.project and query.lang and query.title:  # TODO: and (query.title or query.oldid): ...
+        query.site = get_site(query)
+        if query.site:
+            _get_results(query)
+    return query
+
+def _get_results(query):
     page = query.page = query.site.get_page(query.title)
     try:
         page.get()  # Make sure that the page exists before we check it!

copyvios/misc.py

@@ -27,10 +27,7 @@ class Query(object):
                 pass
 
     def __getattr__(self, key):
-        try:
-            return self.query[key]
-        except KeyError:
-            return None
+        return self.query.get(key)
 
     def __setattr__(self, key, value):
         if key == "query":

templates/index.mako

@@ -1,42 +1,40 @@
+<%!
+    from flask import g, request
+%>\
 <%include file="/support/header.mako" args="title='Earwig\'s Copyvio Detector'"/>
-<%namespace module="copyvios" import="main, highlight_delta"/>\
+<%namespace module="copyvios.highlighter" import="highlight_delta"/>\
 <%namespace module="copyvios.misc" import="urlstrip"/>\
-<%
-    query = main(environ)
-    # Unpack query data:
-    lang, orig_lang, title, oldid, url, nocache = query.lang, query.orig_lang, query.title, query.oldid, query.url, query.nocache
-    bot, site, page, result = query.bot, query.site, query.page, query.result
-%>\
-% if query.project and lang and (title or oldid):
-    % if not site:
+% if query.project and query.lang and (query.title or query.oldid):
+    % if not query.site:
         <div id="info-box" class="red-box">
-            <p>The given site (project=<b><span class="mono">${query.project | h}</span></b>, language=<b><span class="mono">${lang | h}</span></b>) doesn't seem to exist. It may also be closed or private. <a href="//${lang | h}.${query.project | h}.org/">Confirm its URL.</a></p>
+            <p>The given site (project=<b><span class="mono">${query.project | h}</span></b>, language=<b><span class="mono">${query.lang | h}</span></b>) doesn't seem to exist. It may also be closed or private. <a href="//${query.lang | h}.${query.project | h}.org/">Confirm its URL.</a></p>
         </div>
-    % elif title and not result:
+    % elif query.title and not result:
         <div id="info-box" class="red-box">
-            <p>The given page doesn't seem to exist: <a href="${page.url}">${page.title | h}</a>.</p>
+            <p>The given page doesn't seem to exist: <a href="${query.page.url}">${query.page.title | h}</a>.</p>
         </div>
-    % elif oldid and not result:
+    % elif query.oldid and not result:
         <div id="info-box" class="red-box">
-            <p>The given revision ID doesn't seem to exist: <a href="//${site.domain | h}/w/index.php?oldid=${oldid | h}">${oldid | h}</a>.</p>
+            <p>The given revision ID doesn't seem to exist: <a href="//${query.site.domain | h}/w/index.php?oldid=${query.oldid | h}">${query.oldid | h}</a>.</p>
         </div>
-    % elif url and result == "bad URI":
+    % elif query.url and result == "bad URI":
         <% result = None %>
         <div id="info-box" class="red-box">
-            <p>Unsupported URI scheme: <a href="${url | h}">${url | h}</a>.</p>
+            <p>Unsupported URI scheme: <a href="${query.url | h}">${query.url | h}</a>.</p>
         </div>
     % endif
 %endif
-<p>This tool attempts to detect <a href="//en.wikipedia.org/wiki/WP:COPYVIO">copyright violations</a> in articles. Simply give the title of the page or ID of the revision you want to check and hit Submit. The tool will search for similar content elsewhere on the web and display a report if a match is found. If you also provide a URL, it will not query any search engines and instead display a report comparing the article to that particular webpage, like the <a href="//toolserver.org/~dcoetzee/duplicationdetector/">Duplication Detector</a>. Check out the <a href="//en.wikipedia.org/wiki/User:EarwigBot/Copyvios/FAQ">FAQ</a> for more information and technical details.</p>
+<p>This tool attempts to detect <a href="//en.wikipedia.org/wiki/WP:COPYVIO">copyright violations</a> in articles. Simply give the title of the page or ID of the revision you want to check and hit Submit. The tool will search for similar content elsewhere on the web using <a href="//info.yahoo.com/legal/us/yahoo/boss/pricing/">Yahoo! BOSS</a> and then display a report if a match is found. If you give a URL, it will skip the search engine step and directly display a report comparing the article to that particular webpage, like the <a href="//toolserver.org/~dcoetzee/duplicationdetector/">Duplication Detector</a>.</p>
+<p>Specific websites can be excluded from the check (for example, if their content is in the public domain) by being added to the <a href="//en.wikipedia.org/wiki/User:EarwigBot/Copyvios/Exclusions">excluded URL list</a>.</p>
 <p><i>Note:</i> The tool is still in beta. You are completely welcome to use it and provide <a href="//en.wikipedia.org/wiki/User_talk:The_Earwig">feedback</a>, but be aware that it may produce strange or broken results.</p>
-<form action="${environ['REQUEST_URI']}" method="get">
+<form action="${request.base_url}" method="get">
     <table id="cv-form">
         <tr>
             <td>Site:</td>
             <td colspan="3">
                 <span class="mono">http://</span>
                 <select name="lang">
-                    <% selected_lang = orig_lang if orig_lang else cookies["CopyviosDefaultLang"].value if "CopyviosDefaultLang" in cookies else bot.wiki.get_site().lang %>\
+                    <% selected_lang = query.orig_lang if query.orig_lang else g.cookies["CopyviosDefaultLang"].value if "CopyviosDefaultLang" in g.cookies else query.bot.wiki.get_site().lang %>\
                     % for code, name in query.all_langs:
                         % if code == selected_lang:
                             <option value="${code | h}" selected="selected">${name}</option>
@@ -47,7 +45,7 @@
                 </select>
                 <span class="mono">.</span>
                 <select name="project">
-                    <% selected_project = query.project if query.project else cookies["CopyviosDefaultProject"].value if "CopyviosDefaultProject" in cookies else bot.wiki.get_site().project %>\
+                    <% selected_project = query.project if query.project else g.cookies["CopyviosDefaultProject"].value if "CopyviosDefaultProject" in g.cookies else query.bot.wiki.get_site().project %>\
                     % for code, name in query.all_projects:
                         % if code == selected_project:
                             <option value="${code | h}" selected="selected">${name}</option>
@@ -62,18 +60,18 @@
         <tr>
             <td id="cv-col1">Page&nbsp;title:</td>
             <td id="cv-col2">
-                % if page:
-                    <input class="cv-text" type="text" name="title" value="${page.title | h}" />
-                % elif title:
-                    <input class="cv-text" type="text" name="title" value="${title | h}" />
+                % if query.page:
+                    <input class="cv-text" type="text" name="title" value="${query.page.title | h}" />
+                % elif query.title:
+                    <input class="cv-text" type="text" name="title" value="${query.title | h}" />
                 % else:
                     <input class="cv-text" type="text" name="title" />
                 % endif
             </td>
             <td id="cv-col3">or&nbsp;revision&nbsp;ID:</td>
             <td id="cv-col4">
-                % if oldid:
-                    <input class="cv-text" type="text" name="oldid" value="${oldid | h}" />
+                % if query.oldid:
+                    <input class="cv-text" type="text" name="oldid" value="${query.oldid | h}" />
                 % else:
                     <input class="cv-text" type="text" name="oldid" />
                 % endif
@@ -82,18 +80,18 @@
         <tr>
             <td>URL&nbsp;(optional):</td>
             <td colspan="3">
-                % if url:
-                    <input class="cv-text" type="text" name="url" value="${url | h}" />
+                % if query.url:
+                    <input class="cv-text" type="text" name="url" value="${query.url | h}" />
                 % else:
                     <input class="cv-text" type="text" name="url" />
                 % endif
             </td>
         </tr>
-        % if nocache or (result and result.cached):
+        % if query.nocache or (result and result.cached):
             <tr>
                 <td>Bypass&nbsp;cache:</td>
                 <td colspan="3">
-                    % if nocache:
+                    % if query.nocache:
                         <input type="checkbox" name="nocache" value="1" checked="checked" />
                     % else:
                         <input type="checkbox" name="nocache" value="1" />
@@ -109,16 +107,16 @@
     </table>
 </form>
 % if result:
-    <% show_details = "CopyviosShowDetails" in cookies and cookies["CopyviosShowDetails"].value == "True" %>
+    <% show_details = "CopyviosShowDetails" in g.cookies and g.cookies["CopyviosShowDetails"].value == "True" %>
     <div class="divider"></div>
     <div id="cv-result" class="${'red' if result.violation else 'green'}-box">
         % if result.violation:
-            <h2 id="cv-result-header"><a href="${page.url}">${page.title | h}</a> is a suspected violation of <a href="${result.url | h}">${result.url | urlstrip, h}</a>.</h2>
+            <h2 id="cv-result-header"><a href="${query.page.url}">${query.page.title | h}</a> is a suspected violation of <a href="${result.url | h}">${result.url | urlstrip, h}</a>.</h2>
         % else:
-            <h2 id="cv-result-header">No violations detected in <a href="${page.url}">${page.title | h}</a>.</h2>
+            <h2 id="cv-result-header">No violations detected in <a href="${query.page.url}">${query.page.title | h}</a>.</h2>
         % endif
         <ul id="cv-result-list">
-            % if not result.violation and not url:
+            % if not result.violation and not query.url:
                 % if result.url:
                     <li>Best match: <a href="${result.url | h}">${result.url | urlstrip, h}</a>.</li>
                 % else:
@@ -129,7 +127,7 @@
             % if result.cached:
                 <li>Results are <a id="cv-cached" href="#">cached
                     <span>To save time (and money), this tool will retain the results of checks for up to 72 hours. This includes the URL of the "violated" source, but neither its content nor the content of the article. Future checks on the same page (assuming it remains unchanged) will not involve additional search queries, but a fresh comparison against the source URL will be made. If the page is modified, a new check will be run.</span>
-                </a> from ${result.cache_time} (${result.cache_age} ago). <a href="${environ['REQUEST_URI'] | h}&amp;nocache=1">Bypass the cache.</a></li>
+                </a> from ${result.cache_time} (${result.cache_age} ago). <a href="${request.url | h}&amp;nocache=1">Bypass the cache.</a></li>
             % else:
                 <li>Results generated in <span class="mono">${round(result.time, 3)}</span> seconds using <span class="mono">${result.queries}</span> queries.</li>
             % endif