Dynamically load sites from the API and cache this information.

- Also removing some unused imports and cleaning up code.

pages/copyvios.mako

@@ -1,5 +1,4 @@
 <%!
-    from collections import defaultdict
     from datetime import datetime
     from hashlib import sha256
     from itertools import count
@@ -12,8 +11,7 @@
     from earwigbot import bot, exceptions
     import oursql
 
-    def get_results(lang, project, title, url, query):
-        bot = bot.Bot(".earwigbot")
+    def get_results(bot, lang, project, title, url, query):
         try:
             site = bot.wiki.get_site(lang=lang, project=project)
         except exceptions.SiteNotFoundError:
@@ -26,7 +24,7 @@
         # if url:
         #     result = get_url_specific_results(page, url)
         # else:
-        #     conn = open_sql_connection(bot)
+        #     conn = open_sql_connection(bot, "copyvioCache")
         #     if not query.get("nocache"):
         #         result = get_cached_results(page, conn)
         #     if query.get("nocache") or not result:
@@ -44,8 +42,8 @@
         result.tdiff = time() - t_start
         return result
 
-    def open_sql_connection(bot):
-        conn_args = bot.config.wiki["_toolserverSQLCache"]
+    def open_sql_connection(bot, dbname):
+        conn_args = bot.config.wiki["_toolserverSQL"][dbname]
         if "read_default_file" not in conn_args and "user" not in conn_args and "passwd" not in conn_args:
             conn_args["read_default_file"] = expanduser("~/.my.cnf")
         if "autoping" not in conn_args:
@@ -107,6 +105,75 @@
             cursor.execute(query3, (pageid, hash, result.url, result.queries,
                                     result.tdiff))
 
+    def get_sites(bot):
+        max_staleness = 60 * 60 * 24 * 7
+        site = bot.wiki.get_site()
+        conn = open_sql_connection(site, "globals")
+        query1 = "SELECT update_time FROM updates WHERE update_service = ?"
+        query2 = "SELECT lang_code, lang_name FROM languages"
+        query3 = "SELECT project_name FROM projects"
+        tl_normal = '<option value="{0}">{1}</option>'
+        tl_select = '<option value="{0}" selected="selected">{1}</option>'
+
+        with conn.cursor() as cursor:
+            cursor.execute(query1, ("sites",))
+            time_since_update = int(time() - cursor.fetchall()[0][0])
+            if time_since_update > max_staleness:
+                update_sites(bot, cursor)
+
+            cursor.execute(query2)
+            langs = []
+            for lang, lang_name in cursor.fetchall():
+                template = tl_select if lang == site.lang else tl_normal
+                fullname = "{0} ({1})".format(lang, lang_name)
+                langs.append(template.format(lang, fullname))
+
+            cursor.execute(query3)
+            projects = []
+            for (project,) in cursor.fetchall():
+                template = tl_select if project == site.project else tl_normal
+                projects.append(template.format(project, project.capitalize()))
+
+        langs = "\n".join(langs)
+        projects = "\n".join(projects)
+        result = '<select name="lang">\n{0}\n</select>\n'.format(langs)
+        result += '<select name="project">\n{0}\n</select>'.format(projects)
+        return result
+
+    def update_sites(site, cursor):
+        matrix = site.api_query(action="sitematrix")["sitematrix"]
+        del matrix["count"]
+        languages, projects = set(), set()
+        for site in matrix.itervalues():
+            if isinstance(site, list):  # Special sites
+                continue
+            code = site["code"].encode("utf8")
+            name = site["name"].encode("utf8")
+            languages.add((code, name))
+            for web in site["site"]:
+                project = "wikipedia" if web["code"] == "wiki" else web["code"]
+                projects.add(project)
+        save_site_updates(cursor, languages, projects)
+
+    def save_site_updates(cursor, languages, projects):
+        query1 = "SELECT lang_code, lang_name FROM languages"
+        query2 = "DELETE FROM languages WHERE lang_code = ? AND lang_name = ?"
+        query3 = "INSERT INTO languages VALUES (?, ?)"
+        query4 = "SELECT project_name FROM projects"
+        query5 = "DELETE FROM projects WHERE project_name = ?"
+        query6 = "INSERT INTO projects VALUES (?)"
+        query7 = "UPDATE updates SET update_time = ? WHERE update_service = ?"
+        synchronize_sites_with_db(cursor, languages, query1, query2, query3)
+        synchronize_sites_with_db(cursor, projects, query4, query5, query6)
+        cursor.execute(query7, (time(), "sites"))
+
+    def synchronize_sites_with_db(cursor, updates, q_list, q_rmv, q_update):
+        removals = []
+        for site in cursor.execute(q_list):
+            updates.remove(site) if site in updates else removals.append(site)
+        cursor.executemany(q_rmv, removals)
+        cursor.executemany(q_update, updates)
+
     def highlight_delta(chain, delta):
         processed = []
         prev_prev = prev = chain.START
@@ -187,6 +254,7 @@
         return url
 %>\
 <%
+    bot = bot.Bot(".earwigbot")
     query = parse_qs(environ["QUERY_STRING"])
     try:
         lang = query["lang"][0]
@@ -196,7 +264,7 @@
     except (KeyError, IndexError):
         page = None
     else:
-        page, result = get_results(lang, project, title, url, query)
+        page, result = get_results(bot, lang, project, title, url, query)
 %>\
 <%include file="/support/header.mako" args="environ=environ, title='Copyvio Detector', add_css=('copyvios.css',), add_js=('copyvios.js',)"/>
             <h1>Copyvio Detector</h1>
@@ -206,12 +274,7 @@
                     <tr>
                         <td>Site:</td>
                         <td>
-                            <select name="lang">
-                                <option value="en" selected="selected">en (English)</option>
-                            </select>
-                            <select name="project">
-                                <option value="wikipedia" selected="selected">Wikipedia</option>
-                            </select>
+                            ${get_sites(bot)}
                         </td>
                     </tr>
                     <tr>

pages/index.mako

@@ -1,6 +1,5 @@
 <%!
     from os import path
-    import time
 
     tools = [
         ("Home", "home", "index", True, None),

pages/support/header.mako

@@ -1,7 +1,6 @@
 <%page args="environ, title, slug=None, add_css=(), add_js=()"/>\
 <%namespace name="index" file="/index.mako" import="get_tools"/>\
 <%!
-    from itertools import count
     from os import path
 %>\
 <%
@@ -31,8 +30,8 @@
     <body onload="potd_set_background()">
         <div id="header">
             <p id="heading"><a class="dark" href="${pretty}">earwig</a><span class="light">@</span><a class="mid" href="https://wiki.toolserver.org/">toolserver</a><span class="light">:</span><a class="dark" href="${this}">${slug}</a></p>
-            <p id="links"><span class="light">&gt;</span>    
-                % for (name, tool, link, complete, desc), num in zip(tools, count(1)):
+            <p id="links"><span class="light">&gt;</span>
+                % for num, (name, tool, link, complete, desc) in enumerate(tools, 1):
                     <abbr title="${name}${' (incomplete)' if not complete else ''}"><a class="${'dark' if complete else 'mid'}" href="${pretty}/${link}">${tool}</a></abbr>
                     % if num < len(tools):
                         <span class="light">&#124;</span>