From 6eed95c7617b9c1f423684b71f59e78593b35c71 Mon Sep 17 00:00:00 2001 From: Ben Kurtovic Date: Fri, 19 Sep 2014 17:31:55 -0500 Subject: [PATCH] Use a global cache instead of the request-specific one. --- app.fcgi | 24 +++++++++++++----------- copyvios/api.py | 8 +++----- copyvios/background.py | 22 ++++++++++++---------- copyvios/misc.py | 23 ++++++++++++++++++----- copyvios/sites.py | 13 +++++++------ templates/index.mako | 9 +++++---- templates/settings.mako | 5 +++-- 7 files changed, 61 insertions(+), 43 deletions(-) diff --git a/app.fcgi b/app.fcgi index eb9823c..ff6758f 100755 --- a/app.fcgi +++ b/app.fcgi @@ -17,6 +17,7 @@ from flup.server.fcgi import WSGIServer from copyvios.api import format_api_error, handle_api_request from copyvios.checker import do_check from copyvios.cookies import parse_cookies +from copyvios.misc import cache from copyvios.settings import process_settings from copyvios.sites import update_sites @@ -28,8 +29,6 @@ app.logger.addHandler(TimedRotatingFileHandler( "logs/app.log", when="midnight", backupCount=7)) app.logger.info(u"Flask server started " + asctime()) -getLogger("earwigbot.wiki.cvworker").setLevel(INFO) - def catch_errors(func): @wraps(func) def inner(*args, **kwargs): @@ -43,16 +42,19 @@ def catch_errors(func): @app.before_first_request def setup_app(): - g.bot = Bot(".earwigbot", 100) - g.langs, g.projects = set(), set() - g.last_sites_update = 0 - g.background_data = {} - g.last_background_updates = {} + cache.bot = Bot(".earwigbot", 100) + cache.langs, cache.projects = set(), set() + cache.last_sites_update = 0 + cache.background_data = {} + cache.last_background_updates = {} + + getLogger("earwigbot.wiki.cvworker").setLevel(INFO) + # getLogger().handlers .... globalize() @app.before_request def prepare_request(): - g.db = None + g._db = None g.cookies = parse_cookies( request.script_root, request.environ.get("HTTP_COOKIE")) g.new_cookies = [] @@ -72,8 +74,8 @@ def write_access_log(response): @app.teardown_appcontext def close_databases(error): - if g.db: - g.db.close() + if g._db: + g._db.close() @app.route("/") @catch_errors @@ -87,7 +89,7 @@ def index(): def settings(): status = process_settings() if request.method == "POST" else None update_sites() - default = g.bot.wiki.get_site() + default = cache.bot.wiki.get_site() kwargs = {"status": status, "default_lang": default.lang, "default_project": default.project} return render_template("settings.mako", **kwargs) diff --git a/copyvios/api.py b/copyvios/api.py index b69c49a..d11c19f 100644 --- a/copyvios/api.py +++ b/copyvios/api.py @@ -2,10 +2,8 @@ from collections import OrderedDict -from flask import g - from .checker import do_check, T_POSSIBLE, T_SUSPECT -from .misc import Query +from .misc import Query, cache from .sites import update_sites __all__ = ["format_api_error", "handle_api_request"] @@ -95,8 +93,8 @@ def _hook_check(query): def _hook_sites(query): update_sites() - return OrderedDict(( - ("status", "ok"), ("langs", g.langs), ("projects", g.projects))) + return OrderedDict((("status", "ok"), + ("langs", cache.langs), ("projects", cache.projects))) _HOOKS = { "compare": _hook_check, diff --git a/copyvios/background.py b/copyvios/background.py index 8c533d8..bae8708 100644 --- a/copyvios/background.py +++ b/copyvios/background.py @@ -8,13 +8,15 @@ import re from earwigbot import exceptions from flask import g +from .misc import cache + __all__ = ["set_background"] def _get_commons_site(): try: - return g.bot.wiki.get_site("commonswiki") + return cache.bot.wiki.get_site("commonswiki") except exceptions.SiteNotFoundError: - return g.bot.wiki.add_site(project="wikimedia", lang="commons") + return cache.bot.wiki.add_site(project="wikimedia", lang="commons") def _load_file(site, filename): res = site.api_query(action="query", prop="imageinfo", iiprop="url|size", @@ -57,23 +59,23 @@ _BACKGROUNDS = { } def _get_background(selected): - if not g.last_background_updates: + if not cache.last_background_updates: for key in _BACKGROUNDS: - g.last_background_updates[key] = datetime.min + cache.last_background_updates[key] = datetime.min - plus_one = g.last_background_updates[selected] + timedelta(days=1) + plus_one = cache.last_background_updates[selected] + timedelta(days=1) max_age = datetime(plus_one.year, plus_one.month, plus_one.day) if datetime.utcnow() > max_age: update_func = _BACKGROUNDS.get(selected, _get_fresh_list) - g.background_data[selected] = update_func() - g.last_background_updates[selected] = datetime.utcnow() - return g.background_data[selected] + cache.background_data[selected] = update_func() + cache.last_background_updates[selected] = datetime.utcnow() + return cache.background_data[selected] def set_background(selected): if "CopyviosScreenCache" in g.cookies: - cache = g.cookies["CopyviosScreenCache"].value + screen_cache = g.cookies["CopyviosScreenCache"].value try: - screen = loads(cache) + screen = loads(screen_cache) int(screen["width"]) int(screen["height"]) except (ValueError, KeyError): diff --git a/copyvios/misc.py b/copyvios/misc.py index 45ddfaf..6cfb00a 100644 --- a/copyvios/misc.py +++ b/copyvios/misc.py @@ -8,7 +8,7 @@ from sqlalchemy.pool import manage oursql = manage(oursql) -__all__ = ["Query", "get_db", "httpsfix", "urlstrip"] +__all__ = ["Query", "cache", "get_db", "httpsfix", "urlstrip"] class Query(object): def __init__(self, method="GET"): @@ -27,14 +27,27 @@ class Query(object): self.query[key] = value +class _AppCache(object): + def __init__(self): + self._data = {} + + def __getattr__(self, key): + return self._data[key] + + def __setattr__(self, key, value): + self._data[key] = value + + +cache = _AppCache() + def get_db(): - if not g.db: - args = g.bot.config.wiki["_copyviosSQL"] + if not g._db: + args = cache.bot.config.wiki["_copyviosSQL"] args["read_default_file"] = expanduser("~/.my.cnf") args["autoping"] = True args["autoreconnect"] = True - g.db = oursql.connect(**args) - return g.db + g._db = oursql.connect(**args) + return g._db def httpsfix(context, url): if url.startswith("http://"): diff --git a/copyvios/sites.py b/copyvios/sites.py index 85609b1..bafd3ba 100644 --- a/copyvios/sites.py +++ b/copyvios/sites.py @@ -4,13 +4,14 @@ from time import time from urlparse import urlparse from earwigbot import exceptions -from flask import g + +from .misc import cache __all__ = ["get_site", "update_sites"] def get_site(query): lang, project, name = query.lang, query.project, query.name - wiki = g.bot.wiki + wiki = cache.bot.wiki if project not in [proj[0] for proj in query.all_projects]: return None if project == "wikimedia" and name: # Special sites: @@ -30,12 +31,12 @@ def get_site(query): return None def update_sites(): - if time() - g.last_sites_update > 60 * 60 * 24 * 7: - g.langs, g.projects = _load_sites() - g.last_sites_update = time() + if time() - cache.last_sites_update > 60 * 60 * 24 * 7: + cache.langs, cache.projects = _load_sites() + cache.last_sites_update = time() def _load_sites(): - site = g.bot.wiki.get_site() + site = cache.bot.wiki.get_site() matrix = site.api_query(action="sitematrix")["sitematrix"] del matrix["count"] langs, projects = set(), set() diff --git a/templates/index.mako b/templates/index.mako index b6b45ff..63c5b20 100644 --- a/templates/index.mako +++ b/templates/index.mako @@ -1,6 +1,7 @@ <%! from flask import g, request from copyvios.checker import T_POSSIBLE, T_SUSPECT + from copyvios.misc import cache %>\ <%include file="/support/header.mako" args="title='Earwig\'s Copyvio Detector'"/> <%namespace module="copyvios.highlighter" import="highlight_delta"/>\ @@ -50,8 +51,8 @@ https:// . <% selected_lang = g.cookies["CopyviosDefaultLang"].value if "CopyviosDefaultLang" in g.cookies else default_lang %>\ - % for code, name in g.langs: + % for code, name in cache.langs: % if code == selected_lang: % else: @@ -28,7 +29,7 @@ .