From 9a25f97831d07b27d6ba98819c5aba88bd4e8c41 Mon Sep 17 00:00:00 2001 From: Ben Kurtovic Date: Tue, 1 Jul 2014 23:45:32 -0400 Subject: [PATCH] Connection pooling; cleanup. --- README.md | 1 + app.fcgi | 22 ++++++++++++++++------ copyvios/background.py | 11 ++++++----- copyvios/checker.py | 9 +++++---- copyvios/cookies.py | 2 ++ copyvios/highlighter.py | 2 ++ copyvios/misc.py | 45 ++++++++++++++++++++------------------------- copyvios/settings.py | 2 ++ copyvios/sites.py | 16 ++++++++++------ templates/index.mako | 4 ++-- 10 files changed, 66 insertions(+), 48 deletions(-) diff --git a/README.md b/README.md index a83470d..70fef83 100644 --- a/README.md +++ b/README.md @@ -14,6 +14,7 @@ Dependencies * [mako](http://www.makotemplates.org/) >= 0.7.2 * [mwparserfromhell](https://github.com/earwig/mwparserfromhell) >= 0.3 * [oursql](http://packages.python.org/oursql/) >= 0.9.3.1 +* [SQLAlchemy](http://sqlalchemy.org/) >= 0.9.6 * [uglifycss](https://github.com/fmarcia/UglifyCSS/) * [uglifyjs](https://github.com/mishoo/UglifyJS/) >= 1.3.3 diff --git a/app.fcgi b/app.fcgi index 207ea74..3deb3ff 100755 --- a/app.fcgi +++ b/app.fcgi @@ -7,13 +7,13 @@ from logging.handlers import TimedRotatingFileHandler from time import asctime from traceback import format_exc +from earwigbot.bot import Bot from flask import Flask, g, request from flask.ext.mako import MakoTemplates, render_template, TemplateError from flup.server.fcgi import WSGIServer from copyvios.checker import do_check from copyvios.cookies import parse_cookies -from copyvios.misc import get_bot from copyvios.settings import process_settings from copyvios.sites import get_sites @@ -25,6 +25,8 @@ app.logger.addHandler(TimedRotatingFileHandler( "logs/app.log", when="D", interval=1, backupCount=7)) app.logger.info(u"Flask server started " + asctime()) +bot = Bot(".earwigbot", 100) + def catch_errors(func): @wraps(func) def inner(*args, **kwargs): @@ -37,9 +39,11 @@ def catch_errors(func): return inner @app.before_request -def prepare_cookies(): - cookie_string = request.environ.get("HTTP_COOKIE") - g.cookies = parse_cookies(request.script_root, cookie_string) +def prepare_request(): + g.bot = bot + g.globals_db = g.cache_db = None + g.cookies = parse_cookies(request.script_root, + request.environ.get("HTTP_COOKIE")) g.new_cookies = [] @app.after_request @@ -54,6 +58,13 @@ def write_access_log(response): app.logger.debug(msg, asctime(), request.path, response.status_code) return response +@app.teardown_appcontext +def close_databases(error): + if g.globals_db: + g.globals_db.close() + if g.cache_db: + g.cache_db.close() + @app.route("/") @catch_errors def index(): @@ -64,8 +75,7 @@ def index(): @catch_errors def settings(): status = process_settings() if request.method == "POST" else None - bot = get_bot() - langs, projects = get_sites(bot) + langs, projects = get_sites() default = bot.wiki.get_site() kwargs = {"status": status, "langs": langs, "projects": projects, "default_lang": default.lang, "default_project": default.project} diff --git a/copyvios/background.py b/copyvios/background.py index a55d37d..73ae775 100644 --- a/copyvios/background.py +++ b/copyvios/background.py @@ -9,10 +9,12 @@ from time import time from earwigbot import exceptions from flask import g -from .misc import get_bot, open_sql_connection +from .misc import get_globals_db + +__all__ = ["set_background"] def set_background(selected): - conn = open_sql_connection(get_bot(), "globals") + conn = get_globals_db() if "CopyviosScreenCache" in g.cookies: cache = g.cookies["CopyviosScreenCache"].value try: @@ -94,11 +96,10 @@ def _load_file(site, filename): return filename.replace(" ", "_"), url, descurl, width, height def _get_site(): - bot = get_bot() try: - return bot.wiki.get_site("commonswiki") + return g.bot.wiki.get_site("commonswiki") except exceptions.SiteNotFoundError: - return bot.wiki.add_site(project="wikimedia", lang="commons") + return g.bot.wiki.add_site(project="wikimedia", lang="commons") def _build_url(screen, filename, url, imgwidth, imgheight): width = screen["width"] diff --git a/copyvios/checker.py b/copyvios/checker.py index 984507f..fe27f86 100644 --- a/copyvios/checker.py +++ b/copyvios/checker.py @@ -6,9 +6,11 @@ from urlparse import urlparse from earwigbot import exceptions -from .misc import get_bot, Query, open_sql_connection +from .misc import Query, get_cache_db from .sites import get_site, get_sites +__all__ = ["do_check"] + def do_check(): query = Query() if query.lang: @@ -18,8 +20,7 @@ def do_check(): if query.project: query.project = query.project.lower() - query.bot = get_bot() - query.all_langs, query.all_projects = get_sites(query.bot) + query.all_langs, query.all_projects = get_sites() if query.project and query.lang and (query.title or query.oldid): query.site = get_site(query) if query.site: @@ -43,7 +44,7 @@ def _get_results(query): query.result = page.copyvio_compare(query.url) query.result.cached = False else: - conn = open_sql_connection(query.bot, "cache") + conn = get_cache_db() if not query.nocache: query.result = _get_cached_results(page, conn) if not query.result: diff --git a/copyvios/cookies.py b/copyvios/cookies.py index affe99c..2b5b410 100644 --- a/copyvios/cookies.py +++ b/copyvios/cookies.py @@ -6,6 +6,8 @@ from datetime import datetime, timedelta from flask import g +__all__ = ["parse_cookies", "set_cookie", "delete_cookie"] + class _CookieManager(SimpleCookie): MAGIC = "--cpv2" diff --git a/copyvios/highlighter.py b/copyvios/highlighter.py index e069b73..791c64b 100644 --- a/copyvios/highlighter.py +++ b/copyvios/highlighter.py @@ -4,6 +4,8 @@ from re import sub, UNICODE from markupsafe import escape +__all__ = ["highlight_delta"] + def highlight_delta(context, chain, delta): degree = chain.degree - 1 highlights = [False] * degree diff --git a/copyvios/misc.py b/copyvios/misc.py index 3bc7717..c073d2d 100644 --- a/copyvios/misc.py +++ b/copyvios/misc.py @@ -3,12 +3,13 @@ from os.path import expanduser from urlparse import parse_qs -from earwigbot.bot import Bot -from flask import request +from flask import g, request import oursql +from sqlalchemy.pool import manage -_bot = None -_connections = {} +oursql = manage(oursql) + +__all__ = ["Query", "get_globals_db", "get_cache_db", "httpsfix", "urlstrip"] class Query(object): def __init__(self, method="GET"): @@ -36,28 +37,22 @@ class Query(object): self.query[key] = value -def get_bot(): - global _bot - if not _bot: - _bot = Bot(".earwigbot", 100) # Don't print any logs to the console - return _bot +def _connect_db(name): + args = g.bot.config.wiki["_copyviosSQL"][name] + args["read_default_file"] = expanduser("~/.my.cnf") + args["autoping"] = True + args["autoreconnect"] = True + return oursql.connect(**args) + +def get_globals_db(): + if not g.globals_db: + g.globals_db = _connect_db("globals") + return g.globals_db -def open_sql_connection(bot, dbname): - if dbname in _connections: - return _connections[dbname] - conn_args = bot.config.wiki["_copyviosSQL"][dbname] - if "read_default_file" not in conn_args and "user" not in conn_args and "passwd" not in conn_args: - conn_args["read_default_file"] = expanduser("~/.my.cnf") - elif "read_default_file" in conn_args: - default_file = expanduser(conn_args["read_default_file"]) - conn_args["read_default_file"] = default_file - if "autoping" not in conn_args: - conn_args["autoping"] = True - if "autoreconnect" not in conn_args: - conn_args["autoreconnect"] = True - conn = oursql.connect(**conn_args) - _connections[dbname] = conn - return conn +def get_cache_db(): + if not g.cache_db: + g.cache_db = _connect_db("cache") + return g.cache_db def httpsfix(context, url): if url.startswith("http://"): diff --git a/copyvios/settings.py b/copyvios/settings.py index 0ab3b4c..fcf3948 100644 --- a/copyvios/settings.py +++ b/copyvios/settings.py @@ -6,6 +6,8 @@ from markupsafe import escape from .cookies import set_cookie, delete_cookie from .misc import Query +__all__ = ["process_settings"] + def process_settings(): query = Query(method="POST") if query.action == "set": diff --git a/copyvios/sites.py b/copyvios/sites.py index dcc7925..8c186c4 100644 --- a/copyvios/sites.py +++ b/copyvios/sites.py @@ -4,12 +4,15 @@ from time import time from urlparse import urlparse from earwigbot import exceptions +from flask import g -from .misc import open_sql_connection +from .misc import get_globals_db + +__all__ = ["get_site", "get_sites"] def get_site(query): lang, project, name = query.lang, query.project, query.name - wiki = query.bot.wiki + wiki = g.bot.wiki if project not in [proj[0] for proj in query.all_projects]: return None if project == "wikimedia" and name: # Special sites: @@ -28,9 +31,9 @@ def get_site(query): except (exceptions.APIError, exceptions.LoginError): return None -def get_sites(bot): +def get_sites(): max_staleness = 60 * 60 * 24 * 7 - conn = open_sql_connection(bot, "globals") + conn = get_globals_db() query1 = "SELECT update_time FROM updates WHERE update_service = ?" query2 = "SELECT lang_code, lang_name FROM language" query3 = "SELECT project_code, project_name FROM project" @@ -41,7 +44,7 @@ def get_sites(bot): except IndexError: time_since_update = time() if time_since_update > max_staleness: - _update_sites(bot.wiki.get_site(), cursor) + _update_sites(cursor) cursor.execute(query2) langs = [] for code, name in cursor.fetchall(): @@ -52,7 +55,8 @@ def get_sites(bot): projects = cursor.fetchall() return langs, projects -def _update_sites(site, cursor): +def _update_sites(cursor): + site = g.bot.wiki.get_site() matrix = site.api_query(action="sitematrix")["sitematrix"] del matrix["count"] languages, projects = set(), set() diff --git a/templates/index.mako b/templates/index.mako index 281efca..150d839 100644 --- a/templates/index.mako +++ b/templates/index.mako @@ -30,7 +30,7 @@ http:// .