From b7bb0f61d8a3f8a60468c21a2f242ab9199679b8 Mon Sep 17 00:00:00 2001 From: Ben Kurtovic Date: Tue, 27 Dec 2016 21:21:24 -0500 Subject: [PATCH] Add database to campaigns module; flesh out more. --- .gitignore | 2 +- calefaction/module.py | 1 + calefaction/modules/campaigns/__init__.py | 2 +- calefaction/modules/campaigns/database.py | 87 +++++++++++++++++++++++++++++++ calefaction/modules/campaigns/getters.py | 65 ++++++++++++++++++----- calefaction/modules/members.py | 3 +- config/modules/campaigns.yml.sample | 14 +++-- data/schema_campaigns.sql | 20 +++++++ static/main.css | 8 +-- templates/campaigns/campaign.mako | 18 ++++--- 10 files changed, 191 insertions(+), 29 deletions(-) create mode 100644 calefaction/modules/campaigns/database.py create mode 100644 data/schema_campaigns.sql diff --git a/.gitignore b/.gitignore index f9acdd1..b13e3f4 100644 --- a/.gitignore +++ b/.gitignore @@ -10,4 +10,4 @@ logs/ !config/*.sample !config/modules !config/modules/*.sample -!data/schema.sql +!data/schema*.sql diff --git a/calefaction/module.py b/calefaction/module.py index a033ece..0b36483 100644 --- a/calefaction/module.py +++ b/calefaction/module.py @@ -37,6 +37,7 @@ class Module: provided.app = app provided.blueprint = bp provided.config = self._config.load_module_config(self._name) + provided.logger = self._logger sys.modules[provided.__name__] = provided self._module = importlib.import_module(fullname) diff --git a/calefaction/modules/campaigns/__init__.py b/calefaction/modules/campaigns/__init__.py index c17a3a4..60f1066 100644 --- a/calefaction/modules/campaigns/__init__.py +++ b/calefaction/modules/campaigns/__init__.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -from .getters import get_current, get_count, get_summary, get_unit +from .getters import get_current, get_overview, get_summary, get_unit from .routes import home, navitem from .._provided import config diff --git a/calefaction/modules/campaigns/database.py b/calefaction/modules/campaigns/database.py new file mode 100644 index 0000000..e5a132c --- /dev/null +++ b/calefaction/modules/campaigns/database.py @@ -0,0 +1,87 @@ +# -*- coding: utf-8 -*- + +from datetime import datetime +import sqlite3 + +from flask import g +from werkzeug.local import LocalProxy + +__all__ = ["CampaignDB"] + +class CampaignDB: + """Database manager for internal storage for the Campaigns module.""" + path = None + + def __init__(self): + if self.path is None: + raise RuntimeError("CampaignDB.path not set") + self._conn = sqlite3.connect(self.path) + + @classmethod + def _get(cls): + """Return the current database, or allocate a new one if necessary.""" + if not hasattr(g, "_campaign_db"): + g._campaign_db = cls() + return g._campaign_db + + @classmethod + def pre_hook(cls): + """Hook to be called before a request context. + + Sets up the g.campaign_db proxy. + """ + g.campaign_db = LocalProxy(cls._get) + + @classmethod + def post_hook(cls, exc): + """Hook to be called when tearing down an application context. + + Closes the database if necessary. + """ + if hasattr(g, "_campaign_db"): + g._campaign_db.close() + + def close(self): + """Close the database connection.""" + return self._conn.close() + + def check_operation(self, campaign, operation): + """Return the last updated timestamp for the given operation. + + Return None if the given operation was never updated. + """ + query = """SELECT lu_date FROM last_updated + WHERE lu_campaign = ? AND lu_operation = ?""" + res = self._conn.execute(query, (campaign, operation)).fetchall() + if not res: + return None + return datetime.strptime(res[0][0], "%Y-%m-%d %H:%M:%S") + + def add_operation(self, campaign, operation): + """Insert a new operation into the database as just updated.""" + with self._conn as conn: + conn.execute("""INSERT INTO last_updated + (lu_campaign, lu_operation) VALUES (?, ?)""", ( + campaign, operation)) + + def touch_operation(self, campaign, operation): + """Mark the given operation as just updated.""" + with self._conn as conn: + conn.execute("""UPDATE last_updated SET lu_date = CURRENT_TIMESTAMP + WHERE lu_campaign = ? AND lu_operation = ?""", ( + campaign, operation)) + + def set_overview(self, campaign, operation, primary, secondary=None): + """Set overview information for this operation.""" + with self._conn as conn: + conn.execute("""INSERT OR REPLACE INTO overview + (ov_campaign, ov_operation, ov_primary, ov_secondary) + VALUES (?, ?, ?, ?)""", ( + campaign, operation, primary, secondary)) + + def get_overview(self, campaign, operation): + """Return a 2-tuple of overview information for this operation.""" + query = """SELECT ov_primary, ov_secondary FROM overview + WHERE ov_campaign = ? AND ov_operation = ?""" + res = self._conn.execute(query, (campaign, operation)).fetchall() + return res[0] if res else (0, None) diff --git a/calefaction/modules/campaigns/getters.py b/calefaction/modules/campaigns/getters.py index 87eb7b1..ecf770e 100644 --- a/calefaction/modules/campaigns/getters.py +++ b/calefaction/modules/campaigns/getters.py @@ -1,10 +1,36 @@ # -*- coding: utf-8 -*- +from datetime import datetime, timedelta +from pathlib import Path +from threading import Lock + from flask import g -from .._provided import config +from .database import CampaignDB +from .._provided import app, config, logger +from ...database import Database as MainDB + +__all__ = ["get_current", "get_overview", "get_summary", "get_unit"] + +CampaignDB.path = str(Path(MainDB.path).parent / "db_campaigns.sqlite3") -__all__ = ["get_current", "get_count", "get_summary", "get_unit"] +app.before_request(CampaignDB.pre_hook) +app.teardown_appcontext(CampaignDB.post_hook) + +_lock = Lock() + +def _update_operation(cname, opname, new): + """Update a campaign/operation.""" + ... + + operation = config["campaigns"][cname]["operations"][opname] + optype = operation["type"] + qualifiers = operation["qualifiers"] + show_isk = operation.get("isk", True) + + primary = 42 + secondary = 63 + g.campaign_db.set_overview(cname, opname, primary, secondary) def get_current(): """Return the name of the currently selected campaign, or None.""" @@ -15,24 +41,39 @@ def get_current(): return config["enabled"][0] return setting -def get_count(cname, opname): - """Return the primary operation count for the given campaign/operation.""" - key = cname + "." + opname - operation = config["campaigns"][cname]["operations"][opname] - optype = operation["type"] - qualifiers = operation["qualifiers"] +def get_overview(cname, opname): + """Return overview information for the given campaign/operation. - ... - import random - return [random.randint(0, 500), random.randint(10000, 500000), random.randint(10000000, 50000000000)][random.randint(0, 2)] + The overview is a 2-tuple of (primary_count, secondary_count). The latter + may be None, in which case it should not be displayed. + + Updates the database if necessary, so this can take some time. + """ + with _lock: + last_updated = g.campaign_db.check_operation(cname, opname) + if last_updated is None: + logger.debug("Adding campaign=%s operation=%s", cname, opname) + _update_operation(cname, opname, new=True) + g.campaign_db.add_operation(cname, opname) + elif datetime.utcnow() - last_updated > timedelta(seconds=60 * 60): + logger.debug("Updating campaign=%s operation=%s", cname, opname) + _update_operation(cname, opname, new=False) + g.campaign_db.touch_operation(cname, opname) + else: + logger.debug("Using cache for campaign=%s operation=%s", + cname, opname) + return g.campaign_db.get_overview(cname, opname) def get_summary(name, opname, limit=5): """Return a sample fraction of results for the given campaign/operation.""" ... return [] -def get_unit(operation, num): +def get_unit(operation, num, primary=True): """Return the correct form of the unit tracked by the given operation.""" + if not primary: + return "ISK" + types = { "killboard": "ship|ships", "collection": "item|items" diff --git a/calefaction/modules/members.py b/calefaction/modules/members.py index 9c3de35..36d3936 100644 --- a/calefaction/modules/members.py +++ b/calefaction/modules/members.py @@ -5,7 +5,7 @@ from collections import namedtuple from flask import g from flask_mako import render_template -from ._provided import blueprint +from ._provided import blueprint, logger from ..exceptions import EVEAPIForbiddenError SCOPES = {"esi-corporations.read_corporation_membership.v1"} @@ -29,6 +29,7 @@ def get_members(): return [] corp_id = g.config.get("corp.id") + logger.debug("Fetching member list for corp id=%d", corp_id) try: ceo_id = g.eve.esi(token).v2.corporations(corp_id).get()["ceo_id"] cids_r = g.eve.esi(token).v2.corporations(corp_id).members.get() diff --git a/config/modules/campaigns.yml.sample b/config/modules/campaigns.yml.sample index aca74f7..112bb60 100644 --- a/config/modules/campaigns.yml.sample +++ b/config/modules/campaigns.yml.sample @@ -27,18 +27,26 @@ campaigns: operations: # Will track the number of Foo frigates killed by the corp: frigates: + # Full operation name: title: "Operation: Kill Foo Frigates" + # Data source (here, retrieve data from ZKillboard): type: killboard + # Show total ISK killed (defaults to true): + isk: true + # Python function to filter kills: qualifiers: |- - return ((victim_corp == "Foo Corporation") and - (victim_ship_class == "Frigate")) + return (kill["victim"]["corporationName"] == "Foo Corporation" and + kill["victim"]["shipTypeID"] in ...) # Will track possession of Tritanium by all corp members: titan: title: Let's Build a Titan + # Here, retrieve data from EVE's Assets API: type: collection + isk: false + # Report as "10 units" / "1 unit" of Tritanium unit: unit|units qualifiers: |- - return item_type == "Tritanium" + return item_type == "Tritanium" # ... bar: title: Save the Bar operations: [] diff --git a/data/schema_campaigns.sql b/data/schema_campaigns.sql new file mode 100644 index 0000000..5df5c3d --- /dev/null +++ b/data/schema_campaigns.sql @@ -0,0 +1,20 @@ +-- Schema for Calefaction's Campaign module's internal database + +DROP TABLE IF EXISTS last_updated; + +CREATE TABLE last_updated ( + lu_campaign TEXT, + lu_operation TEXT, + lu_date TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + UNIQUE (lu_campaign, lu_operation) +); + +DROP TABLE IF EXISTS overview; + +CREATE TABLE overview ( + ov_campaign TEXT, + ov_operation TEXT, + ov_primary INTEGER DEFAULT 0, + ov_secondary INTEGER DEFAULT NULL, + UNIQUE (ov_campaign, ov_operation) +); diff --git a/static/main.css b/static/main.css index 6ba2ba2..4ddb284 100644 --- a/static/main.css +++ b/static/main.css @@ -376,20 +376,20 @@ h2 .disabled::after { justify-content: space-between; } -.operation .overview .num { +.operation .overview .primary { line-height: 60px; height: 60px; } -.operation .overview .num.big { +.operation .overview .primary.big { font-size: 300%; } -.operation .overview .num.medium { +.operation .overview .primary.medium { font-size: 200%; } -.operation .overview .num.small { +.operation .overview .primary.small { font-size: 150%; } diff --git a/templates/campaigns/campaign.mako b/templates/campaigns/campaign.mako index ee0aa70..3cc07b1 100644 --- a/templates/campaigns/campaign.mako +++ b/templates/campaigns/campaign.mako @@ -14,23 +14,27 @@ % for opname in section: <% operation = campaign["operations"][opname] - num = mod.get_count(name, opname) + primary, secondary = mod.get_overview(name, opname) summary = mod.get_summary(name, opname, limit=5) - klass = "big" if num < 1000 else "medium" if num < 1000000 else "small" + klass = "big" if primary < 1000 else "medium" if primary < 1000000 else "small" %>

${operation["title"] | h}

-
${"{:,}".format(num)}
-
${mod.get_unit(operation, num)}
+
${"{:,}".format(primary)}
+
${mod.get_unit(operation, primary)}
+ % if secondary is not None: +
${"{:,}".format(secondary)}
+
${mod.get_unit(operation, secondary, primary=False)}
+ % endif
% if summary: % endif