Add database to campaigns module; flesh out more.

.gitignore

@@ -10,4 +10,4 @@ logs/
 !config/*.sample
 !config/modules
 !config/modules/*.sample
-!data/schema.sql
+!data/schema*.sql

calefaction/module.py

@@ -37,6 +37,7 @@ class Module:
         provided.app = app
         provided.blueprint = bp
         provided.config = self._config.load_module_config(self._name)
+        provided.logger = self._logger
         sys.modules[provided.__name__] = provided
 
         self._module = importlib.import_module(fullname)

calefaction/modules/campaigns/__init__.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8  -*-
 
-from .getters import get_current, get_count, get_summary, get_unit
+from .getters import get_current, get_overview, get_summary, get_unit
 from .routes import home, navitem
 from .._provided import config

calefaction/modules/campaigns/database.py

@@ -0,0 +1,87 @@
+# -*- coding: utf-8  -*-
+
+from datetime import datetime
+import sqlite3
+
+from flask import g
+from werkzeug.local import LocalProxy
+
+__all__ = ["CampaignDB"]
+
+class CampaignDB:
+    """Database manager for internal storage for the Campaigns module."""
+    path = None
+
+    def __init__(self):
+        if self.path is None:
+            raise RuntimeError("CampaignDB.path not set")
+        self._conn = sqlite3.connect(self.path)
+
+    @classmethod
+    def _get(cls):
+        """Return the current database, or allocate a new one if necessary."""
+        if not hasattr(g, "_campaign_db"):
+            g._campaign_db = cls()
+        return g._campaign_db
+
+    @classmethod
+    def pre_hook(cls):
+        """Hook to be called before a request context.
+
+        Sets up the g.campaign_db proxy.
+        """
+        g.campaign_db = LocalProxy(cls._get)
+
+    @classmethod
+    def post_hook(cls, exc):
+        """Hook to be called when tearing down an application context.
+
+        Closes the database if necessary.
+        """
+        if hasattr(g, "_campaign_db"):
+            g._campaign_db.close()
+
+    def close(self):
+        """Close the database connection."""
+        return self._conn.close()
+
+    def check_operation(self, campaign, operation):
+        """Return the last updated timestamp for the given operation.
+
+        Return None if the given operation was never updated.
+        """
+        query = """SELECT lu_date FROM last_updated
+            WHERE lu_campaign = ? AND lu_operation = ?"""
+        res = self._conn.execute(query, (campaign, operation)).fetchall()
+        if not res:
+            return None
+        return datetime.strptime(res[0][0], "%Y-%m-%d %H:%M:%S")
+
+    def add_operation(self, campaign, operation):
+        """Insert a new operation into the database as just updated."""
+        with self._conn as conn:
+            conn.execute("""INSERT INTO last_updated
+                (lu_campaign, lu_operation) VALUES (?, ?)""", (
+                    campaign, operation))
+
+    def touch_operation(self, campaign, operation):
+        """Mark the given operation as just updated."""
+        with self._conn as conn:
+            conn.execute("""UPDATE last_updated SET lu_date = CURRENT_TIMESTAMP
+                WHERE lu_campaign = ? AND lu_operation = ?""", (
+                    campaign, operation))
+
+    def set_overview(self, campaign, operation, primary, secondary=None):
+        """Set overview information for this operation."""
+        with self._conn as conn:
+            conn.execute("""INSERT OR REPLACE INTO overview
+                (ov_campaign, ov_operation, ov_primary, ov_secondary)
+                VALUES (?, ?, ?, ?)""", (
+                    campaign, operation, primary, secondary))
+
+    def get_overview(self, campaign, operation):
+        """Return a 2-tuple of overview information for this operation."""
+        query = """SELECT ov_primary, ov_secondary FROM overview
+            WHERE ov_campaign = ? AND ov_operation = ?"""
+        res = self._conn.execute(query, (campaign, operation)).fetchall()
+        return res[0] if res else (0, None)

calefaction/modules/campaigns/getters.py

@@ -1,10 +1,36 @@
 # -*- coding: utf-8  -*-
 
+from datetime import datetime, timedelta
+from pathlib import Path
+from threading import Lock
+
 from flask import g
 
-from .._provided import config
+from .database import CampaignDB
+from .._provided import app, config, logger
+from ...database import Database as MainDB
+
+__all__ = ["get_current", "get_overview", "get_summary", "get_unit"]
+
+CampaignDB.path = str(Path(MainDB.path).parent / "db_campaigns.sqlite3")
 
-__all__ = ["get_current", "get_count", "get_summary", "get_unit"]
+app.before_request(CampaignDB.pre_hook)
+app.teardown_appcontext(CampaignDB.post_hook)
+
+_lock = Lock()
+
+def _update_operation(cname, opname, new):
+    """Update a campaign/operation."""
+    ...
+
+    operation = config["campaigns"][cname]["operations"][opname]
+    optype = operation["type"]
+    qualifiers = operation["qualifiers"]
+    show_isk = operation.get("isk", True)
+
+    primary = 42
+    secondary = 63
+    g.campaign_db.set_overview(cname, opname, primary, secondary)
 
 def get_current():
     """Return the name of the currently selected campaign, or None."""
@@ -15,24 +41,39 @@ def get_current():
         return config["enabled"][0]
     return setting
 
-def get_count(cname, opname):
-    """Return the primary operation count for the given campaign/operation."""
-    key = cname + "." + opname
-    operation = config["campaigns"][cname]["operations"][opname]
-    optype = operation["type"]
-    qualifiers = operation["qualifiers"]
+def get_overview(cname, opname):
+    """Return overview information for the given campaign/operation.
 
-    ...
-    import random
-    return [random.randint(0, 500), random.randint(10000, 500000), random.randint(10000000, 50000000000)][random.randint(0, 2)]
+    The overview is a 2-tuple of (primary_count, secondary_count). The latter
+    may be None, in which case it should not be displayed.
+
+    Updates the database if necessary, so this can take some time.
+    """
+    with _lock:
+        last_updated = g.campaign_db.check_operation(cname, opname)
+        if last_updated is None:
+            logger.debug("Adding campaign=%s operation=%s", cname, opname)
+            _update_operation(cname, opname, new=True)
+            g.campaign_db.add_operation(cname, opname)
+        elif datetime.utcnow() - last_updated > timedelta(seconds=60 * 60):
+            logger.debug("Updating campaign=%s operation=%s", cname, opname)
+            _update_operation(cname, opname, new=False)
+            g.campaign_db.touch_operation(cname, opname)
+        else:
+            logger.debug("Using cache for campaign=%s operation=%s",
+                         cname, opname)
+        return g.campaign_db.get_overview(cname, opname)
 
 def get_summary(name, opname, limit=5):
     """Return a sample fraction of results for the given campaign/operation."""
     ...
     return []
 
-def get_unit(operation, num):
+def get_unit(operation, num, primary=True):
     """Return the correct form of the unit tracked by the given operation."""
+    if not primary:
+        return "ISK"
+
     types = {
         "killboard": "ship|ships",
         "collection": "item|items"

calefaction/modules/members.py

@@ -5,7 +5,7 @@ from collections import namedtuple
 from flask import g
 from flask_mako import render_template
 
-from ._provided import blueprint
+from ._provided import blueprint, logger
 from ..exceptions import EVEAPIForbiddenError
 
 SCOPES = {"esi-corporations.read_corporation_membership.v1"}
@@ -29,6 +29,7 @@ def get_members():
         return []
 
     corp_id = g.config.get("corp.id")
+    logger.debug("Fetching member list for corp id=%d", corp_id)
     try:
         ceo_id = g.eve.esi(token).v2.corporations(corp_id).get()["ceo_id"]
         cids_r = g.eve.esi(token).v2.corporations(corp_id).members.get()

config/modules/campaigns.yml.sample

@@ -27,18 +27,26 @@ campaigns:
     operations:
       # Will track the number of Foo frigates killed by the corp:
       frigates:
+        # Full operation name:
         title: "Operation: Kill Foo Frigates"
+        # Data source (here, retrieve data from ZKillboard):
         type:  killboard
+        # Show total ISK killed (defaults to true):
+        isk:   true
+        # Python function to filter kills:
         qualifiers: |-
-          return ((victim_corp == "Foo Corporation") and
-                  (victim_ship_class == "Frigate"))
+          return (kill["victim"]["corporationName"] == "Foo Corporation" and
+                  kill["victim"]["shipTypeID"] in ...)
       # Will track possession of Tritanium by all corp members:
       titan:
         title: Let's Build a Titan
+        # Here, retrieve data from EVE's Assets API:
         type:  collection
+        isk:   false
+        # Report as "10 units" / "1 unit" of Tritanium
         unit:  unit|units
         qualifiers: |-
-          return item_type == "Tritanium"
+          return item_type == "Tritanium"  # ...
   bar:
     title: Save the Bar
     operations: []

data/schema_campaigns.sql

@@ -0,0 +1,20 @@
+-- Schema for Calefaction's Campaign module's internal database
+
+DROP TABLE IF EXISTS last_updated;
+
+CREATE TABLE last_updated (
+    lu_campaign TEXT,
+    lu_operation TEXT,
+    lu_date TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
+    UNIQUE (lu_campaign, lu_operation)
+);
+
+DROP TABLE IF EXISTS overview;
+
+CREATE TABLE overview (
+    ov_campaign TEXT,
+    ov_operation TEXT,
+    ov_primary INTEGER DEFAULT 0,
+    ov_secondary INTEGER DEFAULT NULL,
+    UNIQUE (ov_campaign, ov_operation)
+);

static/main.css

@@ -376,20 +376,20 @@ h2 .disabled::after {
     justify-content: space-between;
 }
 
-.operation .overview .num {
+.operation .overview .primary {
     line-height: 60px;
     height: 60px;
 }
 
-.operation .overview .num.big {
+.operation .overview .primary.big {
     font-size: 300%;
 }
 
-.operation .overview .num.medium {
+.operation .overview .primary.medium {
     font-size: 200%;
 }
 
-.operation .overview .num.small {
+.operation .overview .primary.small {
     font-size: 150%;
 }
 

templates/campaigns/campaign.mako

@@ -14,23 +14,27 @@
             % for opname in section:
                 <%
                     operation = campaign["operations"][opname]
-                    num = mod.get_count(name, opname)
+                    primary, secondary = mod.get_overview(name, opname)
                     summary = mod.get_summary(name, opname, limit=5)
-                    klass = "big" if num < 1000 else "medium" if num < 1000000 else "small"
+                    klass = "big" if primary < 1000 else "medium" if primary < 1000000 else "small"
                 %>
                 <div class="operation">
                     <h3>
                         <a href="${url_for('campaigns.operation', cname=name, opname=opname)}">${operation["title"] | h}</a>
                     </h3>
                     <div class="overview">
-                        <div class="num ${klass}">${"{:,}".format(num)}</div>
-                        <div class="unit">${mod.get_unit(operation, num)}</div>
+                        <div class="primary ${klass}">${"{:,}".format(primary)}</div>
+                        <div class="unit">${mod.get_unit(operation, primary)}</div>
+                        % if secondary is not None:
+                            <div class="secondary">${"{:,}".format(secondary)}</div>
+                            <div class="unit">${mod.get_unit(operation, secondary, primary=False)}</div>
+                        % endif
                     </div>
                     % if summary:
                         <ul class="summary">
-                        % for item in summary:
-                            <li>${item}</li>
-                        % endfor
+                            % for item in summary:
+                                <li>${item}</li>
+                            % endfor
                         </ul>
                     % endif
                 </div>