Add zKillboard API support; implement most of campaign updating.

calefaction/eve/__init__.py

@@ -8,6 +8,7 @@ from .clock import Clock
 from .esi import EVESwaggerInterface
 from .image import ImageServer
 from .sso import SSOManager
+from .zkill import ZKillboard
 from .. import __release__, baseLogger
 
 __all__ = ["EVE"]
@@ -25,6 +26,7 @@ class EVE:
         self._esi = EVESwaggerInterface(session, logger.getChild("esi"))
         self._image = ImageServer()
         self._sso = SSOManager(session, logger.getChild("sso"))
+        self._zkill = ZKillboard(session, logger.getChild("zkill"))
 
     @staticmethod
     def _get_user_agent(contact):
@@ -54,3 +56,8 @@ class EVE:
     def sso(self):
         """The Single Sign-On API module."""
         return self._sso
+
+    @property
+    def zkill(self):
+        """The zKillboard API module."""
+        return self._zkill

calefaction/eve/sso.py

@@ -13,7 +13,6 @@ class SSOManager:
 
     def __init__(self, session, logger):
         self._session = session
-
         self._logger = logger
         self._debug = logger.debug
 

calefaction/eve/zkill.py

@@ -0,0 +1,61 @@
+# -*- coding: utf-8  -*-
+
+import time
+
+import requests
+
+from ..exceptions import ZKillboardError
+
+__all__ = ["ZKillboard"]
+
+class ZKillboard:
+    """EVE API module for zKillboard."""
+    _MAX_RATE = 0.5
+
+    def __init__(self, session, logger):
+        self._session = session
+        self._logger = logger
+        self._debug = logger.debug
+
+        self._base_url = "https://zkillboard.com/api"
+        self._last_query = 0
+
+    def query(self, *args):
+        """Make an API query using the given arguments."""
+        query = "/" + "".join(str(arg) + "/" for arg in args)
+        url = self._base_url + query
+        delta = self._MAX_RATE - (time.time() - self._last_query)
+        if delta > 0:
+            self._debug("[GET] [wait %.2fs] %s", delta, query)
+            time.sleep(delta)
+        else:
+            self._debug("[GET] %s", query)
+
+        try:
+            resp = self._session.get(url, timeout=10)
+            resp.raise_for_status()
+            result = resp.json() if resp.content else None
+        except (requests.RequestException, ValueError):
+            self._logger.exception("zKillboard API query failed")
+            raise ZKillboardError()
+
+        self._last_query = time.time()
+        return result
+
+    def iter_killmails(self, *args):
+        """Return an iterator over killmails using the given API arguments.
+
+        Automagically follows pagination as far as possible. (Be careful.)
+        """
+        page = 1
+        while True:
+            if page > 1:
+                result = self.query(*args, "page", page)
+            else:
+                result = self.query(*args)
+
+            if result:
+                yield from result
+                page += 1
+            else:
+                break

calefaction/exceptions.py

@@ -7,6 +7,7 @@ This module contains exceptions for Calefaction.
     +-- AccessDeniedError
     +-- EVEAPIError
         +-- EVEAPIForbiddenError
+        +-- ZKillboardError
 """
 
 class CalefactionError(RuntimeError):
@@ -24,3 +25,7 @@ class EVEAPIError(CalefactionError):
 class EVEAPIForbiddenError(EVEAPIError):
     """We tried to make an API request that we don't have permission for."""
     pass
+
+class ZKillboardError(EVEAPIError):
+    """Represents an error while using zKillboard's API."""
+    pass

calefaction/modules/campaigns/database.py

@@ -46,30 +46,29 @@ class CampaignDB:
         return self._conn.close()
 
     def check_operation(self, campaign, operation):
-        """Return the last updated timestamp for the given operation.
+        """Return the last updated timestamp and key for the given operation.
 
-        Return None if the given operation was never updated.
+        Return (None, None) if the given operation was never updated.
         """
-        query = """SELECT lu_date FROM last_updated
+        query = """SELECT lu_date, lu_key FROM last_updated
             WHERE lu_campaign = ? AND lu_operation = ?"""
         res = self._conn.execute(query, (campaign, operation)).fetchall()
         if not res:
-            return None
-        return datetime.strptime(res[0][0], "%Y-%m-%d %H:%M:%S")
+            return None, None
+        return datetime.strptime(res[0][0], "%Y-%m-%d %H:%M:%S"), res[0][1]
 
-    def add_operation(self, campaign, operation):
-        """Insert a new operation into the database as just updated."""
+    def touch_operation(self, campaign, operation, key=None):
+        """Mark the given operation as just updated, or add it."""
         with self._conn as conn:
-            conn.execute("""INSERT INTO last_updated
-                (lu_campaign, lu_operation) VALUES (?, ?)""", (
-                    campaign, operation))
-
-    def touch_operation(self, campaign, operation):
-        """Mark the given operation as just updated."""
-        with self._conn as conn:
-            conn.execute("""UPDATE last_updated SET lu_date = CURRENT_TIMESTAMP
+            cur = conn.execute("BEGIN TRANSACTION")
+            cur.execute("""UPDATE last_updated
+                SET lu_date = CURRENT_TIMESTAMP, lu_key = ?
                 WHERE lu_campaign = ? AND lu_operation = ?""", (
-                    campaign, operation))
+                    key, campaign, operation))
+            if cur.rowcount == 0:
+                cur.execute("""INSERT INTO last_updated
+                    (lu_campaign, lu_operation, lu_key) VALUES (?, ?, ?)""", (
+                        campaign, operation, key))
 
     def set_overview(self, campaign, operation, primary, secondary=None):
         """Set overview information for this operation."""
@@ -84,4 +83,55 @@ class CampaignDB:
         query = """SELECT ov_primary, ov_secondary FROM overview
             WHERE ov_campaign = ? AND ov_operation = ?"""
         res = self._conn.execute(query, (campaign, operation)).fetchall()
-        return res[0] if res else (0, None)
+        return tuple(res[0]) if res else (0, None)
+
+    def has_kill(self, kill_id):
+        """Return whether the database has a killmail with the given ID."""
+        query = "SELECT 1 FROM kill WHERE kill_id = ?"
+        res = self._conn.execute(query, (kill_id,)).fetchall()
+        return bool(res)
+
+    def add_kill(self, kill):
+        """Insert a killmail into the database."""
+        try:
+            datetime.strptime(kill["killTime"], "%Y-%m-%d %H:%M:%S")
+        except ValueError:
+            raise RuntimeError("Invalid kill_date=%s for kill_id=%d" % (
+                kill["killTime"], kill["killID"]))
+
+        query = """INSERT OR REPLACE INTO kill (
+                kill_id, kill_date, kill_system, kill_victim_shipid,
+                kill_victim_charid, kill_victim_corpid, kill_victim_allianceid,
+                kill_victim_factionid, kill_value)
+            VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)"""
+        args = (
+            kill["killID"], kill["killTime"], kill["solarSystemID"],
+            kill["victim"]["shipTypeID"], kill["victim"]["characterID"],
+            kill["victim"]["corporationID"], kill["victim"]["allianceID"],
+            kill["victim"]["factionID"], kill["zkb"]["totalValue"])
+        with self._conn as conn:
+            conn.execute(query, args)
+
+    def get_kill_associations(self, campaign, kill_id):
+        """Return a list of operations associated with a campaign and kill."""
+        query = """SELECT ok_operation FROM oper_kill
+            WHERE ok_campaign = ? AND ok_killid = ?"""
+        res = self._conn.execute(query, (campaign, kill_id)).fetchall()
+        return [row[0] for row in res]
+
+    def associate_kill(self, campaign, kill_id, operations):
+        """Associate a killmail with a set of campaign/operations."""
+        query = """INSERT OR IGNORE INTO oper_kill
+            (ok_campaign, ok_operation, ok_killid) VALUES (?, ?, ?)"""
+        arglist = [(campaign, op, kill_id) for op in operations]
+        with self._conn as conn:
+            conn.executemany(query, arglist)
+
+    def count_kills(self, campaign, operation):
+        """Return the number of matching kills and the total kill value."""
+        query = """SELECT COUNT(*), TOTAL(kill_value)
+            FROM oper_kill
+            JOIN kill ON ok_killid = kill_id
+            WHERE ok_campaign = ? AND ok_operation = ?"""
+        res = self._conn.execute(query, (campaign, operation)).fetchall()
+        return tuple(res[0])

calefaction/modules/campaigns/getters.py

@@ -1,12 +1,13 @@
 # -*- coding: utf-8  -*-
 
-from datetime import datetime, timedelta
+from datetime import datetime
 from pathlib import Path
 from threading import Lock
 
 from flask import g
 
 from .database import CampaignDB
+from .update import update_operation
 from .._provided import app, config, logger
 from ...database import Database as MainDB
 
@@ -21,19 +22,6 @@ app.teardown_appcontext(CampaignDB.post_hook)
 
 _lock = Lock()
 
-def _update_operation(cname, opname, new):
-    """Update a campaign/operation."""
-    ...
-
-    operation = config["campaigns"][cname]["operations"][opname]
-    optype = operation["type"]
-    qualifiers = operation["qualifiers"]
-    show_isk = operation.get("isk", True)
-
-    primary = __import__("random").randint(10, 99)
-    secondary = __import__("random").randint(100000, 50000000)
-    g.campaign_db.set_overview(cname, opname, primary, secondary)
-
 def get_current():
     """Return the name of the currently selected campaign, or None."""
     if not config["enabled"]:
@@ -51,20 +39,20 @@ def get_overview(cname, opname):
 
     Updates the database if necessary, so this can take some time.
     """
-    maxdelta = timedelta(seconds=_MAX_STALENESS)
     with _lock:
-        last_updated = g.campaign_db.check_operation(cname, opname)
+        last_updated, _ = g.campaign_db.check_operation(cname, opname)
         if last_updated is None:
             logger.debug("Adding campaign=%s operation=%s", cname, opname)
-            _update_operation(cname, opname, new=True)
-            g.campaign_db.add_operation(cname, opname)
-        elif datetime.utcnow() - last_updated > maxdelta:
-            logger.debug("Updating campaign=%s operation=%s", cname, opname)
-            _update_operation(cname, opname, new=False)
-            g.campaign_db.touch_operation(cname, opname)
+            update_operation(cname, opname, new=True)
         else:
-            logger.debug("Using cache for campaign=%s operation=%s",
-                         cname, opname)
+            age = (datetime.utcnow() - last_updated).total_seconds()
+            if age > _MAX_STALENESS:
+                logger.debug("Updating (stale cache age=%d) campaign=%s "
+                             "operation=%s", age, cname, opname)
+                update_operation(cname, opname, new=False)
+            else:
+                logger.debug("Using cache (age=%d) for campaign=%s "
+                             "operation=%s", age, cname, opname)
         return g.campaign_db.get_overview(cname, opname)
 
 def get_summary(name, opname, limit=5):

calefaction/modules/campaigns/update.py

@@ -0,0 +1,119 @@
+# -*- coding: utf-8  -*-
+
+import sys
+import textwrap
+
+from flask import g
+
+from .._provided import config, logger
+
+__all__ = ["update_operation"]
+
+def _save_operation(cname, opname, primary, secondary, key=None):
+    """Save the given campaign/operation overview info in the database."""
+    secstr = "" if secondary is None else (" secondary=%d" % secondary)
+    logger.debug("Setting overview primary=%d%s campaign=%s operation=%s",
+                 primary, secstr, cname, opname)
+    g.campaign_db.set_overview(cname, opname, primary, secondary)
+    g.campaign_db.touch_operation(cname, opname, key=key)
+
+def _build_filter(qualifiers):
+    """Given a qualifiers string from the config, return a filter function.
+
+    This function is extremely sensitive since it executes arbitrary Python
+    code. It should never be run with a user-provided argument! We trust the
+    contents of a config file because it originates from a known place on the
+    filesystem.
+    """
+    namespace = {}
+    body = "def _func(kill):\n" + textwrap.indent(qualifiers, " " * 4)
+    exec(body, namespace)
+    return namespace["_func"]
+
+def _store_kill(cname, opnames, kill):
+    """Store the given kill and its associations into the database."""
+    kid = kill["killID"]
+    if g.campaign_db.has_kill(kid):
+        current = g.campaign_db.get_kill_associations(cname, kid)
+        opnames -= set(current)
+        if opnames:
+            logger.debug("Adding operations=%s to kill id=%d campaign=%s",
+                         ",".join(opnames), kid, cname)
+    else:
+        logger.debug("Adding kill id=%d campaign=%s operations=%s", kid, cname,
+                     ",".join(opnames))
+        g.campaign_db.add_kill(kill)
+
+    g.campaign_db.associate_kill(cname, kid, opnames)
+
+def _update_killboard_operations(cname, opnames, min_kill_id):
+    """Update all killboard-type operations in the given campaign subset."""
+    operations = config["campaigns"][cname]["operations"]
+    filters = []
+    for opname in opnames:
+        qualif = operations[opname]["qualifiers"]
+        filters.append((_build_filter(qualif), opname))
+
+    args = ["kills", "corporationID", g.config.get("corp.id"), "no-items",
+            "no-attackers", "orderDirection", "asc"]
+    if min_kill_id > 0:
+        args += ["afterKillID", min_kill_id]
+
+    max_kill_id = min_kill_id
+    for kill in g.eve.zkill.iter_killmails(*args):
+        kid = kill["killID"]
+        ktime = kill["killTime"]
+        logger.debug("Evaluating kill date=%s id=%d for campaign=%s "
+                     "operations=%s", ktime, kid, cname, ",".join(opnames))
+        max_kill_id = max(max_kill_id, kid)
+        ops = set()
+        for filt, opname in filters:
+            if filt(kill):
+                ops.add(opname)
+        if ops:
+            _store_kill(cname, ops, kill)
+
+    for opname in opnames:
+        primary, secondary = g.campaign_db.count_kills(cname, opname)
+        show_isk = operations[opname].get("isk", True)
+        if not show_isk:
+            secondary = None
+        _save_operation(cname, opname, primary, secondary, key=max_kill_id)
+
+def _update_collection_operations(cname, opnames):
+    """Update all collection-type operations in the given campaign subset."""
+    campaign = config["campaigns"][cname]
+    for opname in opnames:
+        operation = campaign["operations"][opname]
+        show_isk = operation.get("isk", True)
+
+        ...
+        primary = __import__("random").randint(10, 99)
+        secondary = __import__("random").randint(10000000, 5000000000) / 100 \
+            if show_isk else None
+
+        _save_operation(cname, opname, primary, secondary)
+
+def update_operation(cname, opname, new=False):
+    """Update a campaign/operation. Assumes a thread-exclusive lock is held."""
+    campaign = config["campaigns"][cname]
+    operations = campaign["operations"]
+    optype = operations[opname]["type"]
+    opnames = [opn for opn in campaign["enabled"]
+               if operations[opn]["type"] == optype]
+
+    if optype == "killboard":
+        opsubset = []
+        min_key = 0 if new else sys.maxsize
+        for opname in opnames:
+            last_updated, key = g.campaign_db.check_operation(cname, opname)
+            if new and last_updated is None:
+                opsubset.append(opname)
+            elif not new and last_updated is not None:
+                min_key = min(min_key, key)
+                opsubset.append(opname)
+        _update_killboard_operations(cname, opsubset, min_key)
+    elif optype == "collection":
+        _update_collection_operations(cname, opnames)
+    else:
+        raise RuntimeError("Unknown operation type: %s" % optype)

config/modules/campaigns.yml.sample

@@ -29,7 +29,7 @@ campaigns:
       frigates:
         # Full operation name:
         title: "Operation: Kill Foo Frigates"
-        # Data source (here, retrieve data from ZKillboard):
+        # Data source (here, retrieve data from zKillboard):
         type:  killboard
         # Show total ISK killed (defaults to true):
         isk:   true

data/schema_campaigns.sql

@@ -6,6 +6,7 @@ CREATE TABLE last_updated (
     lu_campaign TEXT,
     lu_operation TEXT,
     lu_date TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
+    lu_key INTEGER DEFAULT NULL,
     UNIQUE (lu_campaign, lu_operation)
 );
 
@@ -15,6 +16,43 @@ CREATE TABLE overview (
     ov_campaign TEXT,
     ov_operation TEXT,
     ov_primary INTEGER DEFAULT 0,
-    ov_secondary INTEGER DEFAULT NULL,
+    ov_secondary REAL DEFAULT NULL,
     UNIQUE (ov_campaign, ov_operation)
 );
+
+DROP TABLE IF EXISTS kill;
+
+CREATE TABLE kill (
+    kill_id INTEGER PRIMARY KEY,
+    kill_date TIMESTAMP,
+    kill_system INTEGER,
+    kill_victim_shipid INTEGER,
+    kill_victim_charid INTEGER,
+    kill_victim_corpid INTEGER,
+    kill_victim_allianceid INTEGER,
+    kill_victim_factionid INTEGER,
+    kill_value REAL
+);
+
+DROP TABLE IF EXISTS oper_kill;
+
+CREATE TABLE oper_kill (
+    ok_campaign TEXT,
+    ok_operation TEXT,
+    ok_killid INTEGER,
+    UNIQUE (ok_campaign, ok_operation, ok_killid),
+    FOREIGN KEY (ok_killid) REFERENCES kill (kill_id)
+        ON DELETE CASCADE ON UPDATE CASCADE
+);
+
+CREATE INDEX ok_campaign_operation ON oper_kill (ok_campaign, ok_operation);
+CREATE INDEX ok_campaign_killid    ON oper_kill (ok_campaign, ok_killid);
+
+DROP TABLE IF EXISTS oper_item;
+
+CREATE TABLE oper_item (
+    oi_campaign TEXT,
+    oi_operation TEXT
+    -- ...
+    -- UNIQUE (oi_campaign, oi_operation, ...)
+);

templates/campaigns/campaign.mako

@@ -29,7 +29,7 @@
                         </div>
                         % if secondary is not None:
                             <div class="secondary">
-                                <span class="num">${"{:,}".format(secondary)}</span>
+                                <span class="num">${"{:,.2f}".format(secondary)}</span>
                                 <span class="unit">${mod.get_unit(operation, secondary, primary=False)}</span>
                             </div>
                         % endif