From 8dd4e6c7f1e69b73247db0be00b783662fe12e42 Mon Sep 17 00:00:00 2001 From: Ben Kurtovic Date: Wed, 28 Dec 2016 06:20:15 -0500 Subject: [PATCH] Add zKillboard API support; implement most of campaign updating. --- calefaction/eve/__init__.py | 7 ++ calefaction/eve/sso.py | 1 - calefaction/eve/zkill.py | 61 +++++++++++++++ calefaction/exceptions.py | 5 ++ calefaction/modules/campaigns/database.py | 84 ++++++++++++++++----- calefaction/modules/campaigns/getters.py | 36 +++------ calefaction/modules/campaigns/update.py | 119 ++++++++++++++++++++++++++++++ config/modules/campaigns.yml.sample | 2 +- data/schema_campaigns.sql | 40 +++++++++- templates/campaigns/campaign.mako | 2 +- 10 files changed, 312 insertions(+), 45 deletions(-) create mode 100644 calefaction/eve/zkill.py create mode 100644 calefaction/modules/campaigns/update.py diff --git a/calefaction/eve/__init__.py b/calefaction/eve/__init__.py index 3dc1ead..99470fc 100644 --- a/calefaction/eve/__init__.py +++ b/calefaction/eve/__init__.py @@ -8,6 +8,7 @@ from .clock import Clock from .esi import EVESwaggerInterface from .image import ImageServer from .sso import SSOManager +from .zkill import ZKillboard from .. import __release__, baseLogger __all__ = ["EVE"] @@ -25,6 +26,7 @@ class EVE: self._esi = EVESwaggerInterface(session, logger.getChild("esi")) self._image = ImageServer() self._sso = SSOManager(session, logger.getChild("sso")) + self._zkill = ZKillboard(session, logger.getChild("zkill")) @staticmethod def _get_user_agent(contact): @@ -54,3 +56,8 @@ class EVE: def sso(self): """The Single Sign-On API module.""" return self._sso + + @property + def zkill(self): + """The zKillboard API module.""" + return self._zkill diff --git a/calefaction/eve/sso.py b/calefaction/eve/sso.py index ca64e50..37a3ed9 100644 --- a/calefaction/eve/sso.py +++ b/calefaction/eve/sso.py @@ -13,7 +13,6 @@ class SSOManager: def __init__(self, session, logger): self._session = session - self._logger = logger self._debug = logger.debug diff --git a/calefaction/eve/zkill.py b/calefaction/eve/zkill.py new file mode 100644 index 0000000..90f3377 --- /dev/null +++ b/calefaction/eve/zkill.py @@ -0,0 +1,61 @@ +# -*- coding: utf-8 -*- + +import time + +import requests + +from ..exceptions import ZKillboardError + +__all__ = ["ZKillboard"] + +class ZKillboard: + """EVE API module for zKillboard.""" + _MAX_RATE = 0.5 + + def __init__(self, session, logger): + self._session = session + self._logger = logger + self._debug = logger.debug + + self._base_url = "https://zkillboard.com/api" + self._last_query = 0 + + def query(self, *args): + """Make an API query using the given arguments.""" + query = "/" + "".join(str(arg) + "/" for arg in args) + url = self._base_url + query + delta = self._MAX_RATE - (time.time() - self._last_query) + if delta > 0: + self._debug("[GET] [wait %.2fs] %s", delta, query) + time.sleep(delta) + else: + self._debug("[GET] %s", query) + + try: + resp = self._session.get(url, timeout=10) + resp.raise_for_status() + result = resp.json() if resp.content else None + except (requests.RequestException, ValueError): + self._logger.exception("zKillboard API query failed") + raise ZKillboardError() + + self._last_query = time.time() + return result + + def iter_killmails(self, *args): + """Return an iterator over killmails using the given API arguments. + + Automagically follows pagination as far as possible. (Be careful.) + """ + page = 1 + while True: + if page > 1: + result = self.query(*args, "page", page) + else: + result = self.query(*args) + + if result: + yield from result + page += 1 + else: + break diff --git a/calefaction/exceptions.py b/calefaction/exceptions.py index 22250b2..56c46b1 100644 --- a/calefaction/exceptions.py +++ b/calefaction/exceptions.py @@ -7,6 +7,7 @@ This module contains exceptions for Calefaction. +-- AccessDeniedError +-- EVEAPIError +-- EVEAPIForbiddenError + +-- ZKillboardError """ class CalefactionError(RuntimeError): @@ -24,3 +25,7 @@ class EVEAPIError(CalefactionError): class EVEAPIForbiddenError(EVEAPIError): """We tried to make an API request that we don't have permission for.""" pass + +class ZKillboardError(EVEAPIError): + """Represents an error while using zKillboard's API.""" + pass diff --git a/calefaction/modules/campaigns/database.py b/calefaction/modules/campaigns/database.py index e5a132c..3faf09d 100644 --- a/calefaction/modules/campaigns/database.py +++ b/calefaction/modules/campaigns/database.py @@ -46,30 +46,29 @@ class CampaignDB: return self._conn.close() def check_operation(self, campaign, operation): - """Return the last updated timestamp for the given operation. + """Return the last updated timestamp and key for the given operation. - Return None if the given operation was never updated. + Return (None, None) if the given operation was never updated. """ - query = """SELECT lu_date FROM last_updated + query = """SELECT lu_date, lu_key FROM last_updated WHERE lu_campaign = ? AND lu_operation = ?""" res = self._conn.execute(query, (campaign, operation)).fetchall() if not res: - return None - return datetime.strptime(res[0][0], "%Y-%m-%d %H:%M:%S") + return None, None + return datetime.strptime(res[0][0], "%Y-%m-%d %H:%M:%S"), res[0][1] - def add_operation(self, campaign, operation): - """Insert a new operation into the database as just updated.""" + def touch_operation(self, campaign, operation, key=None): + """Mark the given operation as just updated, or add it.""" with self._conn as conn: - conn.execute("""INSERT INTO last_updated - (lu_campaign, lu_operation) VALUES (?, ?)""", ( - campaign, operation)) - - def touch_operation(self, campaign, operation): - """Mark the given operation as just updated.""" - with self._conn as conn: - conn.execute("""UPDATE last_updated SET lu_date = CURRENT_TIMESTAMP + cur = conn.execute("BEGIN TRANSACTION") + cur.execute("""UPDATE last_updated + SET lu_date = CURRENT_TIMESTAMP, lu_key = ? WHERE lu_campaign = ? AND lu_operation = ?""", ( - campaign, operation)) + key, campaign, operation)) + if cur.rowcount == 0: + cur.execute("""INSERT INTO last_updated + (lu_campaign, lu_operation, lu_key) VALUES (?, ?, ?)""", ( + campaign, operation, key)) def set_overview(self, campaign, operation, primary, secondary=None): """Set overview information for this operation.""" @@ -84,4 +83,55 @@ class CampaignDB: query = """SELECT ov_primary, ov_secondary FROM overview WHERE ov_campaign = ? AND ov_operation = ?""" res = self._conn.execute(query, (campaign, operation)).fetchall() - return res[0] if res else (0, None) + return tuple(res[0]) if res else (0, None) + + def has_kill(self, kill_id): + """Return whether the database has a killmail with the given ID.""" + query = "SELECT 1 FROM kill WHERE kill_id = ?" + res = self._conn.execute(query, (kill_id,)).fetchall() + return bool(res) + + def add_kill(self, kill): + """Insert a killmail into the database.""" + try: + datetime.strptime(kill["killTime"], "%Y-%m-%d %H:%M:%S") + except ValueError: + raise RuntimeError("Invalid kill_date=%s for kill_id=%d" % ( + kill["killTime"], kill["killID"])) + + query = """INSERT OR REPLACE INTO kill ( + kill_id, kill_date, kill_system, kill_victim_shipid, + kill_victim_charid, kill_victim_corpid, kill_victim_allianceid, + kill_victim_factionid, kill_value) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)""" + args = ( + kill["killID"], kill["killTime"], kill["solarSystemID"], + kill["victim"]["shipTypeID"], kill["victim"]["characterID"], + kill["victim"]["corporationID"], kill["victim"]["allianceID"], + kill["victim"]["factionID"], kill["zkb"]["totalValue"]) + with self._conn as conn: + conn.execute(query, args) + + def get_kill_associations(self, campaign, kill_id): + """Return a list of operations associated with a campaign and kill.""" + query = """SELECT ok_operation FROM oper_kill + WHERE ok_campaign = ? AND ok_killid = ?""" + res = self._conn.execute(query, (campaign, kill_id)).fetchall() + return [row[0] for row in res] + + def associate_kill(self, campaign, kill_id, operations): + """Associate a killmail with a set of campaign/operations.""" + query = """INSERT OR IGNORE INTO oper_kill + (ok_campaign, ok_operation, ok_killid) VALUES (?, ?, ?)""" + arglist = [(campaign, op, kill_id) for op in operations] + with self._conn as conn: + conn.executemany(query, arglist) + + def count_kills(self, campaign, operation): + """Return the number of matching kills and the total kill value.""" + query = """SELECT COUNT(*), TOTAL(kill_value) + FROM oper_kill + JOIN kill ON ok_killid = kill_id + WHERE ok_campaign = ? AND ok_operation = ?""" + res = self._conn.execute(query, (campaign, operation)).fetchall() + return tuple(res[0]) diff --git a/calefaction/modules/campaigns/getters.py b/calefaction/modules/campaigns/getters.py index 9cbb653..24acab3 100644 --- a/calefaction/modules/campaigns/getters.py +++ b/calefaction/modules/campaigns/getters.py @@ -1,12 +1,13 @@ # -*- coding: utf-8 -*- -from datetime import datetime, timedelta +from datetime import datetime from pathlib import Path from threading import Lock from flask import g from .database import CampaignDB +from .update import update_operation from .._provided import app, config, logger from ...database import Database as MainDB @@ -21,19 +22,6 @@ app.teardown_appcontext(CampaignDB.post_hook) _lock = Lock() -def _update_operation(cname, opname, new): - """Update a campaign/operation.""" - ... - - operation = config["campaigns"][cname]["operations"][opname] - optype = operation["type"] - qualifiers = operation["qualifiers"] - show_isk = operation.get("isk", True) - - primary = __import__("random").randint(10, 99) - secondary = __import__("random").randint(100000, 50000000) - g.campaign_db.set_overview(cname, opname, primary, secondary) - def get_current(): """Return the name of the currently selected campaign, or None.""" if not config["enabled"]: @@ -51,20 +39,20 @@ def get_overview(cname, opname): Updates the database if necessary, so this can take some time. """ - maxdelta = timedelta(seconds=_MAX_STALENESS) with _lock: - last_updated = g.campaign_db.check_operation(cname, opname) + last_updated, _ = g.campaign_db.check_operation(cname, opname) if last_updated is None: logger.debug("Adding campaign=%s operation=%s", cname, opname) - _update_operation(cname, opname, new=True) - g.campaign_db.add_operation(cname, opname) - elif datetime.utcnow() - last_updated > maxdelta: - logger.debug("Updating campaign=%s operation=%s", cname, opname) - _update_operation(cname, opname, new=False) - g.campaign_db.touch_operation(cname, opname) + update_operation(cname, opname, new=True) else: - logger.debug("Using cache for campaign=%s operation=%s", - cname, opname) + age = (datetime.utcnow() - last_updated).total_seconds() + if age > _MAX_STALENESS: + logger.debug("Updating (stale cache age=%d) campaign=%s " + "operation=%s", age, cname, opname) + update_operation(cname, opname, new=False) + else: + logger.debug("Using cache (age=%d) for campaign=%s " + "operation=%s", age, cname, opname) return g.campaign_db.get_overview(cname, opname) def get_summary(name, opname, limit=5): diff --git a/calefaction/modules/campaigns/update.py b/calefaction/modules/campaigns/update.py new file mode 100644 index 0000000..3091da6 --- /dev/null +++ b/calefaction/modules/campaigns/update.py @@ -0,0 +1,119 @@ +# -*- coding: utf-8 -*- + +import sys +import textwrap + +from flask import g + +from .._provided import config, logger + +__all__ = ["update_operation"] + +def _save_operation(cname, opname, primary, secondary, key=None): + """Save the given campaign/operation overview info in the database.""" + secstr = "" if secondary is None else (" secondary=%d" % secondary) + logger.debug("Setting overview primary=%d%s campaign=%s operation=%s", + primary, secstr, cname, opname) + g.campaign_db.set_overview(cname, opname, primary, secondary) + g.campaign_db.touch_operation(cname, opname, key=key) + +def _build_filter(qualifiers): + """Given a qualifiers string from the config, return a filter function. + + This function is extremely sensitive since it executes arbitrary Python + code. It should never be run with a user-provided argument! We trust the + contents of a config file because it originates from a known place on the + filesystem. + """ + namespace = {} + body = "def _func(kill):\n" + textwrap.indent(qualifiers, " " * 4) + exec(body, namespace) + return namespace["_func"] + +def _store_kill(cname, opnames, kill): + """Store the given kill and its associations into the database.""" + kid = kill["killID"] + if g.campaign_db.has_kill(kid): + current = g.campaign_db.get_kill_associations(cname, kid) + opnames -= set(current) + if opnames: + logger.debug("Adding operations=%s to kill id=%d campaign=%s", + ",".join(opnames), kid, cname) + else: + logger.debug("Adding kill id=%d campaign=%s operations=%s", kid, cname, + ",".join(opnames)) + g.campaign_db.add_kill(kill) + + g.campaign_db.associate_kill(cname, kid, opnames) + +def _update_killboard_operations(cname, opnames, min_kill_id): + """Update all killboard-type operations in the given campaign subset.""" + operations = config["campaigns"][cname]["operations"] + filters = [] + for opname in opnames: + qualif = operations[opname]["qualifiers"] + filters.append((_build_filter(qualif), opname)) + + args = ["kills", "corporationID", g.config.get("corp.id"), "no-items", + "no-attackers", "orderDirection", "asc"] + if min_kill_id > 0: + args += ["afterKillID", min_kill_id] + + max_kill_id = min_kill_id + for kill in g.eve.zkill.iter_killmails(*args): + kid = kill["killID"] + ktime = kill["killTime"] + logger.debug("Evaluating kill date=%s id=%d for campaign=%s " + "operations=%s", ktime, kid, cname, ",".join(opnames)) + max_kill_id = max(max_kill_id, kid) + ops = set() + for filt, opname in filters: + if filt(kill): + ops.add(opname) + if ops: + _store_kill(cname, ops, kill) + + for opname in opnames: + primary, secondary = g.campaign_db.count_kills(cname, opname) + show_isk = operations[opname].get("isk", True) + if not show_isk: + secondary = None + _save_operation(cname, opname, primary, secondary, key=max_kill_id) + +def _update_collection_operations(cname, opnames): + """Update all collection-type operations in the given campaign subset.""" + campaign = config["campaigns"][cname] + for opname in opnames: + operation = campaign["operations"][opname] + show_isk = operation.get("isk", True) + + ... + primary = __import__("random").randint(10, 99) + secondary = __import__("random").randint(10000000, 5000000000) / 100 \ + if show_isk else None + + _save_operation(cname, opname, primary, secondary) + +def update_operation(cname, opname, new=False): + """Update a campaign/operation. Assumes a thread-exclusive lock is held.""" + campaign = config["campaigns"][cname] + operations = campaign["operations"] + optype = operations[opname]["type"] + opnames = [opn for opn in campaign["enabled"] + if operations[opn]["type"] == optype] + + if optype == "killboard": + opsubset = [] + min_key = 0 if new else sys.maxsize + for opname in opnames: + last_updated, key = g.campaign_db.check_operation(cname, opname) + if new and last_updated is None: + opsubset.append(opname) + elif not new and last_updated is not None: + min_key = min(min_key, key) + opsubset.append(opname) + _update_killboard_operations(cname, opsubset, min_key) + elif optype == "collection": + _update_collection_operations(cname, opnames) + else: + raise RuntimeError("Unknown operation type: %s" % optype) diff --git a/config/modules/campaigns.yml.sample b/config/modules/campaigns.yml.sample index 112bb60..89a5d86 100644 --- a/config/modules/campaigns.yml.sample +++ b/config/modules/campaigns.yml.sample @@ -29,7 +29,7 @@ campaigns: frigates: # Full operation name: title: "Operation: Kill Foo Frigates" - # Data source (here, retrieve data from ZKillboard): + # Data source (here, retrieve data from zKillboard): type: killboard # Show total ISK killed (defaults to true): isk: true diff --git a/data/schema_campaigns.sql b/data/schema_campaigns.sql index 5df5c3d..e460586 100644 --- a/data/schema_campaigns.sql +++ b/data/schema_campaigns.sql @@ -6,6 +6,7 @@ CREATE TABLE last_updated ( lu_campaign TEXT, lu_operation TEXT, lu_date TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + lu_key INTEGER DEFAULT NULL, UNIQUE (lu_campaign, lu_operation) ); @@ -15,6 +16,43 @@ CREATE TABLE overview ( ov_campaign TEXT, ov_operation TEXT, ov_primary INTEGER DEFAULT 0, - ov_secondary INTEGER DEFAULT NULL, + ov_secondary REAL DEFAULT NULL, UNIQUE (ov_campaign, ov_operation) ); + +DROP TABLE IF EXISTS kill; + +CREATE TABLE kill ( + kill_id INTEGER PRIMARY KEY, + kill_date TIMESTAMP, + kill_system INTEGER, + kill_victim_shipid INTEGER, + kill_victim_charid INTEGER, + kill_victim_corpid INTEGER, + kill_victim_allianceid INTEGER, + kill_victim_factionid INTEGER, + kill_value REAL +); + +DROP TABLE IF EXISTS oper_kill; + +CREATE TABLE oper_kill ( + ok_campaign TEXT, + ok_operation TEXT, + ok_killid INTEGER, + UNIQUE (ok_campaign, ok_operation, ok_killid), + FOREIGN KEY (ok_killid) REFERENCES kill (kill_id) + ON DELETE CASCADE ON UPDATE CASCADE +); + +CREATE INDEX ok_campaign_operation ON oper_kill (ok_campaign, ok_operation); +CREATE INDEX ok_campaign_killid ON oper_kill (ok_campaign, ok_killid); + +DROP TABLE IF EXISTS oper_item; + +CREATE TABLE oper_item ( + oi_campaign TEXT, + oi_operation TEXT + -- ... + -- UNIQUE (oi_campaign, oi_operation, ...) +); diff --git a/templates/campaigns/campaign.mako b/templates/campaigns/campaign.mako index 2e25652..129c9b2 100644 --- a/templates/campaigns/campaign.mako +++ b/templates/campaigns/campaign.mako @@ -29,7 +29,7 @@ % if secondary is not None:
- ${"{:,}".format(secondary)} + ${"{:,.2f}".format(secondary)} ${mod.get_unit(operation, secondary, primary=False)}
% endif