Browse Source

Add zKillboard API support; implement most of campaign updating.

master
Ben Kurtovic 7 years ago
parent
commit
8dd4e6c7f1
10 changed files with 312 additions and 45 deletions
  1. +7
    -0
      calefaction/eve/__init__.py
  2. +0
    -1
      calefaction/eve/sso.py
  3. +61
    -0
      calefaction/eve/zkill.py
  4. +5
    -0
      calefaction/exceptions.py
  5. +67
    -17
      calefaction/modules/campaigns/database.py
  6. +12
    -24
      calefaction/modules/campaigns/getters.py
  7. +119
    -0
      calefaction/modules/campaigns/update.py
  8. +1
    -1
      config/modules/campaigns.yml.sample
  9. +39
    -1
      data/schema_campaigns.sql
  10. +1
    -1
      templates/campaigns/campaign.mako

+ 7
- 0
calefaction/eve/__init__.py View File

@@ -8,6 +8,7 @@ from .clock import Clock
from .esi import EVESwaggerInterface from .esi import EVESwaggerInterface
from .image import ImageServer from .image import ImageServer
from .sso import SSOManager from .sso import SSOManager
from .zkill import ZKillboard
from .. import __release__, baseLogger from .. import __release__, baseLogger


__all__ = ["EVE"] __all__ = ["EVE"]
@@ -25,6 +26,7 @@ class EVE:
self._esi = EVESwaggerInterface(session, logger.getChild("esi")) self._esi = EVESwaggerInterface(session, logger.getChild("esi"))
self._image = ImageServer() self._image = ImageServer()
self._sso = SSOManager(session, logger.getChild("sso")) self._sso = SSOManager(session, logger.getChild("sso"))
self._zkill = ZKillboard(session, logger.getChild("zkill"))


@staticmethod @staticmethod
def _get_user_agent(contact): def _get_user_agent(contact):
@@ -54,3 +56,8 @@ class EVE:
def sso(self): def sso(self):
"""The Single Sign-On API module.""" """The Single Sign-On API module."""
return self._sso return self._sso

@property
def zkill(self):
"""The zKillboard API module."""
return self._zkill

+ 0
- 1
calefaction/eve/sso.py View File

@@ -13,7 +13,6 @@ class SSOManager:


def __init__(self, session, logger): def __init__(self, session, logger):
self._session = session self._session = session

self._logger = logger self._logger = logger
self._debug = logger.debug self._debug = logger.debug




+ 61
- 0
calefaction/eve/zkill.py View File

@@ -0,0 +1,61 @@
# -*- coding: utf-8 -*-

import time

import requests

from ..exceptions import ZKillboardError

__all__ = ["ZKillboard"]

class ZKillboard:
"""EVE API module for zKillboard."""
_MAX_RATE = 0.5

def __init__(self, session, logger):
self._session = session
self._logger = logger
self._debug = logger.debug

self._base_url = "https://zkillboard.com/api"
self._last_query = 0

def query(self, *args):
"""Make an API query using the given arguments."""
query = "/" + "".join(str(arg) + "/" for arg in args)
url = self._base_url + query
delta = self._MAX_RATE - (time.time() - self._last_query)
if delta > 0:
self._debug("[GET] [wait %.2fs] %s", delta, query)
time.sleep(delta)
else:
self._debug("[GET] %s", query)

try:
resp = self._session.get(url, timeout=10)
resp.raise_for_status()
result = resp.json() if resp.content else None
except (requests.RequestException, ValueError):
self._logger.exception("zKillboard API query failed")
raise ZKillboardError()

self._last_query = time.time()
return result

def iter_killmails(self, *args):
"""Return an iterator over killmails using the given API arguments.

Automagically follows pagination as far as possible. (Be careful.)
"""
page = 1
while True:
if page > 1:
result = self.query(*args, "page", page)
else:
result = self.query(*args)

if result:
yield from result
page += 1
else:
break

+ 5
- 0
calefaction/exceptions.py View File

@@ -7,6 +7,7 @@ This module contains exceptions for Calefaction.
+-- AccessDeniedError +-- AccessDeniedError
+-- EVEAPIError +-- EVEAPIError
+-- EVEAPIForbiddenError +-- EVEAPIForbiddenError
+-- ZKillboardError
""" """


class CalefactionError(RuntimeError): class CalefactionError(RuntimeError):
@@ -24,3 +25,7 @@ class EVEAPIError(CalefactionError):
class EVEAPIForbiddenError(EVEAPIError): class EVEAPIForbiddenError(EVEAPIError):
"""We tried to make an API request that we don't have permission for.""" """We tried to make an API request that we don't have permission for."""
pass pass

class ZKillboardError(EVEAPIError):
"""Represents an error while using zKillboard's API."""
pass

+ 67
- 17
calefaction/modules/campaigns/database.py View File

@@ -46,30 +46,29 @@ class CampaignDB:
return self._conn.close() return self._conn.close()


def check_operation(self, campaign, operation): def check_operation(self, campaign, operation):
"""Return the last updated timestamp for the given operation.
"""Return the last updated timestamp and key for the given operation.


Return None if the given operation was never updated.
Return (None, None) if the given operation was never updated.
""" """
query = """SELECT lu_date FROM last_updated
query = """SELECT lu_date, lu_key FROM last_updated
WHERE lu_campaign = ? AND lu_operation = ?""" WHERE lu_campaign = ? AND lu_operation = ?"""
res = self._conn.execute(query, (campaign, operation)).fetchall() res = self._conn.execute(query, (campaign, operation)).fetchall()
if not res: if not res:
return None
return datetime.strptime(res[0][0], "%Y-%m-%d %H:%M:%S")
return None, None
return datetime.strptime(res[0][0], "%Y-%m-%d %H:%M:%S"), res[0][1]


def add_operation(self, campaign, operation):
"""Insert a new operation into the database as just updated."""
def touch_operation(self, campaign, operation, key=None):
"""Mark the given operation as just updated, or add it."""
with self._conn as conn: with self._conn as conn:
conn.execute("""INSERT INTO last_updated
(lu_campaign, lu_operation) VALUES (?, ?)""", (
campaign, operation))

def touch_operation(self, campaign, operation):
"""Mark the given operation as just updated."""
with self._conn as conn:
conn.execute("""UPDATE last_updated SET lu_date = CURRENT_TIMESTAMP
cur = conn.execute("BEGIN TRANSACTION")
cur.execute("""UPDATE last_updated
SET lu_date = CURRENT_TIMESTAMP, lu_key = ?
WHERE lu_campaign = ? AND lu_operation = ?""", ( WHERE lu_campaign = ? AND lu_operation = ?""", (
campaign, operation))
key, campaign, operation))
if cur.rowcount == 0:
cur.execute("""INSERT INTO last_updated
(lu_campaign, lu_operation, lu_key) VALUES (?, ?, ?)""", (
campaign, operation, key))


def set_overview(self, campaign, operation, primary, secondary=None): def set_overview(self, campaign, operation, primary, secondary=None):
"""Set overview information for this operation.""" """Set overview information for this operation."""
@@ -84,4 +83,55 @@ class CampaignDB:
query = """SELECT ov_primary, ov_secondary FROM overview query = """SELECT ov_primary, ov_secondary FROM overview
WHERE ov_campaign = ? AND ov_operation = ?""" WHERE ov_campaign = ? AND ov_operation = ?"""
res = self._conn.execute(query, (campaign, operation)).fetchall() res = self._conn.execute(query, (campaign, operation)).fetchall()
return res[0] if res else (0, None)
return tuple(res[0]) if res else (0, None)

def has_kill(self, kill_id):
"""Return whether the database has a killmail with the given ID."""
query = "SELECT 1 FROM kill WHERE kill_id = ?"
res = self._conn.execute(query, (kill_id,)).fetchall()
return bool(res)

def add_kill(self, kill):
"""Insert a killmail into the database."""
try:
datetime.strptime(kill["killTime"], "%Y-%m-%d %H:%M:%S")
except ValueError:
raise RuntimeError("Invalid kill_date=%s for kill_id=%d" % (
kill["killTime"], kill["killID"]))

query = """INSERT OR REPLACE INTO kill (
kill_id, kill_date, kill_system, kill_victim_shipid,
kill_victim_charid, kill_victim_corpid, kill_victim_allianceid,
kill_victim_factionid, kill_value)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)"""
args = (
kill["killID"], kill["killTime"], kill["solarSystemID"],
kill["victim"]["shipTypeID"], kill["victim"]["characterID"],
kill["victim"]["corporationID"], kill["victim"]["allianceID"],
kill["victim"]["factionID"], kill["zkb"]["totalValue"])
with self._conn as conn:
conn.execute(query, args)

def get_kill_associations(self, campaign, kill_id):
"""Return a list of operations associated with a campaign and kill."""
query = """SELECT ok_operation FROM oper_kill
WHERE ok_campaign = ? AND ok_killid = ?"""
res = self._conn.execute(query, (campaign, kill_id)).fetchall()
return [row[0] for row in res]

def associate_kill(self, campaign, kill_id, operations):
"""Associate a killmail with a set of campaign/operations."""
query = """INSERT OR IGNORE INTO oper_kill
(ok_campaign, ok_operation, ok_killid) VALUES (?, ?, ?)"""
arglist = [(campaign, op, kill_id) for op in operations]
with self._conn as conn:
conn.executemany(query, arglist)

def count_kills(self, campaign, operation):
"""Return the number of matching kills and the total kill value."""
query = """SELECT COUNT(*), TOTAL(kill_value)
FROM oper_kill
JOIN kill ON ok_killid = kill_id
WHERE ok_campaign = ? AND ok_operation = ?"""
res = self._conn.execute(query, (campaign, operation)).fetchall()
return tuple(res[0])

+ 12
- 24
calefaction/modules/campaigns/getters.py View File

@@ -1,12 +1,13 @@
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-


from datetime import datetime, timedelta
from datetime import datetime
from pathlib import Path from pathlib import Path
from threading import Lock from threading import Lock


from flask import g from flask import g


from .database import CampaignDB from .database import CampaignDB
from .update import update_operation
from .._provided import app, config, logger from .._provided import app, config, logger
from ...database import Database as MainDB from ...database import Database as MainDB


@@ -21,19 +22,6 @@ app.teardown_appcontext(CampaignDB.post_hook)


_lock = Lock() _lock = Lock()


def _update_operation(cname, opname, new):
"""Update a campaign/operation."""
...

operation = config["campaigns"][cname]["operations"][opname]
optype = operation["type"]
qualifiers = operation["qualifiers"]
show_isk = operation.get("isk", True)

primary = __import__("random").randint(10, 99)
secondary = __import__("random").randint(100000, 50000000)
g.campaign_db.set_overview(cname, opname, primary, secondary)

def get_current(): def get_current():
"""Return the name of the currently selected campaign, or None.""" """Return the name of the currently selected campaign, or None."""
if not config["enabled"]: if not config["enabled"]:
@@ -51,20 +39,20 @@ def get_overview(cname, opname):


Updates the database if necessary, so this can take some time. Updates the database if necessary, so this can take some time.
""" """
maxdelta = timedelta(seconds=_MAX_STALENESS)
with _lock: with _lock:
last_updated = g.campaign_db.check_operation(cname, opname)
last_updated, _ = g.campaign_db.check_operation(cname, opname)
if last_updated is None: if last_updated is None:
logger.debug("Adding campaign=%s operation=%s", cname, opname) logger.debug("Adding campaign=%s operation=%s", cname, opname)
_update_operation(cname, opname, new=True)
g.campaign_db.add_operation(cname, opname)
elif datetime.utcnow() - last_updated > maxdelta:
logger.debug("Updating campaign=%s operation=%s", cname, opname)
_update_operation(cname, opname, new=False)
g.campaign_db.touch_operation(cname, opname)
update_operation(cname, opname, new=True)
else: else:
logger.debug("Using cache for campaign=%s operation=%s",
cname, opname)
age = (datetime.utcnow() - last_updated).total_seconds()
if age > _MAX_STALENESS:
logger.debug("Updating (stale cache age=%d) campaign=%s "
"operation=%s", age, cname, opname)
update_operation(cname, opname, new=False)
else:
logger.debug("Using cache (age=%d) for campaign=%s "
"operation=%s", age, cname, opname)
return g.campaign_db.get_overview(cname, opname) return g.campaign_db.get_overview(cname, opname)


def get_summary(name, opname, limit=5): def get_summary(name, opname, limit=5):


+ 119
- 0
calefaction/modules/campaigns/update.py View File

@@ -0,0 +1,119 @@
# -*- coding: utf-8 -*-

import sys
import textwrap

from flask import g

from .._provided import config, logger

__all__ = ["update_operation"]

def _save_operation(cname, opname, primary, secondary, key=None):
"""Save the given campaign/operation overview info in the database."""
secstr = "" if secondary is None else (" secondary=%d" % secondary)
logger.debug("Setting overview primary=%d%s campaign=%s operation=%s",
primary, secstr, cname, opname)
g.campaign_db.set_overview(cname, opname, primary, secondary)
g.campaign_db.touch_operation(cname, opname, key=key)

def _build_filter(qualifiers):
"""Given a qualifiers string from the config, return a filter function.

This function is extremely sensitive since it executes arbitrary Python
code. It should never be run with a user-provided argument! We trust the
contents of a config file because it originates from a known place on the
filesystem.
"""
namespace = {}
body = "def _func(kill):\n" + textwrap.indent(qualifiers, " " * 4)
exec(body, namespace)
return namespace["_func"]

def _store_kill(cname, opnames, kill):
"""Store the given kill and its associations into the database."""
kid = kill["killID"]
if g.campaign_db.has_kill(kid):
current = g.campaign_db.get_kill_associations(cname, kid)
opnames -= set(current)
if opnames:
logger.debug("Adding operations=%s to kill id=%d campaign=%s",
",".join(opnames), kid, cname)
else:
logger.debug("Adding kill id=%d campaign=%s operations=%s", kid, cname,
",".join(opnames))
g.campaign_db.add_kill(kill)

g.campaign_db.associate_kill(cname, kid, opnames)

def _update_killboard_operations(cname, opnames, min_kill_id):
"""Update all killboard-type operations in the given campaign subset."""
operations = config["campaigns"][cname]["operations"]
filters = []
for opname in opnames:
qualif = operations[opname]["qualifiers"]
filters.append((_build_filter(qualif), opname))

args = ["kills", "corporationID", g.config.get("corp.id"), "no-items",
"no-attackers", "orderDirection", "asc"]
if min_kill_id > 0:
args += ["afterKillID", min_kill_id]

max_kill_id = min_kill_id
for kill in g.eve.zkill.iter_killmails(*args):
kid = kill["killID"]
ktime = kill["killTime"]
logger.debug("Evaluating kill date=%s id=%d for campaign=%s "
"operations=%s", ktime, kid, cname, ",".join(opnames))
max_kill_id = max(max_kill_id, kid)
ops = set()
for filt, opname in filters:
if filt(kill):
ops.add(opname)
if ops:
_store_kill(cname, ops, kill)

for opname in opnames:
primary, secondary = g.campaign_db.count_kills(cname, opname)
show_isk = operations[opname].get("isk", True)
if not show_isk:
secondary = None
_save_operation(cname, opname, primary, secondary, key=max_kill_id)

def _update_collection_operations(cname, opnames):
"""Update all collection-type operations in the given campaign subset."""
campaign = config["campaigns"][cname]
for opname in opnames:
operation = campaign["operations"][opname]
show_isk = operation.get("isk", True)

...
primary = __import__("random").randint(10, 99)
secondary = __import__("random").randint(10000000, 5000000000) / 100 \
if show_isk else None

_save_operation(cname, opname, primary, secondary)

def update_operation(cname, opname, new=False):
"""Update a campaign/operation. Assumes a thread-exclusive lock is held."""
campaign = config["campaigns"][cname]
operations = campaign["operations"]
optype = operations[opname]["type"]
opnames = [opn for opn in campaign["enabled"]
if operations[opn]["type"] == optype]

if optype == "killboard":
opsubset = []
min_key = 0 if new else sys.maxsize
for opname in opnames:
last_updated, key = g.campaign_db.check_operation(cname, opname)
if new and last_updated is None:
opsubset.append(opname)
elif not new and last_updated is not None:
min_key = min(min_key, key)
opsubset.append(opname)
_update_killboard_operations(cname, opsubset, min_key)
elif optype == "collection":
_update_collection_operations(cname, opnames)
else:
raise RuntimeError("Unknown operation type: %s" % optype)

+ 1
- 1
config/modules/campaigns.yml.sample View File

@@ -29,7 +29,7 @@ campaigns:
frigates: frigates:
# Full operation name: # Full operation name:
title: "Operation: Kill Foo Frigates" title: "Operation: Kill Foo Frigates"
# Data source (here, retrieve data from ZKillboard):
# Data source (here, retrieve data from zKillboard):
type: killboard type: killboard
# Show total ISK killed (defaults to true): # Show total ISK killed (defaults to true):
isk: true isk: true


+ 39
- 1
data/schema_campaigns.sql View File

@@ -6,6 +6,7 @@ CREATE TABLE last_updated (
lu_campaign TEXT, lu_campaign TEXT,
lu_operation TEXT, lu_operation TEXT,
lu_date TIMESTAMP DEFAULT CURRENT_TIMESTAMP, lu_date TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
lu_key INTEGER DEFAULT NULL,
UNIQUE (lu_campaign, lu_operation) UNIQUE (lu_campaign, lu_operation)
); );


@@ -15,6 +16,43 @@ CREATE TABLE overview (
ov_campaign TEXT, ov_campaign TEXT,
ov_operation TEXT, ov_operation TEXT,
ov_primary INTEGER DEFAULT 0, ov_primary INTEGER DEFAULT 0,
ov_secondary INTEGER DEFAULT NULL,
ov_secondary REAL DEFAULT NULL,
UNIQUE (ov_campaign, ov_operation) UNIQUE (ov_campaign, ov_operation)
); );

DROP TABLE IF EXISTS kill;

CREATE TABLE kill (
kill_id INTEGER PRIMARY KEY,
kill_date TIMESTAMP,
kill_system INTEGER,
kill_victim_shipid INTEGER,
kill_victim_charid INTEGER,
kill_victim_corpid INTEGER,
kill_victim_allianceid INTEGER,
kill_victim_factionid INTEGER,
kill_value REAL
);

DROP TABLE IF EXISTS oper_kill;

CREATE TABLE oper_kill (
ok_campaign TEXT,
ok_operation TEXT,
ok_killid INTEGER,
UNIQUE (ok_campaign, ok_operation, ok_killid),
FOREIGN KEY (ok_killid) REFERENCES kill (kill_id)
ON DELETE CASCADE ON UPDATE CASCADE
);

CREATE INDEX ok_campaign_operation ON oper_kill (ok_campaign, ok_operation);
CREATE INDEX ok_campaign_killid ON oper_kill (ok_campaign, ok_killid);

DROP TABLE IF EXISTS oper_item;

CREATE TABLE oper_item (
oi_campaign TEXT,
oi_operation TEXT
-- ...
-- UNIQUE (oi_campaign, oi_operation, ...)
);

+ 1
- 1
templates/campaigns/campaign.mako View File

@@ -29,7 +29,7 @@
</div> </div>
% if secondary is not None: % if secondary is not None:
<div class="secondary"> <div class="secondary">
<span class="num">${"{:,}".format(secondary)}</span>
<span class="num">${"{:,.2f}".format(secondary)}</span>
<span class="unit">${mod.get_unit(operation, secondary, primary=False)}</span> <span class="unit">${mod.get_unit(operation, secondary, primary=False)}</span>
</div> </div>
% endif % endif


Loading…
Cancel
Save