Browse Source

Add types to universe data and implement collection campaigns op type.

master
Ben Kurtovic 7 years ago
parent
commit
12c0ebe774
11 changed files with 282 additions and 63 deletions
  1. +64
    -13
      calefaction/auth.py
  2. +13
    -0
      calefaction/database.py
  3. +6
    -6
      calefaction/eve/esi.py
  4. +47
    -4
      calefaction/eve/universe.py
  5. +40
    -0
      calefaction/modules/campaigns/database.py
  6. +2
    -2
      calefaction/modules/campaigns/getters.py
  7. +57
    -13
      calefaction/modules/campaigns/update.py
  8. +2
    -1
      config/modules/campaigns.yml.sample
  9. +7
    -3
      data/schema_campaigns.sql
  10. +28
    -14
      scripts/import_sde.py
  11. +16
    -7
      templates/campaigns/renderers.mako

+ 64
- 13
calefaction/auth.py View File

@@ -181,19 +181,30 @@ class AuthManager:
g.db.update_auth(cid, token, expires, refresh)
return token

def _is_corp_member(self, token, char_id):
""""Return whether the given character is in the site's corp."""
resp = self._eve.esi(token).v3.characters(char_id).get()
return resp.get("corporation_id") == self._config.get("corp.id")

def _check_access(self, token, char_id):
""""Check whether the given character is allowed to access this site.

If allowed, do nothing. If not, raise AccessDeniedError.
"""
resp = self._eve.esi(token).v3.characters(char_id).get()
if resp.get("corporation_id") != self._config.get("corp.id"):
if not self._is_corp_member(token, char_id):
self._debug("Access denied per corp membership for char id=%d "
"session id=%d", char_id, session["id"])
g.db.drop_auth(char_id)
self._invalidate_session()
raise AccessDeniedError()

def _cache_token(self, cid, token):
"""Cache the given token for this request."""
if hasattr(g, "_cached_tokens"):
g._cached_tokens[cid] = token
else:
g._cached_tokens = {cid: token}

def _update_prop_cache(self, module, prop, value):
"""Update the value of a character module property in the cache."""
if hasattr(g, "_character_modprops"):
@@ -282,21 +293,29 @@ class AuthManager:
self._update_prop_cache(module, prop, value)
return True

def get_token(self):
"""Return a valid token for the current character, or None.
def get_token(self, cid=None):
"""Return a valid token for the given character.

If no character is given, we use the current session's character. If a
token couldn't be retrieved, return None.

Assuming this is called in a restricted route (following a True result
from is_authenticated), this function makes no API calls and should
always succeed. If it is called in other circumstances, it may fail and
return None.
Assuming we want the current character's token and this is called in a
restricted route (following a True result from is_authenticated), this
function makes no API calls and should always succeed. If it is called
in other circumstances, it may return None or raise EVEAPIError.
"""
cid = self.get_character_id()
if cid is None:
cid = self.get_character_id()
if not cid:
return None

if not hasattr(g, "_cached_token"):
g._cached_token = self._get_token(cid)
return g._cached_token
if hasattr(g, "_cached_tokens"):
if cid in g._cached_tokens:
return g._cached_tokens[cid]

token = self._get_token(cid)
self._cache_token(cid, token)
return token

def is_authenticated(self):
"""Return whether the user has permission to access this site.
@@ -324,7 +343,7 @@ class AuthManager:
self._debug("Access granted for char id=%d session id=%d", cid,
session["id"])
g.db.touch_session(session["id"])
g._cached_token = token
self._cache_token(cid, token)
return True

def make_login_link(self):
@@ -382,3 +401,35 @@ class AuthManager:
if "id" in session:
self._debug("Logging out session id=%d", session["id"])
self._invalidate_session()

def get_valid_characters(self):
"""Iterate over all valid corp members that we have tokens for.

Each character is returned as a 2-tuple of (char_id, token).

This function may make a large number of API queries (up to three per
character in the corp), hence it is a generator.
"""
chars = g.db.get_authed_characters()
for cid, token, expires, refresh in chars:
seconds_til_expiry = (expires - datetime.utcnow()).total_seconds()

if seconds_til_expiry < self.EXPIRY_THRESHOLD:
result = self._fetch_new_token(refresh, refresh=True)
if not result:
self._debug("Couldn't refresh token for char id=%d", cid)
g.db.drop_auth(cid)
continue

token, expires, refresh, char_id, char_name = result
if char_id != cid:
self._debug("Refreshed token has incorrect char id=%d for "
"char id=%d", char_id, cid)
g.db.drop_auth(cid)
continue

g.db.put_character(cid, char_name)
g.db.update_auth(cid, token, expires, refresh)

if self._is_corp_member(token, cid):
yield cid, token

+ 13
- 0
calefaction/database.py View File

@@ -204,6 +204,19 @@ class Database:
with self._conn as conn:
conn.execute("DELETE FROM auth WHERE auth_character = ?", (cid,))

def get_authed_characters(self):
"""Return a list of characters with authentication info.

Each list item is a 4-tuple of (character_id, access_token,
token_expiry, refresh_token).
"""
query = """SELECT auth_character, auth_token, auth_token_expiry,
auth_refresh FROM auth"""
res = self._conn.execute(query).fetchall()
dtparse = lambda dt: datetime.strptime(dt, "%Y-%m-%d %H:%M:%S")
return [(cid, token, dtparse(expiry), refresh)
for (cid, token, expiry, refresh) in res]

def set_character_modprop(self, cid, module, prop, value):
"""Add or update a character module property."""
with self._conn as conn:


+ 6
- 6
calefaction/eve/esi.py View File

@@ -163,7 +163,7 @@ class EVESwaggerInterface:
self._data_source = "tranquility"
self._cache = _ESICache()

def __call__(self, token):
def __call__(self, token=None):
return _ESIQueryBuilder(self, token)

def _do(self, method, query, params, data, token, can_cache=False):
@@ -183,10 +183,9 @@ class EVESwaggerInterface:
if cached is not None:
return cached

headers = {
"Accept": "application/json",
"Authorization": "Bearer " + token
}
headers = {"Accept": "application/json"}
if token is not None:
headers["Authorization"] = "Bearer " + token
params = params.copy() if params else {}
params["datasource"] = self._data_source
url = self._base_url + query
@@ -198,7 +197,8 @@ class EVESwaggerInterface:
result = resp.json() if resp.content else None
except (requests.RequestException, ValueError) as exc:
self._logger.exception("ESI request failed")
if hasattr(exc, "response") and exc.response.status_code == 403:
if hasattr(exc, "response") and (exc.response and
exc.response.status_code == 403):
raise EVEAPIForbiddenError()
raise EVEAPIError()



+ 47
- 4
calefaction/eve/universe.py View File

@@ -113,6 +113,25 @@ class _Faction(_UniqueObject):
return self._data["name"]


class _Type(_UniqueObject):
"""Represents any type, including ships and materials."""

@property
def name(self):
"""The item's name, as a string."""
return self._data["name"]

@property
def group_id(self):
"""The item's group ID, as an integer."""
return self._data["group_id"]

@property
def market_group_id(self):
"""The item's market group ID, as an integer, or None."""
return self._data.get("market_group_id")


class _Killable(_UniqueObject):
"""Represents a killable object, like a ship, structure, or fighter."""

@@ -186,6 +205,17 @@ class _DummyFaction(_Faction):
})


class _DummyType(_Type):
"""Represents an unknown or invalid type."""

def __init__(self, universe):
super().__init__(universe, -1, {
"name": "Unknown",
"group_id": -1,
"market_group_id": -1
})


class _DummyKillable(_Killable):
"""Represents an unknown or invalid killable object."""

@@ -209,6 +239,7 @@ class Universe:
self._constellations = {}
self._regions = {}
self._factions = {}
self._types = {}
self._killable_idx = {}
self._killable_tab = {}

@@ -237,11 +268,13 @@ class Universe:
self._factions = entities["factions"]
del entities

types = self._load_yaml(self._dir / "types.yml.gz")
self._killable_idx = {kid: cat for cat, kids in types.items()
self._types = self._load_yaml(self._dir / "types.yml.gz")

killables = self._load_yaml(self._dir / "killables.yml.gz")
self._killable_idx = {kid: cat for cat, kids in killables.items()
for kid in kids}
self._killable_tab = types
del types
self._killable_tab = killables
del killables

self._loaded = True

@@ -285,6 +318,16 @@ class Universe:
return _DummyFaction(self)
return _Faction(self, fid, self._factions[fid])

def type(self, tid):
"""Return a _Type with the given ID.

If the ID is invalid, return a dummy unknown object with ID -1.
"""
self._load()
if tid not in self._types:
return _DummyKillable(self)
return _Type(self, tid, self._types[tid])

def killable(self, kid):
"""Return a _Killable with the given ID.



+ 40
- 0
calefaction/modules/campaigns/database.py View File

@@ -181,3 +181,43 @@ class CampaignDB:
},
"value": row[12]
} for row in res]

def update_items(self, campaign, data):
"""Update all item details in the database for the given campaign.

The data should be a multi-layered dictionary. It maps operation names
to a dict that maps character IDs to a dict that maps type IDs to
integer counts.
"""
with self._conn as conn:
cur = conn.execute("BEGIN TRANSACTION")
query = "DELETE FROM oper_item WHERE oi_campaign = ?"
cur.execute(query, (campaign,))

query = """INSERT INTO oper_item (
oi_campaign, oi_operation, oi_character, oi_type, oi_count)
VALUES (?, ?, ?, ?, ?)"""
cur.executemany(query, [
(campaign, operation, int(char_id), int(type_id), int(count))
for operation, chars in data.items()
for char_id, types in chars.items()
for type_id, count in types.items()])

def get_associated_items(self, campaign, operation, limit=5, offset=0):
"""Return a list of items associated with a campaign/operation.

Items are returned as 2-tuples of (item_type, item_count), most recent
first, up to a limit. Use -1 for no limit.
"""
if not isinstance(limit, int):
raise ValueError(limit)
if not isinstance(offset, int):
raise ValueError(offset)

query = """SELECT oi_type, SUM(oi_count) as total_count
FROM oper_item
WHERE oi_campaign = ? AND oi_operation = ?
GROUP BY oi_type ORDER BY total_count DESC LIMIT {} OFFSET {}"""
qform = query.format(limit, offset)
res = self._conn.execute(qform, (campaign, operation)).fetchall()
return [(type_id, count or 0) for type_id, count in res]

+ 2
- 2
calefaction/modules/campaigns/getters.py View File

@@ -62,8 +62,8 @@ def get_summary(cname, opname, limit=5):
kills = g.campaign_db.get_associated_kills(cname, opname, limit=limit)
return kills, "killboard_recent"
elif optype == "collection":
...
return [], None
items = g.campaign_db.get_associated_items(cname, opname, limit=limit)
return items, "collection_items"
else:
raise RuntimeError("Unknown operation type: %s" % optype)



+ 57
- 13
calefaction/modules/campaigns/update.py View File

@@ -6,6 +6,7 @@ import textwrap
from flask import g

from .._provided import config, logger
from ...exceptions import EVEAPIForbiddenError

__all__ = ["update_operation"]

@@ -17,7 +18,7 @@ def _save_operation(cname, opname, primary, secondary, key=None):
g.campaign_db.set_overview(cname, opname, primary, secondary)
g.campaign_db.touch_operation(cname, opname, key=key)

def _build_filter(qualifiers):
def _build_filter(qualifiers, arg):
"""Given a qualifiers string from the config, return a filter function.

This function is extremely sensitive since it executes arbitrary Python
@@ -26,7 +27,7 @@ def _build_filter(qualifiers):
filesystem.
"""
namespace = {"g": g}
body = "def _func(kill):\n" + textwrap.indent(qualifiers, " " * 4)
body = ("def _func(%s):\n" % arg) + textwrap.indent(qualifiers, " " * 4)
exec(body, namespace)
return namespace["_func"]

@@ -52,7 +53,7 @@ def _update_killboard_operations(cname, opnames, min_kill_id):
filters = []
for opname in opnames:
qualif = operations[opname]["qualifiers"]
filters.append((_build_filter(qualif), opname))
filters.append((_build_filter(qualif, "kill"), opname))

args = ["kills", "corporationID", g.config.get("corp.id"), "no-items",
"no-attackers", "orderDirection", "asc"]
@@ -80,21 +81,64 @@ def _update_killboard_operations(cname, opnames, min_kill_id):
secondary = None
_save_operation(cname, opname, primary, secondary, key=max_kill_id)

def _save_collection_overview(cname, opnames, data):
"""Save collection overview data to the database."""
operations = config["campaigns"][cname]["operations"]
if any(operations[opname].get("isk", True) for opname in opnames):
pricelist = g.eve.esi().v1.markets.prices.get()
prices = {entry["type_id"]: entry["average_price"]
for entry in pricelist if "average_price" in entry}
else:
prices = {}

for opname in opnames:
primary = sum(sum(d.values()) for d in data[opname].values())
show_isk = operations[opname].get("isk", True)
if show_isk:
secondary = sum(prices.get(typeid, 0.0) * count
for d in data[opname].values()
for typeid, count in d.items())
else:
secondary = None
_save_operation(cname, opname, primary, secondary)

def _update_collection_operations(cname, opnames):
"""Update all collection-type operations in the given campaign subset."""
campaign = config["campaigns"][cname]
operations = config["campaigns"][cname]["operations"]
filters = []
for opname in opnames:
operation = campaign["operations"][opname]
show_isk = operation.get("isk", True)
qualif = operations[opname]["qualifiers"]
filters.append((_build_filter(qualif, "asset"), opname))

# store per-user counts; update for all users in corp who have fresh
# API keys and leave other data stale
...
primary = __import__("random").randint(10, 99)
secondary = __import__("random").randint(10000000, 5000000000) / 100 \
if show_isk else None
data = {opname: {} for opname in opnames}

_save_operation(cname, opname, primary, secondary)
for char_id, token in g.auth.get_valid_characters():
logger.debug("Fetching assets for char id=%d campaign=%s "
"operations=%s", char_id, cname, ",".join(opnames))
try:
assets = g.eve.esi(token).v1.characters(char_id).assets.get()
except EVEAPIForbiddenError:
logger.debug("Asset access denied for char id=%d", char_id)
continue

for opname in opnames:
data[opname][char_id] = {}

logger.debug("Evaluating %d assets for char id=%d",
len(assets), char_id)
for asset in assets:
for filt, opname in filters:
if filt(asset):
typeid = asset["type_id"]
count = 1 if asset["is_singleton"] else asset["quantity"]
char = data[opname][char_id]
if typeid in char:
char[typeid] += count
else:
char[typeid] = count

g.campaign_db.update_items(cname, data)
_save_collection_overview(cname, opnames, data)

def update_operation(cname, opname, new=False):
"""Update a campaign/operation. Assumes a thread-exclusive lock is held."""


+ 2
- 1
config/modules/campaigns.yml.sample View File

@@ -48,7 +48,8 @@ campaigns:
# Report as "10 units" / "1 unit" of Tritanium
unit: unit|units
qualifiers: |-
return item_type == "Tritanium" # ...
type = g.eve.universe.type(asset["type_id"])
return type.name == "Tritanium"
bar:
title: Save the Bar
operations: []

+ 7
- 3
data/schema_campaigns.sql View File

@@ -56,7 +56,11 @@ DROP TABLE IF EXISTS oper_item;

CREATE TABLE oper_item (
oi_campaign TEXT,
oi_operation TEXT
-- ...
-- UNIQUE (oi_campaign, oi_operation, ...)
oi_operation TEXT,
oi_character INTEGER,
oi_type INTEGER,
oi_count INTEGER,
UNIQUE (oi_campaign, oi_operation, oi_character, oi_type)
);

CREATE INDEX oi_campaign_operation ON oper_item (oi_campaign, oi_operation);

+ 28
- 14
scripts/import_sde.py View File

@@ -66,23 +66,37 @@ def _load_typeids(sde_dir, groups):
assert data[_SOLAR_SYSTEM]["groupID"] == _SOLAR_SYSTEM
assert data[_SOLAR_SYSTEM]["name"]["en"] == "Solar System"

types = {"ships": {}, "structures": {}, "fighters": {}}
types = {}
killables = {"ships": {}, "structures": {}, "fighters": {}}
cat_conv = {_SHIP_CAT: "ships", _FIGHTER_CAT: "fighters"}
cat_conv.update({cid: "structures" for cid in _STRUCT_CATS})
group_conv = {gid: cid for cid, gids in groups.items() for gid in gids}

for tid, type_ in data.items():
name = type_["name"].get("en", "Unknown")
gid = type_["groupID"]

assert isinstance(tid, int)
assert isinstance(gid, int)
assert tid >= 0
assert gid >= 0

types[tid] = {"name": name, "group_id": gid}

if "marketGroupID" in type_:
mgid = type_["marketGroupID"]
assert isinstance(mgid, int)
assert mgid >= 0
types[tid]["market_group_id"] = mgid

if gid in group_conv:
cid = group_conv[gid]
cname = cat_conv[cid]
name = type_["name"]["en"]
group = groups[cid][gid]
assert isinstance(tid, int)
types[cname][tid] = {"name": name, "group": group}
killables[cname][tid] = {"name": name, "group": group}

print("done.")
return types
return types, killables

def _load_ids(sde_dir):
print("Loading itemIDs... ", end="", flush=True)
@@ -248,28 +262,27 @@ def _load_factions(sde_dir):

def _dump_types(out_dir, types):
print("Dumping types... ", end="", flush=True)

_save_yaml(out_dir / "types.yml", types)
print("done.")

def _dump_killables(out_dir, killables):
print("Dumping killables... ", end="", flush=True)
_save_yaml(out_dir / "killables.yml", killables)
print("done.")

def _dump_galaxy(out_dir, galaxy):
print("Dumping galaxy... ", end="", flush=True)

_save_yaml(out_dir / "galaxy.yml", galaxy)

print("done.")

def _dump_entities(out_dir, factions):
print("Dumping entities... ", end="", flush=True)

entities = {"factions": factions}
_save_yaml(out_dir / "entities.yml", entities)

print("done.")

def _compress(out_dir):
targets = ["types", "galaxy", "entities"]
targets = ["types", "killables", "galaxy", "entities"]
for basename in targets:
print("Compressing %s... " % basename, end="", flush=True)

@@ -285,7 +298,7 @@ def _compress(out_dir):
def _cleanup(out_dir):
print("Cleaning up... ", end="", flush=True)

targets = ["types", "galaxy", "entities"]
targets = ["types", "killables", "galaxy", "entities"]
for basename in targets:
(out_dir / (basename + ".yml")).unlink()

@@ -299,9 +312,10 @@ def import_sde(sde_dir, out_dir):

_verify_categoryids(sde_dir)
groups = _load_groupids(sde_dir)
types = _load_typeids(sde_dir, groups)
types, killables = _load_typeids(sde_dir, groups)
_dump_types(out_dir, types)
del groups, types
_dump_killables(out_dir, killables)
del groups, types, killables

ids = _load_ids(sde_dir)
print("Counts: regions=%d, constellations=%d, systems=%d" % (


+ 16
- 7
templates/campaigns/renderers.mako View File

@@ -17,35 +17,35 @@
</a>
</td>
<td class="fluid extra">
<a href="https://zkillboard.com/system/${system.id}/">${system.name}</a> <abbr title="${system.security}" class="${get_security_class(system.security)}">${format_security(system.security)}</abbr><br/>
<a href="https://zkillboard.com/region/${system.region.id}/">${system.region.name}</a>
<a href="https://zkillboard.com/system/${system.id}/">${system.name | h}</a> <abbr title="${system.security}" class="${get_security_class(system.security)}">${format_security(system.security)}</abbr><br/>
<a href="https://zkillboard.com/region/${system.region.id}/">${system.region.name | h}</a>
</td>
<td class="icon">
<a href="https://zkillboard.com/kill/${kill['id']}/">
<img title="Kill ${kill['id']}: ${killed.name}" alt="Kill ${kill['id']}: ${killed.name}" src="${g.eve.image.inventory(victim["ship_id"], 64)}"/>
<img title="Kill ${kill['id']}: ${killed.name | h}" alt="Kill ${kill['id']}: ${killed.name | h}" src="${g.eve.image.inventory(victim["ship_id"], 64)}"/>
</a>
</td>
<td class="icon extra">
<a href="https://zkillboard.com/character/${victim['char_id']}/">
<img title="${victim['char_name']}" alt="${victim['char_name']}" src="${g.eve.image.character(victim["char_id"], 128)}"/>
<img title="${victim['char_name'] | h}" alt="${victim['char_name'] | h}" src="${g.eve.image.character(victim["char_id"], 128)}"/>
</a>
</td>
<td class="icon${' extra' if victim["alliance_id"] and victim["faction_id"] else ''}">
<a href="https://zkillboard.com/corporation/${victim['corp_id']}/">
<img title="${victim['corp_name']}" alt="${victim['corp_name']}" src="${g.eve.image.corp(victim["corp_id"], 128)}"/>
<img title="${victim['corp_name'] | h}" alt="${victim['corp_name'] | h}" src="${g.eve.image.corp(victim["corp_id"], 128)}"/>
</a>
</td>
<td class="icon${'' if victim["alliance_id"] else ' extra'}">
% if victim["alliance_id"]:
<a href="https://zkillboard.com/alliance/${victim['alliance_id']}/">
<img title="${victim['alliance_name']}" alt="${victim['alliance_name']}" src="${g.eve.image.alliance(victim["alliance_id"], 128)}"/>
<img title="${victim['alliance_name'] | h}" alt="${victim['alliance_name'] | h}" src="${g.eve.image.alliance(victim["alliance_id"], 128)}"/>
</a>
% endif
</td>
<td class="icon${'' if victim["faction_id"] else ' extra'}">
% if victim["faction_id"]:
<a href="https://zkillboard.com/faction/${victim['faction_id']}/">
<img title="${victim['faction_name']}" alt="${victim['faction_name']}" src="${g.eve.image.faction(victim["faction_id"], 128)}"/>
<img title="${victim['faction_name'] | h}" alt="${victim['faction_name'] | h}" src="${g.eve.image.faction(victim["faction_id"], 128)}"/>
</a>
% endif
</td>
@@ -64,8 +64,17 @@
</table>
</div>
</%def>
<%def name="_collection_items(summary)">
<div class="head">XXX:</div>
<div class="contents">
${summary | h}
</div>
</%def>

<%def name="render_summary(renderer, summary)"><%
if renderer == "killboard_recent":
return _killboard_recent(summary)
if renderer == "collection_items":
return _collection_items(summary)
raise RuntimeError("Unknown renderer: %s" % renderer)
%></%def>

Loading…
Cancel
Save