diff --git a/calefaction/auth.py b/calefaction/auth.py index 974e46c..eebc085 100644 --- a/calefaction/auth.py +++ b/calefaction/auth.py @@ -181,19 +181,30 @@ class AuthManager: g.db.update_auth(cid, token, expires, refresh) return token + def _is_corp_member(self, token, char_id): + """"Return whether the given character is in the site's corp.""" + resp = self._eve.esi(token).v3.characters(char_id).get() + return resp.get("corporation_id") == self._config.get("corp.id") + def _check_access(self, token, char_id): """"Check whether the given character is allowed to access this site. If allowed, do nothing. If not, raise AccessDeniedError. """ - resp = self._eve.esi(token).v3.characters(char_id).get() - if resp.get("corporation_id") != self._config.get("corp.id"): + if not self._is_corp_member(token, char_id): self._debug("Access denied per corp membership for char id=%d " "session id=%d", char_id, session["id"]) g.db.drop_auth(char_id) self._invalidate_session() raise AccessDeniedError() + def _cache_token(self, cid, token): + """Cache the given token for this request.""" + if hasattr(g, "_cached_tokens"): + g._cached_tokens[cid] = token + else: + g._cached_tokens = {cid: token} + def _update_prop_cache(self, module, prop, value): """Update the value of a character module property in the cache.""" if hasattr(g, "_character_modprops"): @@ -282,21 +293,29 @@ class AuthManager: self._update_prop_cache(module, prop, value) return True - def get_token(self): - """Return a valid token for the current character, or None. + def get_token(self, cid=None): + """Return a valid token for the given character. + + If no character is given, we use the current session's character. If a + token couldn't be retrieved, return None. - Assuming this is called in a restricted route (following a True result - from is_authenticated), this function makes no API calls and should - always succeed. If it is called in other circumstances, it may fail and - return None. + Assuming we want the current character's token and this is called in a + restricted route (following a True result from is_authenticated), this + function makes no API calls and should always succeed. If it is called + in other circumstances, it may return None or raise EVEAPIError. """ - cid = self.get_character_id() + if cid is None: + cid = self.get_character_id() if not cid: return None - if not hasattr(g, "_cached_token"): - g._cached_token = self._get_token(cid) - return g._cached_token + if hasattr(g, "_cached_tokens"): + if cid in g._cached_tokens: + return g._cached_tokens[cid] + + token = self._get_token(cid) + self._cache_token(cid, token) + return token def is_authenticated(self): """Return whether the user has permission to access this site. @@ -324,7 +343,7 @@ class AuthManager: self._debug("Access granted for char id=%d session id=%d", cid, session["id"]) g.db.touch_session(session["id"]) - g._cached_token = token + self._cache_token(cid, token) return True def make_login_link(self): @@ -382,3 +401,35 @@ class AuthManager: if "id" in session: self._debug("Logging out session id=%d", session["id"]) self._invalidate_session() + + def get_valid_characters(self): + """Iterate over all valid corp members that we have tokens for. + + Each character is returned as a 2-tuple of (char_id, token). + + This function may make a large number of API queries (up to three per + character in the corp), hence it is a generator. + """ + chars = g.db.get_authed_characters() + for cid, token, expires, refresh in chars: + seconds_til_expiry = (expires - datetime.utcnow()).total_seconds() + + if seconds_til_expiry < self.EXPIRY_THRESHOLD: + result = self._fetch_new_token(refresh, refresh=True) + if not result: + self._debug("Couldn't refresh token for char id=%d", cid) + g.db.drop_auth(cid) + continue + + token, expires, refresh, char_id, char_name = result + if char_id != cid: + self._debug("Refreshed token has incorrect char id=%d for " + "char id=%d", char_id, cid) + g.db.drop_auth(cid) + continue + + g.db.put_character(cid, char_name) + g.db.update_auth(cid, token, expires, refresh) + + if self._is_corp_member(token, cid): + yield cid, token diff --git a/calefaction/database.py b/calefaction/database.py index 9dd2447..012b2f9 100644 --- a/calefaction/database.py +++ b/calefaction/database.py @@ -204,6 +204,19 @@ class Database: with self._conn as conn: conn.execute("DELETE FROM auth WHERE auth_character = ?", (cid,)) + def get_authed_characters(self): + """Return a list of characters with authentication info. + + Each list item is a 4-tuple of (character_id, access_token, + token_expiry, refresh_token). + """ + query = """SELECT auth_character, auth_token, auth_token_expiry, + auth_refresh FROM auth""" + res = self._conn.execute(query).fetchall() + dtparse = lambda dt: datetime.strptime(dt, "%Y-%m-%d %H:%M:%S") + return [(cid, token, dtparse(expiry), refresh) + for (cid, token, expiry, refresh) in res] + def set_character_modprop(self, cid, module, prop, value): """Add or update a character module property.""" with self._conn as conn: diff --git a/calefaction/eve/esi.py b/calefaction/eve/esi.py index d6209fc..dbd8b9f 100644 --- a/calefaction/eve/esi.py +++ b/calefaction/eve/esi.py @@ -163,7 +163,7 @@ class EVESwaggerInterface: self._data_source = "tranquility" self._cache = _ESICache() - def __call__(self, token): + def __call__(self, token=None): return _ESIQueryBuilder(self, token) def _do(self, method, query, params, data, token, can_cache=False): @@ -183,10 +183,9 @@ class EVESwaggerInterface: if cached is not None: return cached - headers = { - "Accept": "application/json", - "Authorization": "Bearer " + token - } + headers = {"Accept": "application/json"} + if token is not None: + headers["Authorization"] = "Bearer " + token params = params.copy() if params else {} params["datasource"] = self._data_source url = self._base_url + query @@ -198,7 +197,8 @@ class EVESwaggerInterface: result = resp.json() if resp.content else None except (requests.RequestException, ValueError) as exc: self._logger.exception("ESI request failed") - if hasattr(exc, "response") and exc.response.status_code == 403: + if hasattr(exc, "response") and (exc.response and + exc.response.status_code == 403): raise EVEAPIForbiddenError() raise EVEAPIError() diff --git a/calefaction/eve/universe.py b/calefaction/eve/universe.py index f2864a0..0ccb179 100644 --- a/calefaction/eve/universe.py +++ b/calefaction/eve/universe.py @@ -113,6 +113,25 @@ class _Faction(_UniqueObject): return self._data["name"] +class _Type(_UniqueObject): + """Represents any type, including ships and materials.""" + + @property + def name(self): + """The item's name, as a string.""" + return self._data["name"] + + @property + def group_id(self): + """The item's group ID, as an integer.""" + return self._data["group_id"] + + @property + def market_group_id(self): + """The item's market group ID, as an integer, or None.""" + return self._data.get("market_group_id") + + class _Killable(_UniqueObject): """Represents a killable object, like a ship, structure, or fighter.""" @@ -186,6 +205,17 @@ class _DummyFaction(_Faction): }) +class _DummyType(_Type): + """Represents an unknown or invalid type.""" + + def __init__(self, universe): + super().__init__(universe, -1, { + "name": "Unknown", + "group_id": -1, + "market_group_id": -1 + }) + + class _DummyKillable(_Killable): """Represents an unknown or invalid killable object.""" @@ -209,6 +239,7 @@ class Universe: self._constellations = {} self._regions = {} self._factions = {} + self._types = {} self._killable_idx = {} self._killable_tab = {} @@ -237,11 +268,13 @@ class Universe: self._factions = entities["factions"] del entities - types = self._load_yaml(self._dir / "types.yml.gz") - self._killable_idx = {kid: cat for cat, kids in types.items() + self._types = self._load_yaml(self._dir / "types.yml.gz") + + killables = self._load_yaml(self._dir / "killables.yml.gz") + self._killable_idx = {kid: cat for cat, kids in killables.items() for kid in kids} - self._killable_tab = types - del types + self._killable_tab = killables + del killables self._loaded = True @@ -285,6 +318,16 @@ class Universe: return _DummyFaction(self) return _Faction(self, fid, self._factions[fid]) + def type(self, tid): + """Return a _Type with the given ID. + + If the ID is invalid, return a dummy unknown object with ID -1. + """ + self._load() + if tid not in self._types: + return _DummyKillable(self) + return _Type(self, tid, self._types[tid]) + def killable(self, kid): """Return a _Killable with the given ID. diff --git a/calefaction/modules/campaigns/database.py b/calefaction/modules/campaigns/database.py index 3018c5e..44cf9a3 100644 --- a/calefaction/modules/campaigns/database.py +++ b/calefaction/modules/campaigns/database.py @@ -181,3 +181,43 @@ class CampaignDB: }, "value": row[12] } for row in res] + + def update_items(self, campaign, data): + """Update all item details in the database for the given campaign. + + The data should be a multi-layered dictionary. It maps operation names + to a dict that maps character IDs to a dict that maps type IDs to + integer counts. + """ + with self._conn as conn: + cur = conn.execute("BEGIN TRANSACTION") + query = "DELETE FROM oper_item WHERE oi_campaign = ?" + cur.execute(query, (campaign,)) + + query = """INSERT INTO oper_item ( + oi_campaign, oi_operation, oi_character, oi_type, oi_count) + VALUES (?, ?, ?, ?, ?)""" + cur.executemany(query, [ + (campaign, operation, int(char_id), int(type_id), int(count)) + for operation, chars in data.items() + for char_id, types in chars.items() + for type_id, count in types.items()]) + + def get_associated_items(self, campaign, operation, limit=5, offset=0): + """Return a list of items associated with a campaign/operation. + + Items are returned as 2-tuples of (item_type, item_count), most recent + first, up to a limit. Use -1 for no limit. + """ + if not isinstance(limit, int): + raise ValueError(limit) + if not isinstance(offset, int): + raise ValueError(offset) + + query = """SELECT oi_type, SUM(oi_count) as total_count + FROM oper_item + WHERE oi_campaign = ? AND oi_operation = ? + GROUP BY oi_type ORDER BY total_count DESC LIMIT {} OFFSET {}""" + qform = query.format(limit, offset) + res = self._conn.execute(qform, (campaign, operation)).fetchall() + return [(type_id, count or 0) for type_id, count in res] diff --git a/calefaction/modules/campaigns/getters.py b/calefaction/modules/campaigns/getters.py index 9f22d80..5201375 100644 --- a/calefaction/modules/campaigns/getters.py +++ b/calefaction/modules/campaigns/getters.py @@ -62,8 +62,8 @@ def get_summary(cname, opname, limit=5): kills = g.campaign_db.get_associated_kills(cname, opname, limit=limit) return kills, "killboard_recent" elif optype == "collection": - ... - return [], None + items = g.campaign_db.get_associated_items(cname, opname, limit=limit) + return items, "collection_items" else: raise RuntimeError("Unknown operation type: %s" % optype) diff --git a/calefaction/modules/campaigns/update.py b/calefaction/modules/campaigns/update.py index 1b339de..27c3f6f 100644 --- a/calefaction/modules/campaigns/update.py +++ b/calefaction/modules/campaigns/update.py @@ -6,6 +6,7 @@ import textwrap from flask import g from .._provided import config, logger +from ...exceptions import EVEAPIForbiddenError __all__ = ["update_operation"] @@ -17,7 +18,7 @@ def _save_operation(cname, opname, primary, secondary, key=None): g.campaign_db.set_overview(cname, opname, primary, secondary) g.campaign_db.touch_operation(cname, opname, key=key) -def _build_filter(qualifiers): +def _build_filter(qualifiers, arg): """Given a qualifiers string from the config, return a filter function. This function is extremely sensitive since it executes arbitrary Python @@ -26,7 +27,7 @@ def _build_filter(qualifiers): filesystem. """ namespace = {"g": g} - body = "def _func(kill):\n" + textwrap.indent(qualifiers, " " * 4) + body = ("def _func(%s):\n" % arg) + textwrap.indent(qualifiers, " " * 4) exec(body, namespace) return namespace["_func"] @@ -52,7 +53,7 @@ def _update_killboard_operations(cname, opnames, min_kill_id): filters = [] for opname in opnames: qualif = operations[opname]["qualifiers"] - filters.append((_build_filter(qualif), opname)) + filters.append((_build_filter(qualif, "kill"), opname)) args = ["kills", "corporationID", g.config.get("corp.id"), "no-items", "no-attackers", "orderDirection", "asc"] @@ -80,21 +81,64 @@ def _update_killboard_operations(cname, opnames, min_kill_id): secondary = None _save_operation(cname, opname, primary, secondary, key=max_kill_id) +def _save_collection_overview(cname, opnames, data): + """Save collection overview data to the database.""" + operations = config["campaigns"][cname]["operations"] + if any(operations[opname].get("isk", True) for opname in opnames): + pricelist = g.eve.esi().v1.markets.prices.get() + prices = {entry["type_id"]: entry["average_price"] + for entry in pricelist if "average_price" in entry} + else: + prices = {} + + for opname in opnames: + primary = sum(sum(d.values()) for d in data[opname].values()) + show_isk = operations[opname].get("isk", True) + if show_isk: + secondary = sum(prices.get(typeid, 0.0) * count + for d in data[opname].values() + for typeid, count in d.items()) + else: + secondary = None + _save_operation(cname, opname, primary, secondary) + def _update_collection_operations(cname, opnames): """Update all collection-type operations in the given campaign subset.""" - campaign = config["campaigns"][cname] + operations = config["campaigns"][cname]["operations"] + filters = [] for opname in opnames: - operation = campaign["operations"][opname] - show_isk = operation.get("isk", True) + qualif = operations[opname]["qualifiers"] + filters.append((_build_filter(qualif, "asset"), opname)) - # store per-user counts; update for all users in corp who have fresh - # API keys and leave other data stale - ... - primary = __import__("random").randint(10, 99) - secondary = __import__("random").randint(10000000, 5000000000) / 100 \ - if show_isk else None + data = {opname: {} for opname in opnames} - _save_operation(cname, opname, primary, secondary) + for char_id, token in g.auth.get_valid_characters(): + logger.debug("Fetching assets for char id=%d campaign=%s " + "operations=%s", char_id, cname, ",".join(opnames)) + try: + assets = g.eve.esi(token).v1.characters(char_id).assets.get() + except EVEAPIForbiddenError: + logger.debug("Asset access denied for char id=%d", char_id) + continue + + for opname in opnames: + data[opname][char_id] = {} + + logger.debug("Evaluating %d assets for char id=%d", + len(assets), char_id) + for asset in assets: + for filt, opname in filters: + if filt(asset): + typeid = asset["type_id"] + count = 1 if asset["is_singleton"] else asset["quantity"] + char = data[opname][char_id] + if typeid in char: + char[typeid] += count + else: + char[typeid] = count + + g.campaign_db.update_items(cname, data) + _save_collection_overview(cname, opnames, data) def update_operation(cname, opname, new=False): """Update a campaign/operation. Assumes a thread-exclusive lock is held.""" diff --git a/config/modules/campaigns.yml.sample b/config/modules/campaigns.yml.sample index 1d3c609..01ea6c8 100644 --- a/config/modules/campaigns.yml.sample +++ b/config/modules/campaigns.yml.sample @@ -48,7 +48,8 @@ campaigns: # Report as "10 units" / "1 unit" of Tritanium unit: unit|units qualifiers: |- - return item_type == "Tritanium" # ... + type = g.eve.universe.type(asset["type_id"]) + return type.name == "Tritanium" bar: title: Save the Bar operations: [] diff --git a/data/schema_campaigns.sql b/data/schema_campaigns.sql index f9310f1..56d8fc7 100644 --- a/data/schema_campaigns.sql +++ b/data/schema_campaigns.sql @@ -56,7 +56,11 @@ DROP TABLE IF EXISTS oper_item; CREATE TABLE oper_item ( oi_campaign TEXT, - oi_operation TEXT - -- ... - -- UNIQUE (oi_campaign, oi_operation, ...) + oi_operation TEXT, + oi_character INTEGER, + oi_type INTEGER, + oi_count INTEGER, + UNIQUE (oi_campaign, oi_operation, oi_character, oi_type) ); + +CREATE INDEX oi_campaign_operation ON oper_item (oi_campaign, oi_operation); diff --git a/scripts/import_sde.py b/scripts/import_sde.py index c5f6a85..96485b1 100755 --- a/scripts/import_sde.py +++ b/scripts/import_sde.py @@ -66,23 +66,37 @@ def _load_typeids(sde_dir, groups): assert data[_SOLAR_SYSTEM]["groupID"] == _SOLAR_SYSTEM assert data[_SOLAR_SYSTEM]["name"]["en"] == "Solar System" - types = {"ships": {}, "structures": {}, "fighters": {}} + types = {} + killables = {"ships": {}, "structures": {}, "fighters": {}} cat_conv = {_SHIP_CAT: "ships", _FIGHTER_CAT: "fighters"} cat_conv.update({cid: "structures" for cid in _STRUCT_CATS}) group_conv = {gid: cid for cid, gids in groups.items() for gid in gids} for tid, type_ in data.items(): + name = type_["name"].get("en", "Unknown") gid = type_["groupID"] + + assert isinstance(tid, int) + assert isinstance(gid, int) + assert tid >= 0 + assert gid >= 0 + + types[tid] = {"name": name, "group_id": gid} + + if "marketGroupID" in type_: + mgid = type_["marketGroupID"] + assert isinstance(mgid, int) + assert mgid >= 0 + types[tid]["market_group_id"] = mgid + if gid in group_conv: cid = group_conv[gid] cname = cat_conv[cid] - name = type_["name"]["en"] group = groups[cid][gid] - assert isinstance(tid, int) - types[cname][tid] = {"name": name, "group": group} + killables[cname][tid] = {"name": name, "group": group} print("done.") - return types + return types, killables def _load_ids(sde_dir): print("Loading itemIDs... ", end="", flush=True) @@ -248,28 +262,27 @@ def _load_factions(sde_dir): def _dump_types(out_dir, types): print("Dumping types... ", end="", flush=True) - _save_yaml(out_dir / "types.yml", types) + print("done.") +def _dump_killables(out_dir, killables): + print("Dumping killables... ", end="", flush=True) + _save_yaml(out_dir / "killables.yml", killables) print("done.") def _dump_galaxy(out_dir, galaxy): print("Dumping galaxy... ", end="", flush=True) - _save_yaml(out_dir / "galaxy.yml", galaxy) - print("done.") def _dump_entities(out_dir, factions): print("Dumping entities... ", end="", flush=True) - entities = {"factions": factions} _save_yaml(out_dir / "entities.yml", entities) - print("done.") def _compress(out_dir): - targets = ["types", "galaxy", "entities"] + targets = ["types", "killables", "galaxy", "entities"] for basename in targets: print("Compressing %s... " % basename, end="", flush=True) @@ -285,7 +298,7 @@ def _compress(out_dir): def _cleanup(out_dir): print("Cleaning up... ", end="", flush=True) - targets = ["types", "galaxy", "entities"] + targets = ["types", "killables", "galaxy", "entities"] for basename in targets: (out_dir / (basename + ".yml")).unlink() @@ -299,9 +312,10 @@ def import_sde(sde_dir, out_dir): _verify_categoryids(sde_dir) groups = _load_groupids(sde_dir) - types = _load_typeids(sde_dir, groups) + types, killables = _load_typeids(sde_dir, groups) _dump_types(out_dir, types) - del groups, types + _dump_killables(out_dir, killables) + del groups, types, killables ids = _load_ids(sde_dir) print("Counts: regions=%d, constellations=%d, systems=%d" % ( diff --git a/templates/campaigns/renderers.mako b/templates/campaigns/renderers.mako index 19885b4..7fcbb13 100644 --- a/templates/campaigns/renderers.mako +++ b/templates/campaigns/renderers.mako @@ -17,35 +17,35 @@ - ${system.name} ${format_security(system.security)}
- ${system.region.name} + ${system.name | h} ${format_security(system.security)}
+ ${system.region.name | h} - Kill ${kill['id']}: ${killed.name} + Kill ${kill['id']}: ${killed.name | h} - ${victim['char_name']} + ${victim['char_name'] | h} - ${victim['corp_name']} + ${victim['corp_name'] | h} % if victim["alliance_id"]: - ${victim['alliance_name']} + ${victim['alliance_name'] | h} % endif % if victim["faction_id"]: - ${victim['faction_name']} + ${victim['faction_name'] | h} % endif @@ -64,8 +64,17 @@ +<%def name="_collection_items(summary)"> +
XXX:
+
+ ${summary | h} +
+ + <%def name="render_summary(renderer, summary)"><% if renderer == "killboard_recent": return _killboard_recent(summary) + if renderer == "collection_items": + return _collection_items(summary) raise RuntimeError("Unknown renderer: %s" % renderer) %>