Add types to universe data and implement collection campaigns op type.

calefaction/auth.py

@@ -181,19 +181,30 @@ class AuthManager:
         g.db.update_auth(cid, token, expires, refresh)
         return token
 
+    def _is_corp_member(self, token, char_id):
+        """"Return whether the given character is in the site's corp."""
+        resp = self._eve.esi(token).v3.characters(char_id).get()
+        return resp.get("corporation_id") == self._config.get("corp.id")
+
     def _check_access(self, token, char_id):
         """"Check whether the given character is allowed to access this site.
 
         If allowed, do nothing. If not, raise AccessDeniedError.
         """
-        resp = self._eve.esi(token).v3.characters(char_id).get()
-        if resp.get("corporation_id") != self._config.get("corp.id"):
+        if not self._is_corp_member(token, char_id):
             self._debug("Access denied per corp membership for char id=%d "
                         "session id=%d", char_id, session["id"])
             g.db.drop_auth(char_id)
             self._invalidate_session()
             raise AccessDeniedError()
 
+    def _cache_token(self, cid, token):
+        """Cache the given token for this request."""
+        if hasattr(g, "_cached_tokens"):
+            g._cached_tokens[cid] = token
+        else:
+            g._cached_tokens = {cid: token}
+
     def _update_prop_cache(self, module, prop, value):
         """Update the value of a character module property in the cache."""
         if hasattr(g, "_character_modprops"):
@@ -282,21 +293,29 @@ class AuthManager:
         self._update_prop_cache(module, prop, value)
         return True
 
-    def get_token(self):
-        """Return a valid token for the current character, or None.
+    def get_token(self, cid=None):
+        """Return a valid token for the given character.
+
+        If no character is given, we use the current session's character. If a
+        token couldn't be retrieved, return None.
 
-        Assuming this is called in a restricted route (following a True result
-        from is_authenticated), this function makes no API calls and should
-        always succeed. If it is called in other circumstances, it may fail and
-        return None.
+        Assuming we want the current character's token and this is called in a
+        restricted route (following a True result from is_authenticated), this
+        function makes no API calls and should always succeed. If it is called
+        in other circumstances, it may return None or raise EVEAPIError.
         """
-        cid = self.get_character_id()
+        if cid is None:
+            cid = self.get_character_id()
         if not cid:
             return None
 
-        if not hasattr(g, "_cached_token"):
-            g._cached_token = self._get_token(cid)
-        return g._cached_token
+        if hasattr(g, "_cached_tokens"):
+            if cid in g._cached_tokens:
+                return g._cached_tokens[cid]
+
+        token = self._get_token(cid)
+        self._cache_token(cid, token)
+        return token
 
     def is_authenticated(self):
         """Return whether the user has permission to access this site.
@@ -324,7 +343,7 @@ class AuthManager:
         self._debug("Access granted for char id=%d session id=%d", cid,
                     session["id"])
         g.db.touch_session(session["id"])
-        g._cached_token = token
+        self._cache_token(cid, token)
         return True
 
     def make_login_link(self):
@@ -382,3 +401,35 @@ class AuthManager:
         if "id" in session:
             self._debug("Logging out session id=%d", session["id"])
         self._invalidate_session()
+
+    def get_valid_characters(self):
+        """Iterate over all valid corp members that we have tokens for.
+
+        Each character is returned as a 2-tuple of (char_id, token).
+
+        This function may make a large number of API queries (up to three per
+        character in the corp), hence it is a generator.
+        """
+        chars = g.db.get_authed_characters()
+        for cid, token, expires, refresh in chars:
+            seconds_til_expiry = (expires - datetime.utcnow()).total_seconds()
+
+            if seconds_til_expiry < self.EXPIRY_THRESHOLD:
+                result = self._fetch_new_token(refresh, refresh=True)
+                if not result:
+                    self._debug("Couldn't refresh token for char id=%d", cid)
+                    g.db.drop_auth(cid)
+                    continue
+
+                token, expires, refresh, char_id, char_name = result
+                if char_id != cid:
+                    self._debug("Refreshed token has incorrect char id=%d for "
+                                "char id=%d", char_id, cid)
+                    g.db.drop_auth(cid)
+                    continue
+
+                g.db.put_character(cid, char_name)
+                g.db.update_auth(cid, token, expires, refresh)
+
+            if self._is_corp_member(token, cid):
+                yield cid, token

calefaction/database.py

@@ -204,6 +204,19 @@ class Database:
         with self._conn as conn:
             conn.execute("DELETE FROM auth WHERE auth_character = ?", (cid,))
 
+    def get_authed_characters(self):
+        """Return a list of characters with authentication info.
+
+        Each list item is a 4-tuple of (character_id, access_token,
+        token_expiry, refresh_token).
+        """
+        query = """SELECT auth_character, auth_token, auth_token_expiry,
+            auth_refresh FROM auth"""
+        res = self._conn.execute(query).fetchall()
+        dtparse = lambda dt: datetime.strptime(dt, "%Y-%m-%d %H:%M:%S")
+        return [(cid, token, dtparse(expiry), refresh)
+                for (cid, token, expiry, refresh) in res]
+
     def set_character_modprop(self, cid, module, prop, value):
         """Add or update a character module property."""
         with self._conn as conn:

calefaction/eve/esi.py

@@ -163,7 +163,7 @@ class EVESwaggerInterface:
         self._data_source = "tranquility"
         self._cache = _ESICache()
 
-    def __call__(self, token):
+    def __call__(self, token=None):
         return _ESIQueryBuilder(self, token)
 
     def _do(self, method, query, params, data, token, can_cache=False):
@@ -183,10 +183,9 @@ class EVESwaggerInterface:
         if cached is not None:
             return cached
 
-        headers = {
-            "Accept": "application/json",
-            "Authorization": "Bearer " + token
-        }
+        headers = {"Accept": "application/json"}
+        if token is not None:
+            headers["Authorization"] = "Bearer " + token
         params = params.copy() if params else {}
         params["datasource"] = self._data_source
         url = self._base_url + query
@@ -198,7 +197,8 @@ class EVESwaggerInterface:
             result = resp.json() if resp.content else None
         except (requests.RequestException, ValueError) as exc:
             self._logger.exception("ESI request failed")
-            if hasattr(exc, "response") and exc.response.status_code == 403:
+            if hasattr(exc, "response") and (exc.response and
+                                             exc.response.status_code == 403):
                 raise EVEAPIForbiddenError()
             raise EVEAPIError()
 

calefaction/eve/universe.py

@@ -113,6 +113,25 @@ class _Faction(_UniqueObject):
         return self._data["name"]
 
 
+class _Type(_UniqueObject):
+    """Represents any type, including ships and materials."""
+
+    @property
+    def name(self):
+        """The item's name, as a string."""
+        return self._data["name"]
+
+    @property
+    def group_id(self):
+        """The item's group ID, as an integer."""
+        return self._data["group_id"]
+
+    @property
+    def market_group_id(self):
+        """The item's market group ID, as an integer, or None."""
+        return self._data.get("market_group_id")
+
+
 class _Killable(_UniqueObject):
     """Represents a killable object, like a ship, structure, or fighter."""
 
@@ -186,6 +205,17 @@ class _DummyFaction(_Faction):
         })
 
 
+class _DummyType(_Type):
+    """Represents an unknown or invalid type."""
+
+    def __init__(self, universe):
+        super().__init__(universe, -1, {
+            "name": "Unknown",
+            "group_id": -1,
+            "market_group_id": -1
+        })
+
+
 class _DummyKillable(_Killable):
     """Represents an unknown or invalid killable object."""
 
@@ -209,6 +239,7 @@ class Universe:
         self._constellations = {}
         self._regions = {}
         self._factions = {}
+        self._types = {}
         self._killable_idx = {}
         self._killable_tab = {}
 
@@ -237,11 +268,13 @@ class Universe:
             self._factions = entities["factions"]
             del entities
 
-            types = self._load_yaml(self._dir / "types.yml.gz")
-            self._killable_idx = {kid: cat for cat, kids in types.items()
+            self._types = self._load_yaml(self._dir / "types.yml.gz")
+
+            killables = self._load_yaml(self._dir / "killables.yml.gz")
+            self._killable_idx = {kid: cat for cat, kids in killables.items()
                                   for kid in kids}
-            self._killable_tab = types
-            del types
+            self._killable_tab = killables
+            del killables
 
             self._loaded = True
 
@@ -285,6 +318,16 @@ class Universe:
             return _DummyFaction(self)
         return _Faction(self, fid, self._factions[fid])
 
+    def type(self, tid):
+        """Return a _Type with the given ID.
+
+        If the ID is invalid, return a dummy unknown object with ID -1.
+        """
+        self._load()
+        if tid not in self._types:
+            return _DummyKillable(self)
+        return _Type(self, tid, self._types[tid])
+
     def killable(self, kid):
         """Return a _Killable with the given ID.
 

calefaction/modules/campaigns/database.py

@@ -181,3 +181,43 @@ class CampaignDB:
             },
             "value": row[12]
         } for row in res]
+
+    def update_items(self, campaign, data):
+        """Update all item details in the database for the given campaign.
+
+        The data should be a multi-layered dictionary. It maps operation names
+        to a dict that maps character IDs to a dict that maps type IDs to
+        integer counts.
+        """
+        with self._conn as conn:
+            cur = conn.execute("BEGIN TRANSACTION")
+            query = "DELETE FROM oper_item WHERE oi_campaign = ?"
+            cur.execute(query, (campaign,))
+
+            query = """INSERT INTO oper_item (
+                    oi_campaign, oi_operation, oi_character, oi_type, oi_count)
+                VALUES (?, ?, ?, ?, ?)"""
+            cur.executemany(query, [
+                (campaign, operation, int(char_id), int(type_id), int(count))
+                for operation, chars in data.items()
+                for char_id, types in chars.items()
+                for type_id, count in types.items()])
+
+    def get_associated_items(self, campaign, operation, limit=5, offset=0):
+        """Return a list of items associated with a campaign/operation.
+
+        Items are returned as 2-tuples of (item_type, item_count), most recent
+        first, up to a limit. Use -1 for no limit.
+        """
+        if not isinstance(limit, int):
+            raise ValueError(limit)
+        if not isinstance(offset, int):
+            raise ValueError(offset)
+
+        query = """SELECT oi_type, SUM(oi_count) as total_count
+            FROM oper_item
+            WHERE oi_campaign = ? AND oi_operation = ?
+            GROUP BY oi_type ORDER BY total_count DESC LIMIT {} OFFSET {}"""
+        qform = query.format(limit, offset)
+        res = self._conn.execute(qform, (campaign, operation)).fetchall()
+        return [(type_id, count or 0) for type_id, count in res]

calefaction/modules/campaigns/getters.py

@@ -62,8 +62,8 @@ def get_summary(cname, opname, limit=5):
         kills = g.campaign_db.get_associated_kills(cname, opname, limit=limit)
         return kills, "killboard_recent"
     elif optype == "collection":
-        ...
-        return [], None
+        items = g.campaign_db.get_associated_items(cname, opname, limit=limit)
+        return items, "collection_items"
     else:
         raise RuntimeError("Unknown operation type: %s" % optype)
 

calefaction/modules/campaigns/update.py

@@ -6,6 +6,7 @@ import textwrap
 from flask import g
 
 from .._provided import config, logger
+from ...exceptions import EVEAPIForbiddenError
 
 __all__ = ["update_operation"]
 
@@ -17,7 +18,7 @@ def _save_operation(cname, opname, primary, secondary, key=None):
     g.campaign_db.set_overview(cname, opname, primary, secondary)
     g.campaign_db.touch_operation(cname, opname, key=key)
 
-def _build_filter(qualifiers):
+def _build_filter(qualifiers, arg):
     """Given a qualifiers string from the config, return a filter function.
 
     This function is extremely sensitive since it executes arbitrary Python
@@ -26,7 +27,7 @@ def _build_filter(qualifiers):
     filesystem.
     """
     namespace = {"g": g}
-    body = "def _func(kill):\n" + textwrap.indent(qualifiers, " " * 4)
+    body = ("def _func(%s):\n" % arg) + textwrap.indent(qualifiers, " " * 4)
     exec(body, namespace)
     return namespace["_func"]
 
@@ -52,7 +53,7 @@ def _update_killboard_operations(cname, opnames, min_kill_id):
     filters = []
     for opname in opnames:
         qualif = operations[opname]["qualifiers"]
-        filters.append((_build_filter(qualif), opname))
+        filters.append((_build_filter(qualif, "kill"), opname))
 
     args = ["kills", "corporationID", g.config.get("corp.id"), "no-items",
             "no-attackers", "orderDirection", "asc"]
@@ -80,21 +81,64 @@ def _update_killboard_operations(cname, opnames, min_kill_id):
             secondary = None
         _save_operation(cname, opname, primary, secondary, key=max_kill_id)
 
+def _save_collection_overview(cname, opnames, data):
+    """Save collection overview data to the database."""
+    operations = config["campaigns"][cname]["operations"]
+    if any(operations[opname].get("isk", True) for opname in opnames):
+        pricelist = g.eve.esi().v1.markets.prices.get()
+        prices = {entry["type_id"]: entry["average_price"]
+                  for entry in pricelist if "average_price" in entry}
+    else:
+        prices = {}
+
+    for opname in opnames:
+        primary = sum(sum(d.values()) for d in data[opname].values())
+        show_isk = operations[opname].get("isk", True)
+        if show_isk:
+            secondary = sum(prices.get(typeid, 0.0) * count
+                            for d in data[opname].values()
+                            for typeid, count in d.items())
+        else:
+            secondary = None
+        _save_operation(cname, opname, primary, secondary)
+
 def _update_collection_operations(cname, opnames):
     """Update all collection-type operations in the given campaign subset."""
-    campaign = config["campaigns"][cname]
+    operations = config["campaigns"][cname]["operations"]
+    filters = []
     for opname in opnames:
-        operation = campaign["operations"][opname]
-        show_isk = operation.get("isk", True)
+        qualif = operations[opname]["qualifiers"]
+        filters.append((_build_filter(qualif, "asset"), opname))
 
-        # store per-user counts; update for all users in corp who have fresh
-        # API keys and leave other data stale
-        ...
-        primary = __import__("random").randint(10, 99)
-        secondary = __import__("random").randint(10000000, 5000000000) / 100 \
-            if show_isk else None
+    data = {opname: {} for opname in opnames}
 
-        _save_operation(cname, opname, primary, secondary)
+    for char_id, token in g.auth.get_valid_characters():
+        logger.debug("Fetching assets for char id=%d campaign=%s "
+                     "operations=%s", char_id, cname, ",".join(opnames))
+        try:
+            assets = g.eve.esi(token).v1.characters(char_id).assets.get()
+        except EVEAPIForbiddenError:
+            logger.debug("Asset access denied for char id=%d", char_id)
+            continue
+
+        for opname in opnames:
+            data[opname][char_id] = {}
+
+        logger.debug("Evaluating %d assets for char id=%d",
+                     len(assets), char_id)
+        for asset in assets:
+            for filt, opname in filters:
+                if filt(asset):
+                    typeid = asset["type_id"]
+                    count = 1 if asset["is_singleton"] else asset["quantity"]
+                    char = data[opname][char_id]
+                    if typeid in char:
+                        char[typeid] += count
+                    else:
+                        char[typeid] = count
+
+    g.campaign_db.update_items(cname, data)
+    _save_collection_overview(cname, opnames, data)
 
 def update_operation(cname, opname, new=False):
     """Update a campaign/operation. Assumes a thread-exclusive lock is held."""

config/modules/campaigns.yml.sample

@@ -48,7 +48,8 @@ campaigns:
         # Report as "10 units" / "1 unit" of Tritanium
         unit:  unit|units
         qualifiers: |-
-          return item_type == "Tritanium"  # ...
+          type = g.eve.universe.type(asset["type_id"])
+          return type.name == "Tritanium"
   bar:
     title: Save the Bar
     operations: []

data/schema_campaigns.sql

@@ -56,7 +56,11 @@ DROP TABLE IF EXISTS oper_item;
 
 CREATE TABLE oper_item (
     oi_campaign TEXT,
-    oi_operation TEXT
-    -- ...
-    -- UNIQUE (oi_campaign, oi_operation, ...)
+    oi_operation TEXT,
+    oi_character INTEGER,
+    oi_type INTEGER,
+    oi_count INTEGER,
+    UNIQUE (oi_campaign, oi_operation, oi_character, oi_type)
 );
+
+CREATE INDEX oi_campaign_operation ON oper_item (oi_campaign, oi_operation);

scripts/import_sde.py

@@ -66,23 +66,37 @@ def _load_typeids(sde_dir, groups):
     assert data[_SOLAR_SYSTEM]["groupID"] == _SOLAR_SYSTEM
     assert data[_SOLAR_SYSTEM]["name"]["en"] == "Solar System"
 
-    types = {"ships": {}, "structures": {}, "fighters": {}}
+    types = {}
+    killables = {"ships": {}, "structures": {}, "fighters": {}}
     cat_conv = {_SHIP_CAT: "ships", _FIGHTER_CAT: "fighters"}
     cat_conv.update({cid: "structures" for cid in _STRUCT_CATS})
     group_conv = {gid: cid for cid, gids in groups.items() for gid in gids}
 
     for tid, type_ in data.items():
+        name = type_["name"].get("en", "Unknown")
         gid = type_["groupID"]
+
+        assert isinstance(tid, int)
+        assert isinstance(gid, int)
+        assert tid >= 0
+        assert gid >= 0
+
+        types[tid] = {"name": name, "group_id": gid}
+
+        if "marketGroupID" in type_:
+            mgid = type_["marketGroupID"]
+            assert isinstance(mgid, int)
+            assert mgid >= 0
+            types[tid]["market_group_id"] = mgid
+
         if gid in group_conv:
             cid = group_conv[gid]
             cname = cat_conv[cid]
-            name = type_["name"]["en"]
             group = groups[cid][gid]
-            assert isinstance(tid, int)
-            types[cname][tid] = {"name": name, "group": group}
+            killables[cname][tid] = {"name": name, "group": group}
 
     print("done.")
-    return types
+    return types, killables
 
 def _load_ids(sde_dir):
     print("Loading itemIDs... ", end="", flush=True)
@@ -248,28 +262,27 @@ def _load_factions(sde_dir):
 
 def _dump_types(out_dir, types):
     print("Dumping types... ", end="", flush=True)
-
     _save_yaml(out_dir / "types.yml", types)
+    print("done.")
 
+def _dump_killables(out_dir, killables):
+    print("Dumping killables... ", end="", flush=True)
+    _save_yaml(out_dir / "killables.yml", killables)
     print("done.")
 
 def _dump_galaxy(out_dir, galaxy):
     print("Dumping galaxy... ", end="", flush=True)
-
     _save_yaml(out_dir / "galaxy.yml", galaxy)
-
     print("done.")
 
 def _dump_entities(out_dir, factions):
     print("Dumping entities... ", end="", flush=True)
-
     entities = {"factions": factions}
     _save_yaml(out_dir / "entities.yml", entities)
-
     print("done.")
 
 def _compress(out_dir):
-    targets = ["types", "galaxy", "entities"]
+    targets = ["types", "killables", "galaxy", "entities"]
     for basename in targets:
         print("Compressing %s... " % basename, end="", flush=True)
 
@@ -285,7 +298,7 @@ def _compress(out_dir):
 def _cleanup(out_dir):
     print("Cleaning up... ", end="", flush=True)
 
-    targets = ["types", "galaxy", "entities"]
+    targets = ["types", "killables", "galaxy", "entities"]
     for basename in targets:
         (out_dir / (basename + ".yml")).unlink()
 
@@ -299,9 +312,10 @@ def import_sde(sde_dir, out_dir):
 
     _verify_categoryids(sde_dir)
     groups = _load_groupids(sde_dir)
-    types = _load_typeids(sde_dir, groups)
+    types, killables = _load_typeids(sde_dir, groups)
     _dump_types(out_dir, types)
-    del groups, types
+    _dump_killables(out_dir, killables)
+    del groups, types, killables
 
     ids = _load_ids(sde_dir)
     print("Counts: regions=%d, constellations=%d, systems=%d" % (

templates/campaigns/renderers.mako

@@ -17,35 +17,35 @@
             </a>
         </td>
         <td class="fluid extra">
-            <a href="https://zkillboard.com/system/${system.id}/">${system.name}</a> <abbr title="${system.security}" class="${get_security_class(system.security)}">${format_security(system.security)}</abbr><br/>
-            <a href="https://zkillboard.com/region/${system.region.id}/">${system.region.name}</a>
+            <a href="https://zkillboard.com/system/${system.id}/">${system.name | h}</a> <abbr title="${system.security}" class="${get_security_class(system.security)}">${format_security(system.security)}</abbr><br/>
+            <a href="https://zkillboard.com/region/${system.region.id}/">${system.region.name | h}</a>
         </td>
         <td class="icon">
             <a href="https://zkillboard.com/kill/${kill['id']}/">
-                <img title="Kill ${kill['id']}: ${killed.name}" alt="Kill ${kill['id']}: ${killed.name}" src="${g.eve.image.inventory(victim["ship_id"], 64)}"/>
+                <img title="Kill ${kill['id']}: ${killed.name | h}" alt="Kill ${kill['id']}: ${killed.name | h}" src="${g.eve.image.inventory(victim["ship_id"], 64)}"/>
             </a>
         </td>
         <td class="icon extra">
             <a href="https://zkillboard.com/character/${victim['char_id']}/">
-                <img title="${victim['char_name']}" alt="${victim['char_name']}" src="${g.eve.image.character(victim["char_id"], 128)}"/>
+                <img title="${victim['char_name'] | h}" alt="${victim['char_name'] | h}" src="${g.eve.image.character(victim["char_id"], 128)}"/>
             </a>
         </td>
         <td class="icon${' extra' if victim["alliance_id"] and victim["faction_id"] else ''}">
             <a href="https://zkillboard.com/corporation/${victim['corp_id']}/">
-                <img title="${victim['corp_name']}" alt="${victim['corp_name']}" src="${g.eve.image.corp(victim["corp_id"], 128)}"/>
+                <img title="${victim['corp_name'] | h}" alt="${victim['corp_name'] | h}" src="${g.eve.image.corp(victim["corp_id"], 128)}"/>
             </a>
         </td>
         <td class="icon${'' if victim["alliance_id"] else ' extra'}">
         % if victim["alliance_id"]:
             <a href="https://zkillboard.com/alliance/${victim['alliance_id']}/">
-                <img title="${victim['alliance_name']}" alt="${victim['alliance_name']}" src="${g.eve.image.alliance(victim["alliance_id"], 128)}"/>
+                <img title="${victim['alliance_name'] | h}" alt="${victim['alliance_name'] | h}" src="${g.eve.image.alliance(victim["alliance_id"], 128)}"/>
             </a>
         % endif
         </td>
         <td class="icon${'' if victim["faction_id"] else ' extra'}">
         % if victim["faction_id"]:
             <a href="https://zkillboard.com/faction/${victim['faction_id']}/">
-                <img title="${victim['faction_name']}" alt="${victim['faction_name']}" src="${g.eve.image.faction(victim["faction_id"], 128)}"/>
+                <img title="${victim['faction_name'] | h}" alt="${victim['faction_name'] | h}" src="${g.eve.image.faction(victim["faction_id"], 128)}"/>
             </a>
         % endif
         </td>
@@ -64,8 +64,17 @@
         </table>
     </div>
 </%def>
+<%def name="_collection_items(summary)">
+    <div class="head">XXX:</div>
+    <div class="contents">
+        ${summary | h}
+    </div>
+</%def>
+
 <%def name="render_summary(renderer, summary)"><%
     if renderer == "killboard_recent":
         return _killboard_recent(summary)
+    if renderer == "collection_items":
+        return _collection_items(summary)
     raise RuntimeError("Unknown renderer: %s" % renderer)
 %></%def>