Add caching to ESI and clean up.

calefaction/eve/esi.py

@@ -1,6 +1,8 @@
 # -*- coding: utf-8  -*-
 
-from urllib.parse import urlencode
+from datetime import datetime
+import random
+from threading import Lock
 
 import requests
 
@@ -8,6 +10,79 @@ from ..exceptions import EVEAPIError
 
 __all__ = ["EVESwaggerInterface"]
 
+class _ESICache:
+    """Caches ESI API responses according to their headers.
+
+    This interface is thread-safe.
+    """
+    EXPIRATION_RATE = 0.2
+
+    def __init__(self):
+        self._index = {}
+        self._lock = Lock()
+
+    @staticmethod
+    def _calculate_expiry(resp):
+        """Calculate the expiration date for the given response object."""
+        if "Cache-Control" not in resp.headers:
+            return None
+
+        directives = resp.headers["Cache-Control"].lower().split(";")
+        directives = [d.strip() for d in directives]
+        if "public" not in directives:
+            return None
+
+        expires = resp.headers.get("Expires")
+        if not expires:
+            return None
+
+        try:
+            return datetime.strptime(expires, "%a, %d %b %Y %H:%M:%S GMT")
+        except ValueError:
+            return None
+
+    @staticmethod
+    def freeze_dict(d):
+        """Return a hashable string key for the given dictionary."""
+        if not d:
+            return "{}"
+        items = sorted((repr(str(k)), repr(str(v))) for k, v in d.items())
+        return "{" + ",".join(":".join(p) for p in items) + "}"
+
+    def _expire(self):
+        """Remove old entries from the cache. Assumes lock is acquired."""
+        condemned = []
+        now = datetime.utcnow()
+        for key, (expiry, _) in self._index.items():
+            if expiry < now:
+                condemned.append(key)
+        for key in condemned:
+            del self._index[key]
+
+    def fetch(self, key):
+        """Try to look up a key in the cache. Return None if not found.
+
+        The key should be a string.
+
+        This will periodically clear the cache of expired entries.
+        """
+        with self._lock:
+            if random.random() < self.EXPIRATION_RATE:
+                self._expire()
+            if key in self._index:
+                expiry, value = self._index[key]
+                if expiry > datetime.utcnow():
+                    return value
+            return None
+
+    def insert(self, key, value, response):
+        """Store a key-value pair using the response as cache control."""
+        expiry = self._calculate_expiry(response)
+        if expiry and expiry > datetime.utcnow():
+            with self._lock:
+                self._index[key] = (expiry, value)
+
+
 class _ESIQueryBuilder:
     """Stores an ESI query that is being built by the client."""
 
@@ -24,9 +99,9 @@ class _ESIQueryBuilder:
         self._path += str(item) + "/"
         return self
 
-    def get(self):
+    def get(self, **kwargs):
         """Do an HTTP GET request for the built query."""
-        return self._esi.get(self._path, self._token)
+        return self._esi.get(self._path, self._token, params=kwargs)
 
     def post(self, **kwargs):
         """Do an HTTP POST request for the built query."""
@@ -36,9 +111,9 @@ class _ESIQueryBuilder:
         """Do an HTTP PUT request for the built query."""
         return self._esi.put(self._path, self._token, data=kwargs)
 
-    def delete(self):
+    def delete(self, **kwargs):
         """Do an HTTP DELETE request for the built query."""
-        return self._esi.delete(self._path, self._token)
+        return self._esi.delete(self._path, self._token, params=kwargs)
 
 
 class EVESwaggerInterface:
@@ -51,7 +126,7 @@ class EVESwaggerInterface:
 
     For more complex requests:
 
-    data = esi.post("/v1/universe/names/", token, {"ids": [entity_id]})
+    data = esi.post("/v1/universe/names/", token, data={"ids": [entity_id]})
     data = esi(token).v1.universe.names.post(ids=[entity_id]})
     """
 
@@ -59,43 +134,59 @@ class EVESwaggerInterface:
         self._session = session
         self._base_url = "https://esi.tech.ccp.is"
         self._data_source = "tranquility"
+        self._cache = _ESICache()
 
     def __call__(self, token):
         return _ESIQueryBuilder(self, token)
 
-    def _do(self, query, data, token, method):
+    def _do(self, method, query, params, data, token, can_cache=False):
         """Execute a query using a token with the given session method.
 
         Return the JSON result, if any. Raise EVEAPIError for any errors.
         """
-        ...  # cache requests
+        if can_cache:
+            pkey = self._cache.freeze_dict(params)
+            key = "|".join((method.__name__, self._data_source, query, pkey))
+            cached = self._cache.fetch(key)
+            if cached:
+                return cached
 
-        params = {"datasource": self._data_source}
         headers = {
             "Accept": "application/json",
             "Authorization": "Bearer " + token
         }
-        url = self._base_url + query + "?" + urlencode(params)
+        params = params.copy() if params else {}
+        params["datasource"] = self._data_source
+        url = self._base_url + query
 
         try:
-            resp = method(url, json=data or None, timeout=10, headers=headers)
+            resp = method(url, params=params, json=data or None,
+                          headers=headers, timeout=10)
             resp.raise_for_status()
-            return resp.json() if resp.content else None
+            result = resp.json() if resp.content else None
         except (requests.RequestException, ValueError) as exc:
             raise EVEAPIError(str(exc))
 
-    def get(self, query, token):
+        if can_cache and result is not None:
+            self._cache.insert(key, result, resp)
+        return result
+
+    def get(self, query, token, params=None):
         """Do an HTTP GET request for a query using a token."""
-        return self._do(query, None, token, self._session.get)
+        meth = self._session.get
+        return self._do(meth, query, params, None, token, True)
 
-    def post(self, query, token, data=None):
+    def post(self, query, token, params=None, data=None):
         """Do an HTTP POST request for a query using a token."""
-        return self._do(query, data, token, self._session.post)
+        meth = self._session.post
+        return self._do(meth, query, params, data, token)
 
-    def put(self, query, token, data=None):
+    def put(self, query, token, params=None, data=None):
         """Do an HTTP PUT request for a query using a token."""
-        return self._do(query, data, token, self._session.put)
+        meth = self._session.put
+        return self._do(meth, query, params, data, token)
 
-    def delete(self, query, token):
+    def delete(self, query, token, params=None):
         """Do an HTTP DELETE request for a query using a token."""
-        return self._do(query, None, token, self._session.delete)
+        meth = self._session.delete
+        return self._do(meth, query, params, None, token)