From 0e9949b31bb6630b6105178648301f0b93a08174 Mon Sep 17 00:00:00 2001 From: Ben Kurtovic Date: Tue, 20 Dec 2016 03:13:15 -0500 Subject: [PATCH] Add caching to ESI and clean up. --- calefaction/eve/esi.py | 131 +++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 111 insertions(+), 20 deletions(-) diff --git a/calefaction/eve/esi.py b/calefaction/eve/esi.py index e47dcdd..625cea9 100644 --- a/calefaction/eve/esi.py +++ b/calefaction/eve/esi.py @@ -1,6 +1,8 @@ # -*- coding: utf-8 -*- -from urllib.parse import urlencode +from datetime import datetime +import random +from threading import Lock import requests @@ -8,6 +10,79 @@ from ..exceptions import EVEAPIError __all__ = ["EVESwaggerInterface"] +class _ESICache: + """Caches ESI API responses according to their headers. + + This interface is thread-safe. + """ + EXPIRATION_RATE = 0.2 + + def __init__(self): + self._index = {} + self._lock = Lock() + + @staticmethod + def _calculate_expiry(resp): + """Calculate the expiration date for the given response object.""" + if "Cache-Control" not in resp.headers: + return None + + directives = resp.headers["Cache-Control"].lower().split(";") + directives = [d.strip() for d in directives] + if "public" not in directives: + return None + + expires = resp.headers.get("Expires") + if not expires: + return None + + try: + return datetime.strptime(expires, "%a, %d %b %Y %H:%M:%S GMT") + except ValueError: + return None + + @staticmethod + def freeze_dict(d): + """Return a hashable string key for the given dictionary.""" + if not d: + return "{}" + items = sorted((repr(str(k)), repr(str(v))) for k, v in d.items()) + return "{" + ",".join(":".join(p) for p in items) + "}" + + def _expire(self): + """Remove old entries from the cache. Assumes lock is acquired.""" + condemned = [] + now = datetime.utcnow() + for key, (expiry, _) in self._index.items(): + if expiry < now: + condemned.append(key) + for key in condemned: + del self._index[key] + + def fetch(self, key): + """Try to look up a key in the cache. Return None if not found. + + The key should be a string. + + This will periodically clear the cache of expired entries. + """ + with self._lock: + if random.random() < self.EXPIRATION_RATE: + self._expire() + if key in self._index: + expiry, value = self._index[key] + if expiry > datetime.utcnow(): + return value + return None + + def insert(self, key, value, response): + """Store a key-value pair using the response as cache control.""" + expiry = self._calculate_expiry(response) + if expiry and expiry > datetime.utcnow(): + with self._lock: + self._index[key] = (expiry, value) + + class _ESIQueryBuilder: """Stores an ESI query that is being built by the client.""" @@ -24,9 +99,9 @@ class _ESIQueryBuilder: self._path += str(item) + "/" return self - def get(self): + def get(self, **kwargs): """Do an HTTP GET request for the built query.""" - return self._esi.get(self._path, self._token) + return self._esi.get(self._path, self._token, params=kwargs) def post(self, **kwargs): """Do an HTTP POST request for the built query.""" @@ -36,9 +111,9 @@ class _ESIQueryBuilder: """Do an HTTP PUT request for the built query.""" return self._esi.put(self._path, self._token, data=kwargs) - def delete(self): + def delete(self, **kwargs): """Do an HTTP DELETE request for the built query.""" - return self._esi.delete(self._path, self._token) + return self._esi.delete(self._path, self._token, params=kwargs) class EVESwaggerInterface: @@ -51,7 +126,7 @@ class EVESwaggerInterface: For more complex requests: - data = esi.post("/v1/universe/names/", token, {"ids": [entity_id]}) + data = esi.post("/v1/universe/names/", token, data={"ids": [entity_id]}) data = esi(token).v1.universe.names.post(ids=[entity_id]}) """ @@ -59,43 +134,59 @@ class EVESwaggerInterface: self._session = session self._base_url = "https://esi.tech.ccp.is" self._data_source = "tranquility" + self._cache = _ESICache() def __call__(self, token): return _ESIQueryBuilder(self, token) - def _do(self, query, data, token, method): + def _do(self, method, query, params, data, token, can_cache=False): """Execute a query using a token with the given session method. Return the JSON result, if any. Raise EVEAPIError for any errors. """ - ... # cache requests + if can_cache: + pkey = self._cache.freeze_dict(params) + key = "|".join((method.__name__, self._data_source, query, pkey)) + cached = self._cache.fetch(key) + if cached: + return cached - params = {"datasource": self._data_source} headers = { "Accept": "application/json", "Authorization": "Bearer " + token } - url = self._base_url + query + "?" + urlencode(params) + params = params.copy() if params else {} + params["datasource"] = self._data_source + url = self._base_url + query try: - resp = method(url, json=data or None, timeout=10, headers=headers) + resp = method(url, params=params, json=data or None, + headers=headers, timeout=10) resp.raise_for_status() - return resp.json() if resp.content else None + result = resp.json() if resp.content else None except (requests.RequestException, ValueError) as exc: raise EVEAPIError(str(exc)) - def get(self, query, token): + if can_cache and result is not None: + self._cache.insert(key, result, resp) + return result + + def get(self, query, token, params=None): """Do an HTTP GET request for a query using a token.""" - return self._do(query, None, token, self._session.get) + meth = self._session.get + return self._do(meth, query, params, None, token, True) - def post(self, query, token, data=None): + def post(self, query, token, params=None, data=None): """Do an HTTP POST request for a query using a token.""" - return self._do(query, data, token, self._session.post) + meth = self._session.post + return self._do(meth, query, params, data, token) - def put(self, query, token, data=None): + def put(self, query, token, params=None, data=None): """Do an HTTP PUT request for a query using a token.""" - return self._do(query, data, token, self._session.put) + meth = self._session.put + return self._do(meth, query, params, data, token) - def delete(self, query, token): + def delete(self, query, token, params=None): """Do an HTTP DELETE request for a query using a token.""" - return self._do(query, None, token, self._session.delete) + meth = self._session.delete + return self._do(meth, query, params, None, token)