Browse Source

Add caching to ESI and clean up.

master
Ben Kurtovic 7 years ago
parent
commit
0e9949b31b
1 changed files with 111 additions and 20 deletions
  1. +111
    -20
      calefaction/eve/esi.py

+ 111
- 20
calefaction/eve/esi.py View File

@@ -1,6 +1,8 @@
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-


from urllib.parse import urlencode
from datetime import datetime
import random
from threading import Lock


import requests import requests


@@ -8,6 +10,79 @@ from ..exceptions import EVEAPIError


__all__ = ["EVESwaggerInterface"] __all__ = ["EVESwaggerInterface"]


class _ESICache:
"""Caches ESI API responses according to their headers.

This interface is thread-safe.
"""
EXPIRATION_RATE = 0.2

def __init__(self):
self._index = {}
self._lock = Lock()

@staticmethod
def _calculate_expiry(resp):
"""Calculate the expiration date for the given response object."""
if "Cache-Control" not in resp.headers:
return None

directives = resp.headers["Cache-Control"].lower().split(";")
directives = [d.strip() for d in directives]
if "public" not in directives:
return None

expires = resp.headers.get("Expires")
if not expires:
return None

try:
return datetime.strptime(expires, "%a, %d %b %Y %H:%M:%S GMT")
except ValueError:
return None

@staticmethod
def freeze_dict(d):
"""Return a hashable string key for the given dictionary."""
if not d:
return "{}"
items = sorted((repr(str(k)), repr(str(v))) for k, v in d.items())
return "{" + ",".join(":".join(p) for p in items) + "}"

def _expire(self):
"""Remove old entries from the cache. Assumes lock is acquired."""
condemned = []
now = datetime.utcnow()
for key, (expiry, _) in self._index.items():
if expiry < now:
condemned.append(key)
for key in condemned:
del self._index[key]

def fetch(self, key):
"""Try to look up a key in the cache. Return None if not found.

The key should be a string.

This will periodically clear the cache of expired entries.
"""
with self._lock:
if random.random() < self.EXPIRATION_RATE:
self._expire()
if key in self._index:
expiry, value = self._index[key]
if expiry > datetime.utcnow():
return value
return None

def insert(self, key, value, response):
"""Store a key-value pair using the response as cache control."""
expiry = self._calculate_expiry(response)
if expiry and expiry > datetime.utcnow():
with self._lock:
self._index[key] = (expiry, value)


class _ESIQueryBuilder: class _ESIQueryBuilder:
"""Stores an ESI query that is being built by the client.""" """Stores an ESI query that is being built by the client."""


@@ -24,9 +99,9 @@ class _ESIQueryBuilder:
self._path += str(item) + "/" self._path += str(item) + "/"
return self return self


def get(self):
def get(self, **kwargs):
"""Do an HTTP GET request for the built query.""" """Do an HTTP GET request for the built query."""
return self._esi.get(self._path, self._token)
return self._esi.get(self._path, self._token, params=kwargs)


def post(self, **kwargs): def post(self, **kwargs):
"""Do an HTTP POST request for the built query.""" """Do an HTTP POST request for the built query."""
@@ -36,9 +111,9 @@ class _ESIQueryBuilder:
"""Do an HTTP PUT request for the built query.""" """Do an HTTP PUT request for the built query."""
return self._esi.put(self._path, self._token, data=kwargs) return self._esi.put(self._path, self._token, data=kwargs)


def delete(self):
def delete(self, **kwargs):
"""Do an HTTP DELETE request for the built query.""" """Do an HTTP DELETE request for the built query."""
return self._esi.delete(self._path, self._token)
return self._esi.delete(self._path, self._token, params=kwargs)




class EVESwaggerInterface: class EVESwaggerInterface:
@@ -51,7 +126,7 @@ class EVESwaggerInterface:


For more complex requests: For more complex requests:


data = esi.post("/v1/universe/names/", token, {"ids": [entity_id]})
data = esi.post("/v1/universe/names/", token, data={"ids": [entity_id]})
data = esi(token).v1.universe.names.post(ids=[entity_id]}) data = esi(token).v1.universe.names.post(ids=[entity_id]})
""" """


@@ -59,43 +134,59 @@ class EVESwaggerInterface:
self._session = session self._session = session
self._base_url = "https://esi.tech.ccp.is" self._base_url = "https://esi.tech.ccp.is"
self._data_source = "tranquility" self._data_source = "tranquility"
self._cache = _ESICache()


def __call__(self, token): def __call__(self, token):
return _ESIQueryBuilder(self, token) return _ESIQueryBuilder(self, token)


def _do(self, query, data, token, method):
def _do(self, method, query, params, data, token, can_cache=False):
"""Execute a query using a token with the given session method. """Execute a query using a token with the given session method.


Return the JSON result, if any. Raise EVEAPIError for any errors. Return the JSON result, if any. Raise EVEAPIError for any errors.
""" """
... # cache requests
if can_cache:
pkey = self._cache.freeze_dict(params)
key = "|".join((method.__name__, self._data_source, query, pkey))
cached = self._cache.fetch(key)
if cached:
return cached


params = {"datasource": self._data_source}
headers = { headers = {
"Accept": "application/json", "Accept": "application/json",
"Authorization": "Bearer " + token "Authorization": "Bearer " + token
} }
url = self._base_url + query + "?" + urlencode(params)
params = params.copy() if params else {}
params["datasource"] = self._data_source
url = self._base_url + query


try: try:
resp = method(url, json=data or None, timeout=10, headers=headers)
resp = method(url, params=params, json=data or None,
headers=headers, timeout=10)
resp.raise_for_status() resp.raise_for_status()
return resp.json() if resp.content else None
result = resp.json() if resp.content else None
except (requests.RequestException, ValueError) as exc: except (requests.RequestException, ValueError) as exc:
raise EVEAPIError(str(exc)) raise EVEAPIError(str(exc))


def get(self, query, token):
if can_cache and result is not None:
self._cache.insert(key, result, resp)
return result

def get(self, query, token, params=None):
"""Do an HTTP GET request for a query using a token.""" """Do an HTTP GET request for a query using a token."""
return self._do(query, None, token, self._session.get)
meth = self._session.get
return self._do(meth, query, params, None, token, True)


def post(self, query, token, data=None):
def post(self, query, token, params=None, data=None):
"""Do an HTTP POST request for a query using a token.""" """Do an HTTP POST request for a query using a token."""
return self._do(query, data, token, self._session.post)
meth = self._session.post
return self._do(meth, query, params, data, token)


def put(self, query, token, data=None):
def put(self, query, token, params=None, data=None):
"""Do an HTTP PUT request for a query using a token.""" """Do an HTTP PUT request for a query using a token."""
return self._do(query, data, token, self._session.put)
meth = self._session.put
return self._do(meth, query, params, data, token)


def delete(self, query, token):
def delete(self, query, token, params=None):
"""Do an HTTP DELETE request for a query using a token.""" """Do an HTTP DELETE request for a query using a token."""
return self._do(query, None, token, self._session.delete)
meth = self._session.delete
return self._do(meth, query, params, None, token)

Loading…
Cancel
Save