import time, json, pytz, requests from datetime import datetime, timedelta from sseclient import SSEClient import xml.etree.ElementTree as ET from toolbox import Util class API(object): def __init__(self, url=None, user=None, key=None): self.setupRequests(url=url, user=user, key=key) self.reset() self.statistics = dict(speed=0, time=0, size=0) self.format = 'json' def reset(self): self.tElapsed, self.lines, self.util = 0, [], Util() def setupRequests(self, url, user=None, key=None, rateLimit=300, api_suffix=''): """ Setup the request parameters. This should be called once within __init__. Parametes --------- url: string The URL for the requests library. user : string, optional username for HTTPBasicAuth key : string, optional key or password for HTTPBasicAuth rateLimit : integer, optional How many seconds to wait betrween two requests. 3600s default. api_suffix : string, optional String to append to the URL for the request. Example: '&page=1' """ self.api_url, self.rateLimit = url, rateLimit self.lastUpdate, self.response = 0, dict() self.api_suffix, self.request_url = '', '' self.status = False self.api_auth = requests.auth.HTTPBasicAuth(user, key) if user and key else None def preQuery(self): """ This is called before the requests methods. The self.query() will use the self.request_url. It can be modified here. """ self.request_url = self.api_url pass def postQuery(self): """ This is called after a successfull reuqest and before the end of the query() methodself. The data from the requests lies in self.response. This method can be used to check the data for validity. self.query() sets self.status to True before callig this and returns it after this. If the data check fails set self.status to false. """ pass def query(self, stream=False): """ Perfom a GET-Request on the URL in self.request_url. Returns ------- bool True if successfull, False otherwise. True means that new data was fetched from the URL. False means the request has failed or it hits the rate limit. """ if time.time() - self.lastUpdate > self.rateLimit: self.status = False self.preQuery() tStart = time.time() result = requests.get(self.request_url, auth=self.api_auth, stream=stream) self.tElapsed = time.time() - tStart self.lastUpdate += 5 if result.status_code == 202 else 0 if result.status_code == 200: if not stream: self.stats(self.tElapsed, len(result.content)) if self.format == 'json': self.response = result.json() elif self.format == 'xml': self.response = ET.fromstring(result.content) elif stream: self.response = None self.sseclient = SSEClient(result) self.lastUpdate = time.time() self.status = True self.postQuery() else: pass else: self.postQuery() return self.status def stats(self, time, size): self.statistics.update(time=time, size=size) self.statistics.update(speed=round(self.statistics['speed']/self.statistics['time'])) def update(self): pass