http_API.py 3.76 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
import time, json, pytz, requests
from datetime import datetime, timedelta
from sseclient import SSEClient
import xml.etree.ElementTree as ET

from toolbox import Util

class API(object):
    def __init__(self, url=None, user=None, key=None):
        self.setupRequests(url=url, user=user, key=key)
        self.reset()
        self.statistics = dict(speed=0, time=0, size=0)
        self.format = 'json'

    def reset(self):
        self.tElapsed, self.lines, self.util = 0, [], Util()

18
    def setupRequests(self, url, user=None, key=None, rateLimit=300, api_suffix=''):
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
        """
        Setup the request parameters.

        This should be called once within __init__.

        Parametes
        ---------
        url: string
            The URL for the requests library.
        user : string, optional
            username for HTTPBasicAuth
        key : string, optional
            key or password for HTTPBasicAuth
        rateLimit : integer, optional
            How many seconds to wait betrween two requests. 3600s default.
        api_suffix : string, optional
            String to append to the URL for the request. Example: '&page=1'

        """
        self.api_url, self.rateLimit = url, rateLimit
        self.lastUpdate, self.response = 0, dict()
        self.api_suffix, self.request_url = '', ''
        self.status = False
        self.api_auth = requests.auth.HTTPBasicAuth(user, key) if user and key else None

    def preQuery(self):
        """
        This is called before the requests methods.

        The self.query() will use the self.request_url. It can be modified here.

        """
        self.request_url = self.api_url
        pass

    def postQuery(self):
        """
        This is called after a successfull reuqest and before the end of the
        query() methodself.

        The data from the requests lies in self.response. This method can be
        used to check the data for validity. self.query() sets self.status to
        True before callig this and returns it after this. If the data check
        fails set self.status to false.

        """
        pass

    def query(self, stream=False):
        """
        Perfom a GET-Request on the URL in self.request_url.

        Returns
        -------
        bool
            True if successfull, False otherwise.

            True means that new data was fetched from the URL. False means the
            request has failed or it hits the rate limit.

        """
        if time.time() - self.lastUpdate > self.rateLimit:
            self.status = False
            self.preQuery()
            tStart = time.time()
            result = requests.get(self.request_url, auth=self.api_auth, stream=stream)
            self.tElapsed = time.time() - tStart
            self.lastUpdate += 5 if result.status_code == 202 else 0
            if result.status_code == 200:
                if not stream:
                    self.stats(self.tElapsed, len(result.content))
                if self.format == 'json':
                    self.response = result.json()
                elif self.format == 'xml':
                    self.response = ET.fromstring(result.content)
                elif stream:
                    self.response = None
                    self.sseclient = SSEClient(result)
                self.lastUpdate = time.time()
                self.status = True
                self.postQuery()
            else:
                pass
        else:
            self.postQuery()
        return self.status

    def stats(self, time, size):
        self.statistics.update(time=time, size=size)
        self.statistics.update(speed=round(self.statistics['speed']/self.statistics['time']))

    def update(self):
        pass