Networking #33
Networking #33
Conversation
| @@ -4,7 +4,7 @@ | |||
| import os | |||
| import sys | |||
|
|
|||
| from pytangle.connectivity import Paginator | |||
| #from pytangle.connectivity import RateLimitedPaginator | |||
There was a problem hiding this comment.
best to remove the lines commented out, just for the sake of cleanliness
| #yield from Paginator(endpoint=PostsEndpoint(args=remove_null_values_from_dict(params))) | ||
| #yield from RateLimitedPaginator(PostsEndpoint(args = remove_null_values_from_dict(params))) | ||
| #return PostsEndpoint.call_rate_limited_paginator(args = remove_null_values_from_dict(params)) | ||
| yield from PostsEndpoint(args = remove_null_values_from_dict(params)) |
| @@ -9,29 +9,38 @@ | |||
| from sys import exit | |||
| from dateutil.parser import parse as date_parse | |||
| import requests | |||
| from ratelimit import limits, sleep_and_retry, RateLimitException | |||
|
|
|||
| from ratelimit import sleep_and_retry, RateLimitException | |||
There was a problem hiding this comment.
this shouldn't be needed, right?
| # ONE_MINUTE = 60 | ||
| # TEN_SECONDS = 10 | ||
| # THIRTY_SECONDS = 30 | ||
| MAX_RETRIES = read_max_retries(read_config()) if read_max_retries(read_config()) else 5 |
There was a problem hiding this comment.
would be cleaner to read once from config and then adjusting the value, along the lines of
MAX_RETRIES = read_max_retries(read_config())
MAX_RETRIES = max(MAX_RETRIES, 5)
There was a problem hiding this comment.
is it actually used in the paginator?
|
|
||
|
|
||
| # the following use of ratelimit should be removed |
| self.num_calls = 1 #num_calls | ||
| self.clock = time.time() | ||
| self.last_reset = time.time() | ||
| self.clamped_calls = max(1, min(sys.maxsize, floor(self.num_calls))) |
There was a problem hiding this comment.
not sure what is happening here. could you guide me through?
|
|
||
| self.request_fun = make_request # have make request here instead of importing in endpoints.py | ||
| self.time_unit = time_unit / num_calls | ||
| self.num_calls = 1 #num_calls |
| self.num_calls_so_far = 0 | ||
| self.last_reset = time.time() | ||
|
|
There was a problem hiding this comment.
would be easier down the line to have it made into a __reset() method
|
|
||
| # If the number of attempts to call the function exceeds the | ||
| # maximum then raise an exception. | ||
| if self.num_calls_so_far / 100 >= self.clamped_calls and self.__period_remaining() > 0: |
There was a problem hiding this comment.
I suspect that we would not need the check on __period_remaining . if it expires at this point it'd still be cleaner to throw the error, so that you can properly reset the num_calles_so_far counter
speaking of, why /100?
| # If the number of attempts to call the function exceeds the | ||
| # maximum then raise an exception. | ||
| if self.num_calls_so_far / 100 >= self.clamped_calls and self.__period_remaining() > 0: | ||
| time.sleep(self.__period_remaining()) # sleep for time remaining |
There was a problem hiding this comment.
sleep on negative values may throw errors; you could have a max(0, ...), or save the value returned at the start of the method in line 304
To ensure 'global' ratelimiting of various endpoints: