From 6f2d77a2f8188b11d11343aef53597fb601c5725 Mon Sep 17 00:00:00 2001 From: Julian Perelli Date: Thu, 27 Jun 2024 13:04:16 +0200 Subject: [PATCH] upgrade geopy_arcgis --- apps/catastro/geopy_arcgis.py | 227 +++++++++++++++++----------------- 1 file changed, 116 insertions(+), 111 deletions(-) diff --git a/apps/catastro/geopy_arcgis.py b/apps/catastro/geopy_arcgis.py index 31cb8058..ce5e7184 100644 --- a/apps/catastro/geopy_arcgis.py +++ b/apps/catastro/geopy_arcgis.py @@ -2,16 +2,16 @@ # y modificado un poco para agregar params a la url import json -import warnings +from functools import partial from time import time +from urllib.parse import urlencode -from geopy.compat import Request, string_compare, urlencode from geopy.exc import ( ConfigurationError, GeocoderAuthenticationFailure, GeocoderServiceError, ) -from geopy.geocoders.base import DEFAULT_SENTINEL, Geocoder +from geopy.geocoders.base import DEFAULT_SENTINEL, Geocoder, _synchronized from geopy.location import Location from geopy.util import logger @@ -28,21 +28,26 @@ class ArcGIS(Geocoder): """ _TOKEN_EXPIRED = 498 - _MAX_RETRIES = 3 - auth_api = 'https://www.arcgis.com/sharing/generateToken' + + auth_path = '/sharing/generateToken' + geocode_path = '/arcgis/rest/services/World/GeocodeServer/findAddressCandidates' + reverse_path = '/arcgis/rest/services/World/GeocodeServer/reverseGeocode' def __init__( self, username=None, password=None, + *, referer=None, token_lifetime=60, scheme=None, timeout=DEFAULT_SENTINEL, proxies=DEFAULT_SENTINEL, user_agent=None, - format_string=None, ssl_context=DEFAULT_SENTINEL, + adapter_factory=None, + auth_domain='www.arcgis.com', + domain='geocode.arcgis.com' ): """ @@ -74,26 +79,29 @@ def __init__( :param str user_agent: See :attr:`geopy.geocoders.options.default_user_agent`. - .. versionadded:: 1.12.0 - - :param str format_string: - See :attr:`geopy.geocoders.options.default_format_string`. - - .. versionadded:: 1.14.0 - :type ssl_context: :class:`ssl.SSLContext` :param ssl_context: See :attr:`geopy.geocoders.options.default_ssl_context`. - .. versionadded:: 1.14.0 + :param callable adapter_factory: + See :attr:`geopy.geocoders.options.default_adapter_factory`. + + .. versionadded:: 2.0 + + :param str auth_domain: Domain where the target ArcGIS auth service + is hosted. Used only in authenticated mode (i.e. username, + password and referer are set). + + :param str domain: Domain where the target ArcGIS service + is hosted. """ - super(ArcGIS, self).__init__( - format_string=format_string, + super().__init__( scheme=scheme, timeout=timeout, proxies=proxies, user_agent=user_agent, ssl_context=ssl_context, + adapter_factory=adapter_factory, ) if username or password or referer: if not (username and password and referer): @@ -105,40 +113,30 @@ def __init__( raise ConfigurationError( "Authenticated mode requires scheme of 'https'" ) - self._base_call_geocoder = self._call_geocoder - self._call_geocoder = self._authenticated_call_geocoder self.username = username self.password = password self.referer = referer + self.auth_domain = auth_domain.strip('/') + self.auth_api = ( + '%s://%s%s' % (self.scheme, self.auth_domain, self.auth_path) + ) - self.token = None self.token_lifetime = token_lifetime * 60 # store in seconds - self.token_expiry = None - self.retry = 1 + self.domain = domain.strip('/') self.api = ( - '%s://geocode.arcgis.com/arcgis/rest/services/' - 'World/GeocodeServer/findAddressCandidates' % self.scheme + '%s://%s%s' % (self.scheme, self.domain, self.geocode_path) ) self.reverse_api = ( - '%s://geocode.arcgis.com/arcgis/rest/services/' - 'World/GeocodeServer/reverseGeocode' % self.scheme + '%s://%s%s' % (self.scheme, self.domain, self.reverse_path) ) - def _authenticated_call_geocoder(self, url, timeout=DEFAULT_SENTINEL): - """ - Wrap self._call_geocoder, handling tokens. - """ - if self.token is None or int(time()) > self.token_expiry: - self._refresh_authentication_token() - request = Request( - "&".join((url, urlencode({"token": self.token}))), - headers={"Referer": self.referer} - ) - return self._base_call_geocoder(request, timeout=timeout) + # Mutable state + self.token = None + self.token_expiry = None - def geocode(self, query, exactly_one=True, timeout=DEFAULT_SENTINEL, + def geocode(self, query, *, exactly_one=True, timeout=DEFAULT_SENTINEL, out_fields=None, extra_params=None): """ Return a location point by address. @@ -146,7 +144,7 @@ def geocode(self, query, exactly_one=True, timeout=DEFAULT_SENTINEL, :param str query: The address or query you wish to geocode. :param bool exactly_one: Return one result or a list of results, if - available. + available.` :param int timeout: Time, in seconds, to wait for the geocoding service to respond before raising a :class:`geopy.exc.GeocoderTimedOut` @@ -159,18 +157,16 @@ def geocode(self, query, exactly_one=True, timeout=DEFAULT_SENTINEL, https://developers.arcgis.com/rest/geocode/api-reference/geocoding-service-output.htm for a list of supported output fields. If you want to return all supported output fields, set ``out_fields="*"``. - - .. versionadded:: 1.14.0 :type out_fields: str or iterable :rtype: ``None``, :class:`geopy.location.Location` or a list of them, if ``exactly_one=False``. """ - params = {'singleLine': self.format_string % query, 'f': 'json'} + params = {'singleLine': query, 'f': 'json'} if exactly_one: params['maxLocations'] = 1 if out_fields is not None: - if isinstance(out_fields, string_compare): + if isinstance(out_fields, str): params['outFields'] = out_fields else: params['outFields'] = ",".join(out_fields) @@ -178,16 +174,11 @@ def geocode(self, query, exactly_one=True, timeout=DEFAULT_SENTINEL, params.update(extra_params) url = "?".join((self.api, urlencode(params))) logger.debug("%s.geocode: %s", self.__class__.__name__, url) - response = self._call_geocoder(url, timeout=timeout) + callback = partial(self._parse_geocode, exactly_one=exactly_one) + return self._authenticated_call_geocoder(url, callback, timeout=timeout) - # Handle any errors; recursing in the case of an expired token. + def _parse_geocode(self, response, exactly_one): if 'error' in response: - if response['error']['code'] == self._TOKEN_EXPIRED: - self.retry += 1 - self._refresh_authentication_token() - return self.geocode( - query, exactly_one=exactly_one, timeout=timeout - ) raise GeocoderServiceError(str(response['error'])) # Success; convert from the ArcGIS JSON format. @@ -205,8 +196,8 @@ def geocode(self, query, exactly_one=True, timeout=DEFAULT_SENTINEL, return geocoded[0] return geocoded - def reverse(self, query, exactly_one=True, timeout=DEFAULT_SENTINEL, - distance=None, wkid=DEFAULT_WKID): + def reverse(self, query, *, exactly_one=True, timeout=DEFAULT_SENTINEL, + distance=None): """ Return an address by location point. @@ -227,48 +218,23 @@ def reverse(self, query, exactly_one=True, timeout=DEFAULT_SENTINEL, within which to search. ArcGIS has a default of 100 meters, if not specified. - :param str wkid: WKID to use for both input and output coordinates. - - .. deprecated:: 1.14.0 - It wasn't working before because it was specified incorrectly - in the request parameters, and won't work even if we fix the - request, because :class:`geopy.point.Point` normalizes the - coordinates according to WKID 4326. Please open an issue in - the geopy issue tracker if you believe that custom wkid values - should be supported. - :rtype: ``None``, :class:`geopy.location.Location` or a list of them, if ``exactly_one=False``. """ - # ArcGIS is lon,lat; maintain lat,lon convention of geopy - point = self._coerce_point_to_string(query).split(",") - if wkid != DEFAULT_WKID: - warnings.warn("%s.reverse: custom wkid value has been ignored. " - "It wasn't working before because it was specified " - "incorrectly in the request parameters, and won't " - "work even if we fix the request, because geopy.Point " - "normalizes the coordinates according to WKID %s. " - "Please open an issue in the geopy issue tracker " - "if you believe that custom wkid values should be " - "supported." % (type(self).__name__, DEFAULT_WKID), - DeprecationWarning) - wkid = DEFAULT_WKID - location = ",".join((point[1], point[0])) + location = self._coerce_point_to_string(query, "%(lon)s,%(lat)s") + wkid = DEFAULT_WKID params = {'location': location, 'f': 'json', 'outSR': wkid} if distance is not None: params['distance'] = distance url = "?".join((self.reverse_api, urlencode(params))) logger.debug("%s.reverse: %s", self.__class__.__name__, url) - response = self._call_geocoder(url, timeout=timeout) + callback = partial(self._parse_reverse, exactly_one=exactly_one) + return self._authenticated_call_geocoder(url, callback, timeout=timeout) + + def _parse_reverse(self, response, exactly_one): if not len(response): return None if 'error' in response: - if response['error']['code'] == self._TOKEN_EXPIRED: - self.retry += 1 - self._refresh_authentication_token() - return self.reverse(query, exactly_one=exactly_one, - timeout=timeout, distance=distance, - wkid=wkid) # https://developers.arcgis.com/rest/geocode/api-reference/geocoding-service-output.htm if response['error']['code'] == 400: # 'details': ['Unable to find address for the specified location.']} @@ -278,10 +244,15 @@ def reverse(self, query, exactly_one=True, timeout=DEFAULT_SENTINEL, except (KeyError, IndexError): pass raise GeocoderServiceError(str(response['error'])) - address = ( - "%(Address)s, %(City)s, %(Region)s %(Postal)s," - " %(CountryCode)s" % response['address'] - ) + + if response['address'].get('Address'): + address = ( + "%(Address)s, %(City)s, %(Region)s %(Postal)s," + " %(CountryCode)s" % response['address'] + ) + else: + address = response['address']['LongLabel'] + location = Location( address, (response['location']['y'], response['location']['x']), @@ -292,14 +263,50 @@ def reverse(self, query, exactly_one=True, timeout=DEFAULT_SENTINEL, else: return [location] - def _refresh_authentication_token(self): - """ - POST to ArcGIS requesting a new token. - """ - if self.retry == self._MAX_RETRIES: - raise GeocoderAuthenticationFailure( - 'Too many retries for auth: %s' % self.retry + def _authenticated_call_geocoder( + self, url, parse_callback, *, timeout=DEFAULT_SENTINEL + ): + if not self.username: + return self._call_geocoder(url, parse_callback, timeout=timeout) + + def query_callback(): + call_url = "&".join((url, urlencode({"token": self.token}))) + headers = {"Referer": self.referer} + return self._call_geocoder( + call_url, + partial(maybe_reauthenticate_callback, from_token=self.token), + timeout=timeout, + headers=headers, + ) + + def maybe_reauthenticate_callback(response, *, from_token): + if "error" in response: + if response["error"]["code"] == self._TOKEN_EXPIRED: + return self._refresh_authentication_token( + query_retry_callback, timeout=timeout, from_token=from_token + ) + return parse_callback(response) + + def query_retry_callback(): + call_url = "&".join((url, urlencode({"token": self.token}))) + headers = {"Referer": self.referer} + return self._call_geocoder( + call_url, parse_callback, timeout=timeout, headers=headers + ) + + if self.token is None or int(time()) > self.token_expiry: + return self._refresh_authentication_token( + query_callback, timeout=timeout, from_token=self.token ) + else: + return query_callback() + + @_synchronized + def _refresh_authentication_token(self, callback_success, *, timeout, from_token): + if from_token != self.token: + # Token has already been updated by a concurrent call. + return callback_success() + token_request_arguments = { 'username': self.username, 'password': self.password, @@ -312,16 +319,18 @@ def _refresh_authentication_token(self): "%s._refresh_authentication_token: %s", self.__class__.__name__, url ) - self.token_expiry = int(time()) + self.token_lifetime - response = self._base_call_geocoder(url) - if 'token' not in response: - raise GeocoderAuthenticationFailure( - 'Missing token in auth request.' - 'Request URL: %s; response JSON: %s' % - (url, json.dumps(response)) - ) - self.retry = 0 - self.token = response['token'] + + def cb(response): + if "token" not in response: + raise GeocoderAuthenticationFailure( + "Missing token in auth request." + "Request URL: %s; response JSON: %s" % (url, json.dumps(response)) + ) + self.token = response["token"] + self.token_expiry = int(time()) + self.token_lifetime + return callback_success() + + return self._call_geocoder(url, cb, timeout=timeout) class ArcGISSuggest(ArcGIS): @@ -374,7 +383,7 @@ def geocode(self, query, exactly_one=True, timeout=DEFAULT_SENTINEL, if exactly_one: params['maxLocations'] = 1 if out_fields is not None: - if isinstance(out_fields, string_compare): + if isinstance(out_fields, str): params['outFields'] = out_fields else: params['outFields'] = ",".join(out_fields) @@ -387,12 +396,6 @@ def geocode(self, query, exactly_one=True, timeout=DEFAULT_SENTINEL, # Handle any errors; recursing in the case of an expired token. if 'error' in response: - if response['error']['code'] == self._TOKEN_EXPIRED: - self.retry += 1 - self._refresh_authentication_token() - return self.geocode( - query, exactly_one=exactly_one, timeout=timeout - ) raise GeocoderServiceError(str(response['error'])) # Success; convert from the ArcGIS JSON format. @@ -405,6 +408,8 @@ def geocode(self, query, exactly_one=True, timeout=DEFAULT_SENTINEL, "magickey": resource["magicKey"] } ) + if exactly_one: return geocoded[0] + return geocoded