Skip to content

Commit

Permalink
upgrade geopy_arcgis
Browse files Browse the repository at this point in the history
  • Loading branch information
@jperelli
jperelli committed Jun 27, 2024
1 parent e59de74 commit 6f2d77a
Showing 1 changed file with 116 additions and 111 deletions.
227 changes: 116 additions & 111 deletions apps/catastro/geopy_arcgis.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,16 +2,16 @@
# y modificado un poco para agregar params a la url

import json
import warnings
from functools import partial
from time import time
from urllib.parse import urlencode

from geopy.compat import Request, string_compare, urlencode
from geopy.exc import (
ConfigurationError,
GeocoderAuthenticationFailure,
GeocoderServiceError,
)
from geopy.geocoders.base import DEFAULT_SENTINEL, Geocoder
from geopy.geocoders.base import DEFAULT_SENTINEL, Geocoder, _synchronized
from geopy.location import Location
from geopy.util import logger

Expand All @@ -28,21 +28,26 @@ class ArcGIS(Geocoder):
"""

_TOKEN_EXPIRED = 498
_MAX_RETRIES = 3
auth_api = 'https://www.arcgis.com/sharing/generateToken'

auth_path = '/sharing/generateToken'
geocode_path = '/arcgis/rest/services/World/GeocodeServer/findAddressCandidates'
reverse_path = '/arcgis/rest/services/World/GeocodeServer/reverseGeocode'

def __init__(
self,
username=None,
password=None,
*,
referer=None,
token_lifetime=60,
scheme=None,
timeout=DEFAULT_SENTINEL,
proxies=DEFAULT_SENTINEL,
user_agent=None,
format_string=None,
ssl_context=DEFAULT_SENTINEL,
adapter_factory=None,
auth_domain='www.arcgis.com',
domain='geocode.arcgis.com'
):
"""
Expand Down Expand Up @@ -74,26 +79,29 @@ def __init__(
:param str user_agent:
See :attr:`geopy.geocoders.options.default_user_agent`.
.. versionadded:: 1.12.0
:param str format_string:
See :attr:`geopy.geocoders.options.default_format_string`.
.. versionadded:: 1.14.0
:type ssl_context: :class:`ssl.SSLContext`
:param ssl_context:
See :attr:`geopy.geocoders.options.default_ssl_context`.
.. versionadded:: 1.14.0
:param callable adapter_factory:
See :attr:`geopy.geocoders.options.default_adapter_factory`.
.. versionadded:: 2.0
:param str auth_domain: Domain where the target ArcGIS auth service
is hosted. Used only in authenticated mode (i.e. username,
password and referer are set).
:param str domain: Domain where the target ArcGIS service
is hosted.
"""
super(ArcGIS, self).__init__(
format_string=format_string,
super().__init__(
scheme=scheme,
timeout=timeout,
proxies=proxies,
user_agent=user_agent,
ssl_context=ssl_context,
adapter_factory=adapter_factory,
)
if username or password or referer:
if not (username and password and referer):
Expand All @@ -105,48 +113,38 @@ def __init__(
raise ConfigurationError(
"Authenticated mode requires scheme of 'https'"
)
self._base_call_geocoder = self._call_geocoder
self._call_geocoder = self._authenticated_call_geocoder

self.username = username
self.password = password
self.referer = referer
self.auth_domain = auth_domain.strip('/')
self.auth_api = (
'%s://%s%s' % (self.scheme, self.auth_domain, self.auth_path)
)

self.token = None
self.token_lifetime = token_lifetime * 60 # store in seconds
self.token_expiry = None
self.retry = 1

self.domain = domain.strip('/')
self.api = (
'%s://geocode.arcgis.com/arcgis/rest/services/'
'World/GeocodeServer/findAddressCandidates' % self.scheme
'%s://%s%s' % (self.scheme, self.domain, self.geocode_path)
)
self.reverse_api = (
'%s://geocode.arcgis.com/arcgis/rest/services/'
'World/GeocodeServer/reverseGeocode' % self.scheme
'%s://%s%s' % (self.scheme, self.domain, self.reverse_path)
)

def _authenticated_call_geocoder(self, url, timeout=DEFAULT_SENTINEL):
"""
Wrap self._call_geocoder, handling tokens.
"""
if self.token is None or int(time()) > self.token_expiry:
self._refresh_authentication_token()
request = Request(
"&".join((url, urlencode({"token": self.token}))),
headers={"Referer": self.referer}
)
return self._base_call_geocoder(request, timeout=timeout)
# Mutable state
self.token = None
self.token_expiry = None

def geocode(self, query, exactly_one=True, timeout=DEFAULT_SENTINEL,
def geocode(self, query, *, exactly_one=True, timeout=DEFAULT_SENTINEL,
out_fields=None, extra_params=None):
"""
Return a location point by address.
:param str query: The address or query you wish to geocode.
:param bool exactly_one: Return one result or a list of results, if
available.
available.`
:param int timeout: Time, in seconds, to wait for the geocoding service
to respond before raising a :class:`geopy.exc.GeocoderTimedOut`
Expand All @@ -159,35 +157,28 @@ def geocode(self, query, exactly_one=True, timeout=DEFAULT_SENTINEL,
https://developers.arcgis.com/rest/geocode/api-reference/geocoding-service-output.htm
for a list of supported output fields. If you want to return all
supported output fields, set ``out_fields="*"``.
.. versionadded:: 1.14.0
:type out_fields: str or iterable
:rtype: ``None``, :class:`geopy.location.Location` or a list of them, if
``exactly_one=False``.
"""
params = {'singleLine': self.format_string % query, 'f': 'json'}
params = {'singleLine': query, 'f': 'json'}
if exactly_one:
params['maxLocations'] = 1
if out_fields is not None:
if isinstance(out_fields, string_compare):
if isinstance(out_fields, str):
params['outFields'] = out_fields
else:
params['outFields'] = ",".join(out_fields)
if extra_params is not None:
params.update(extra_params)
url = "?".join((self.api, urlencode(params)))
logger.debug("%s.geocode: %s", self.__class__.__name__, url)
response = self._call_geocoder(url, timeout=timeout)
callback = partial(self._parse_geocode, exactly_one=exactly_one)
return self._authenticated_call_geocoder(url, callback, timeout=timeout)

# Handle any errors; recursing in the case of an expired token.
def _parse_geocode(self, response, exactly_one):
if 'error' in response:
if response['error']['code'] == self._TOKEN_EXPIRED:
self.retry += 1
self._refresh_authentication_token()
return self.geocode(
query, exactly_one=exactly_one, timeout=timeout
)
raise GeocoderServiceError(str(response['error']))

# Success; convert from the ArcGIS JSON format.
Expand All @@ -205,8 +196,8 @@ def geocode(self, query, exactly_one=True, timeout=DEFAULT_SENTINEL,
return geocoded[0]
return geocoded

def reverse(self, query, exactly_one=True, timeout=DEFAULT_SENTINEL,
distance=None, wkid=DEFAULT_WKID):
def reverse(self, query, *, exactly_one=True, timeout=DEFAULT_SENTINEL,
distance=None):
"""
Return an address by location point.
Expand All @@ -227,48 +218,23 @@ def reverse(self, query, exactly_one=True, timeout=DEFAULT_SENTINEL,
within which to search. ArcGIS has a default of 100 meters, if not
specified.
:param str wkid: WKID to use for both input and output coordinates.
.. deprecated:: 1.14.0
It wasn't working before because it was specified incorrectly
in the request parameters, and won't work even if we fix the
request, because :class:`geopy.point.Point` normalizes the
coordinates according to WKID 4326. Please open an issue in
the geopy issue tracker if you believe that custom wkid values
should be supported.
:rtype: ``None``, :class:`geopy.location.Location` or a list of them, if
``exactly_one=False``.
"""
# ArcGIS is lon,lat; maintain lat,lon convention of geopy
point = self._coerce_point_to_string(query).split(",")
if wkid != DEFAULT_WKID:
warnings.warn("%s.reverse: custom wkid value has been ignored. "
"It wasn't working before because it was specified "
"incorrectly in the request parameters, and won't "
"work even if we fix the request, because geopy.Point "
"normalizes the coordinates according to WKID %s. "
"Please open an issue in the geopy issue tracker "
"if you believe that custom wkid values should be "
"supported." % (type(self).__name__, DEFAULT_WKID),
DeprecationWarning)
wkid = DEFAULT_WKID
location = ",".join((point[1], point[0]))
location = self._coerce_point_to_string(query, "%(lon)s,%(lat)s")
wkid = DEFAULT_WKID
params = {'location': location, 'f': 'json', 'outSR': wkid}
if distance is not None:
params['distance'] = distance
url = "?".join((self.reverse_api, urlencode(params)))
logger.debug("%s.reverse: %s", self.__class__.__name__, url)
response = self._call_geocoder(url, timeout=timeout)
callback = partial(self._parse_reverse, exactly_one=exactly_one)
return self._authenticated_call_geocoder(url, callback, timeout=timeout)

def _parse_reverse(self, response, exactly_one):
if not len(response):
return None
if 'error' in response:
if response['error']['code'] == self._TOKEN_EXPIRED:
self.retry += 1
self._refresh_authentication_token()
return self.reverse(query, exactly_one=exactly_one,
timeout=timeout, distance=distance,
wkid=wkid)
# https://developers.arcgis.com/rest/geocode/api-reference/geocoding-service-output.htm
if response['error']['code'] == 400:
# 'details': ['Unable to find address for the specified location.']}
Expand All @@ -278,10 +244,15 @@ def reverse(self, query, exactly_one=True, timeout=DEFAULT_SENTINEL,
except (KeyError, IndexError):
pass
raise GeocoderServiceError(str(response['error']))
address = (
"%(Address)s, %(City)s, %(Region)s %(Postal)s,"
" %(CountryCode)s" % response['address']
)

if response['address'].get('Address'):
address = (
"%(Address)s, %(City)s, %(Region)s %(Postal)s,"
" %(CountryCode)s" % response['address']
)
else:
address = response['address']['LongLabel']

location = Location(
address,
(response['location']['y'], response['location']['x']),
Expand All @@ -292,14 +263,50 @@ def reverse(self, query, exactly_one=True, timeout=DEFAULT_SENTINEL,
else:
return [location]

def _refresh_authentication_token(self):
"""
POST to ArcGIS requesting a new token.
"""
if self.retry == self._MAX_RETRIES:
raise GeocoderAuthenticationFailure(
'Too many retries for auth: %s' % self.retry
def _authenticated_call_geocoder(
self, url, parse_callback, *, timeout=DEFAULT_SENTINEL
):
if not self.username:
return self._call_geocoder(url, parse_callback, timeout=timeout)

def query_callback():
call_url = "&".join((url, urlencode({"token": self.token})))
headers = {"Referer": self.referer}
return self._call_geocoder(
call_url,
partial(maybe_reauthenticate_callback, from_token=self.token),
timeout=timeout,
headers=headers,
)

def maybe_reauthenticate_callback(response, *, from_token):
if "error" in response:
if response["error"]["code"] == self._TOKEN_EXPIRED:
return self._refresh_authentication_token(
query_retry_callback, timeout=timeout, from_token=from_token
)
return parse_callback(response)

def query_retry_callback():
call_url = "&".join((url, urlencode({"token": self.token})))
headers = {"Referer": self.referer}
return self._call_geocoder(
call_url, parse_callback, timeout=timeout, headers=headers
)

if self.token is None or int(time()) > self.token_expiry:
return self._refresh_authentication_token(
query_callback, timeout=timeout, from_token=self.token
)
else:
return query_callback()

@_synchronized
def _refresh_authentication_token(self, callback_success, *, timeout, from_token):
if from_token != self.token:
# Token has already been updated by a concurrent call.
return callback_success()

token_request_arguments = {
'username': self.username,
'password': self.password,
Expand All @@ -312,16 +319,18 @@ def _refresh_authentication_token(self):
"%s._refresh_authentication_token: %s",
self.__class__.__name__, url
)
self.token_expiry = int(time()) + self.token_lifetime
response = self._base_call_geocoder(url)
if 'token' not in response:
raise GeocoderAuthenticationFailure(
'Missing token in auth request.'
'Request URL: %s; response JSON: %s' %
(url, json.dumps(response))
)
self.retry = 0
self.token = response['token']

def cb(response):
if "token" not in response:
raise GeocoderAuthenticationFailure(
"Missing token in auth request."
"Request URL: %s; response JSON: %s" % (url, json.dumps(response))
)
self.token = response["token"]
self.token_expiry = int(time()) + self.token_lifetime
return callback_success()

return self._call_geocoder(url, cb, timeout=timeout)


class ArcGISSuggest(ArcGIS):
Expand Down Expand Up @@ -374,7 +383,7 @@ def geocode(self, query, exactly_one=True, timeout=DEFAULT_SENTINEL,
if exactly_one:
params['maxLocations'] = 1
if out_fields is not None:
if isinstance(out_fields, string_compare):
if isinstance(out_fields, str):
params['outFields'] = out_fields
else:
params['outFields'] = ",".join(out_fields)
Expand All @@ -387,12 +396,6 @@ def geocode(self, query, exactly_one=True, timeout=DEFAULT_SENTINEL,

# Handle any errors; recursing in the case of an expired token.
if 'error' in response:
if response['error']['code'] == self._TOKEN_EXPIRED:
self.retry += 1
self._refresh_authentication_token()
return self.geocode(
query, exactly_one=exactly_one, timeout=timeout
)
raise GeocoderServiceError(str(response['error']))

# Success; convert from the ArcGIS JSON format.
Expand All @@ -405,6 +408,8 @@ def geocode(self, query, exactly_one=True, timeout=DEFAULT_SENTINEL,
"magickey": resource["magicKey"]
}
)

if exactly_one:
return geocoded[0]

return geocoded

0 comments on commit 6f2d77a

Please sign in to comment.