Browse Source

fix rate limit timeout issues

by pulling new client code from upstream (1fb8828)
and porting back to QGIS2.18
feature/processing_qgs2
Norwin 2 months ago
parent
commit
1380296d37

+ 2
- 2
Makefile View File

@@ -38,7 +38,7 @@ PLUGINNAME = OSMtools

PY_FILES = \
__init__.py osm_tools.py \
core gui osmtools_processing
core gui osmtools_processing utils

# translation
SOURCES = $(PY_FILES)
@@ -71,7 +71,7 @@ default: deploy
compile: $(COMPILED_RESOURCE_FILES)

%.py : %.qrc $(RESOURCES_SRC)
pyrcc4 -o $*.py $<
#pyrcc4 -o $*.py $<

%.qm : %.ts
$(LRELEASE) $<

+ 45
- 0
core/__init__.py View File

@@ -0,0 +1,45 @@
# -*- coding: utf-8 -*-
"""
/***************************************************************************
OSMtools
A QGIS plugin
QGIS client to query openrouteservice
-------------------
begin : 2017-02-01
git sha : $Format:%H$
copyright : (C) 2017 by Nils Nolde
email : nils.nolde@gmail.com
***************************************************************************/

This plugin provides access to the various APIs from OpenRouteService
(https://openrouteservice.org), developed and
maintained by GIScience team at University of Heidelberg, Germany. By using
this plugin you agree to the ORS terms of service
(https://openrouteservice.org/terms-of-service/).

/***************************************************************************
* *
* This program is free software; you can redistribute it and/or modify *
* it under the terms of the GNU General Public License as published by *
* the Free Software Foundation; either version 2 of the License, or *
* (at your option) any later version. *
* *
***************************************************************************/
"""

PROFILES = [
'driving-car',
'driving-hgv',
'cycling-regular',
'cycling-road',
'cycling-safe',
'cycling-mountain',
'cycling-electric',
'foot-walking',
'foot-hiking',
'wheelchair'
]

DIMENSIONS = ['time', 'distance']

PREFERENCES = ['fastest', 'shortest']

+ 162
- 179
core/client.py View File

@@ -1,82 +1,92 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Sat Feb 17 13:51:13 2018

@author: nilsnolde
/***************************************************************************
OSMtools
A QGIS plugin
QGIS client to query openrouteservice
-------------------
begin : 2017-02-01
git sha : $Format:%H$
copyright : (C) 2017 by Nils Nolde
email : nils.nolde@gmail.com
***************************************************************************/

This plugin provides access to the various APIs from OpenRouteService
(https://openrouteservice.org), developed and
maintained by GIScience team at University of Heidelberg, Germany. By using
this plugin you agree to the ORS terms of service
(https://openrouteservice.org/terms-of-service/).

/***************************************************************************
* *
* This program is free software; you can redistribute it and/or modify *
* it under the terms of the GNU General Public License as published by *
* the Free Software Foundation; either version 2 of the License, or *
* (at your option) any later version. *
* *
***************************************************************************/
"""


from datetime import datetime, timedelta
import requests
import random
import time
import collections
from urllib.parse import urlencode
from urllib import urlencode
import random
import json

import OSMtools
from OSMtools.core import exceptions, auxiliary
from PyQt4.QtCore import QObject, pyqtSignal

_USER_AGENT = "ORSClientQGIS/%s".format(OSMtools.__version__)
_RETRIABLE_STATUSES = [503]
_DEFAULT_BASE_URL = "https://api.openrouteservice.org"
from OSMtools.core import networkaccessmanager
from OSMtools.utils import exceptions, configmanager, logger

class Client(object):
_USER_AGENT = "ORSQGISClient@v1"


class Client(QObject):
"""Performs requests to the ORS API services."""

def __init__(self, iface=None,
apiKey=None,
retry_timeout=60,
requests_kwargs=None,
retry_over_query_limit=False):
def __init__(self,
provider=None,
retry_timeout=60):
"""
:param key: Optional ORS API key. If not provided, it's read from
config.yml. Requests will fail if no key is in config.yml
:type key: string

:param iface: Optional QGIS interface instance used to push notifications about status
:param iface: A QGIS interface instance.
:type iface: QgisInterface

:param provider: A openrouteservice provider from config.yml
:type provider: dict

:param retry_timeout: Timeout across multiple retriable requests, in
seconds.
:type retry_timeout: int

:param requests_kwargs: Extra keyword arguments for the requests
library, which among other things allow for proxy auth to be
implemented. See the official requests docs for more info:
http://docs.python-requests.org/en/latest/api/#main-interface
:type requests_kwargs: dict
"""
QObject.__init__(self)

base_params = auxiliary.readConfig()

(self.key,
self.base_url,
self.queries_per_minute) = [v for (k, v) in sorted(base_params.items())]
self.iface = iface
self.key = provider['key']
self.base_url = provider['base_url']
self.ENV_VARS = provider.get('ENV_VARS')

if apiKey:
self.key = apiKey
# self.session = requests.Session()
self.nam = networkaccessmanager.NetworkAccessManager(debug=False)

self.session = requests.Session()

self.retry_over_query_limit = retry_over_query_limit
self.retry_timeout = timedelta(seconds=retry_timeout)
self.requests_kwargs = requests_kwargs or {}
self.requests_kwargs.update({
"headers": {"User-Agent": _USER_AGENT,
'Content-type': 'application/json'}
})
self.headers = {
"User-Agent": _USER_AGENT,
'Content-type': 'application/json',
'Authorization': provider['key']
}

self.sent_times = collections.deque("", self.queries_per_minute)
# Save some references to retrieve in client instances
self.url = None
self.warnings = None

overQueryLimit = pyqtSignal()
def request(self,
url, params,
first_request_time=None,
retry_counter=0,
requests_kwargs=None,
post_json=None):
"""Performs HTTP GET/POST with credentials, returning the body as JSON.
url, params,
first_request_time=None,
retry_counter=0,
post_json=None):
"""Performs HTTP GET/POST with credentials, returning the body as
JSON.

:param url: URL extension for request. Should begin with a slash.
:type url: string
@@ -88,20 +98,13 @@ class Client(object):
retries have occurred).
:type first_request_time: datetime.datetime

:param retry_counter: The number of this retry, or zero for first attempt.
:type retry_counter: int

:param requests_kwargs: Same extra keywords arg for requests as per
__init__, but provided here to allow overriding internally on a
per-request basis.
:type requests_kwargs: dict
:param post_json: Parameters for POST endpoints
:type post_json: dict

:raises ApiError: when the API returns an error.
:raises Timeout: if the request timed out.
:raises TransportError: when something went wrong while trying to
execute a request.
:raises OSMtools.utils.exceptions.ApiError: when the API returns an error.

:rtype: dict from JSON response.
:returns: openrouteservice response body
:rtype: dict
"""

if not first_request_time:
@@ -115,7 +118,7 @@ class Client(object):
# 0.5 * (1.5 ^ i) is an increased sleep time of 1.5x per iteration,
# starting at 0.5s when retry_counter=1. The first retry will occur
# at 1, so subtract that first.
delay_seconds = 1.5 ** (retry_counter - 1)
delay_seconds = 1.5**(retry_counter - 1)

# Jitter this value by 50% and pause.
time.sleep(delay_seconds * (random.random() + 0.5))
@@ -123,91 +126,115 @@ class Client(object):
authed_url = self._generate_auth_url(url,
params,
)
self.url = self.base_url + authed_url

# Default to the client-level self.requests_kwargs, with method-level
# requests_kwargs arg overriding.
requests_kwargs = requests_kwargs or {}
final_requests_kwargs = dict(self.requests_kwargs, **requests_kwargs)

# Check if the time of the nth previous query (where n is
# queries_per_second) is under a second ago - if so, sleep for
# the difference.
if self.sent_times and len(self.sent_times) == self.queries_per_minute:
elapsed_since_earliest = time.time() - self.sent_times[0]
if elapsed_since_earliest < 60:
if self.iface:
self.iface.messageBar().pushInfo('Limit exceeded',
'Request limit of {} per minute exceeded. '
'Wait for {} seconds'.format(self.queries_per_minute,
60 - elapsed_since_earliest))
else:
print "Ratelimit exceeded. Please wait 60s before the next request"
time.sleep(60 - elapsed_since_earliest)

# Determine GET/POST.
# post_json is so far only sent from matrix call
requests_method = self.session.get
# final_requests_kwargs = self.requests_kwargs

# Determine GET/POST
# requests_method = self.session.get
requests_method = 'GET'
body = None
if post_json is not None:
requests_method = self.session.post
final_requests_kwargs["json"] = post_json
# requests_method = self.session.post
# final_requests_kwargs["json"] = post_json
body = post_json
requests_method = 'POST'

logger.log(
"url: {}\nParameters: {}".format(
self.url,
# final_requests_kwargs
body
),
0
)

try:
response = requests_method(self.base_url + authed_url,
**final_requests_kwargs)
except requests.exceptions.Timeout:
raise exceptions.Timeout()
except Exception as e:
raise #exceptions.TransportError(e)
# response = requests_method(
# self.base_url + authed_url,
# **final_requests_kwargs
# )
response, content = self.nam.request(self.url,
method=requests_method,
body=body,
headers=self.headers,
blocking=True)
# except requests.exceptions.Timeout:
# raise exceptions.Timeout()
except networkaccessmanager.RequestsExceptionTimeout:
raise exceptions.Timeout

except networkaccessmanager.RequestsException:
try:
# result = self._get_body(response)
self._check_status()

if response.status_code in _RETRIABLE_STATUSES:
# Retry request.
print('Server down.\nRetrying for the {}th time.'.format(retry_counter + 1))
except exceptions.OverQueryLimit as e:

return self.request(url, params, first_request_time,
retry_counter + 1, requests_kwargs, post_json)
# Let the instances know smth happened
self.overQueryLimit.emit()
logger.log("{}: {}".format(e.__class__.__name__, str(e)), 1)

try:
result = self._get_body(response)
self.sent_times.append(time.time())
return result
except exceptions._RetriableRequest as e:
if isinstance(e, exceptions._OverQueryLimit) and not self.retry_over_query_limit:
return self.request(url, params, first_request_time, retry_counter + 1, post_json)

except exceptions.ApiError as e:
logger.log("Feature ID {} caused a {}: {}".format(params.get('id', 'unknown'), e.__class__.__name__, str(e)), 2)
raise

if self.iface:
self.iface.messageBar().pushInfo('Rate limit exceeded.\nRetrying for the {}th time.'.format(retry_counter + 1))
else:
print "Ratelimit exceeded. Retrying.."
return self.request(url, params, first_request_time,
retry_counter + 1, requests_kwargs, post_json)
except:
raise

# Write env variables if successful
if self.ENV_VARS:
for env_var in self.ENV_VARS:
configmanager.write_env_var(env_var, response.headers.get(self.ENV_VARS[env_var], 'None'))

return json.loads(content.decode('utf-8'))

def _get_body(self, response):
def _check_status(self):
"""
Casts JSON response to dict

:param response: The HTTP response of the request.
:type reponse: JSON object
:raises OSMtools.utils.exceptions.OverQueryLimitError: when rate limit is exhausted, HTTP 429
:raises OSMtools.utils.exceptions.ApiError: when the backend API throws an error, HTTP 400
:raises OSMtools.utils.exceptions.InvalidKey: when API key is invalid (or quota is exceeded), HTTP 403
:raises OSMtools.utils.exceptions.GenericServerError: all other HTTP errors

:rtype: dict from JSON
:returns: response body
:rtype: dict
"""
body = response.json()
error = body.get('error')
status_code = response.status_code

if status_code == 429:
raise exceptions._OverQueryLimit(
str(status_code), error)
if status_code != 200:
try:
errorMsg = error['message']
except TypeError:
errorMsg = error
raise exceptions.ApiError(status_code, errorMsg)
status_code = self.nam.http_call_result.status_code
message = self.nam.http_call_result.text if self.nam.http_call_result.text != '' else self.nam.http_call_result.reason

return body
if status_code == 403:
raise exceptions.InvalidKey(
str(status_code),
# error,
message
)

if status_code == 429:
raise exceptions.OverQueryLimit(
str(status_code),
# error,
message
)
# Internal error message for Bad Request
if 400 < status_code < 500:
raise exceptions.ApiError(
str(status_code),
# error,
message
)
# Other HTTP errors have different formatting
if status_code != 200:
raise exceptions.GenericServerError(
str(status_code),
# error,
message
)

def _generate_auth_url(self, path, params):
"""Returns the path and query string portion of the request URL, first
@@ -219,8 +246,8 @@ class Client(object):
:param params: URL parameters.
:type params: dict or list of key/value tuples

:returns: encoded URL
:rtype: string

"""

if type(params) is dict:
@@ -228,51 +255,7 @@ class Client(object):

# Only auto-add API key when using ORS. If own instance, API key must
# be explicitly added to params
if self.key:
params.append(("api_key", self.key))
return path + "?" + _urlencode_params(params)
elif self.base_url != _DEFAULT_BASE_URL:
return path + "?" + _urlencode_params(params)

raise exceptions.ApiError("No API key specified. "
"Visit https://go.openrouteservice.org/dev-dashboard/ "
"to create one.")


def _urlencode_params(params):
"""URL encodes the parameters.

:param params: The parameters
:type params: list of key/value tuples.

:rtype: string
"""
# urlencode does not handle unicode strings in Python 2.
# Firstly, normalize the values so they get encoded correctly.
params = [(key, _normalize_for_urlencode(val)) for key, val in params]
# Secondly, unquote unreserved chars which are incorrectly quoted
# by urllib.urlencode, causing invalid auth signatures. See GH #72
# for more info.
return requests.utils.unquote_unreserved(urlencode(params))


try:
unicode
# NOTE(cbro): `unicode` was removed in Python 3. In Python 3, NameError is
# raised here, and caught below.

def _normalize_for_urlencode(value):
"""(Python 2) Converts the value to a `str` (raw bytes)."""
if isinstance(value, unicode):
return value.encode('utf8')

if isinstance(value, str):
return value

return _normalize_for_urlencode(str(value))

except NameError:
def _normalize_for_urlencode(value):
"""(Python 3) No-op."""
# urlencode in Python 3 handles all the types we are passing it.
return value
# if self.key:
# params.append(("api_key", self.key))

return path + "?" + requests.utils.unquote_unreserved(urlencode(params))

+ 396
- 0
core/networkaccessmanager.py View File

@@ -0,0 +1,396 @@
# -*- coding: utf-8 -*-
"""
***************************************************************************
An httplib2 replacement that uses QgsNetworkAccessManager

---------------------
Date : August 2016
Copyright : (C) 2016 Boundless, http://boundlessgeo.com
Email : apasotti at boundlessgeo dot com
***************************************************************************
* *
* This program is free software; you can redistribute it and/or modify *
* it under the terms of the GNU General Public License as published by *
* the Free Software Foundation; either version 2 of the License, or *
* (at your option) any later version. *
* *
***************************************************************************
"""

from builtins import str
from builtins import object

__author__ = 'Alessandro Pasotti'
__date__ = 'August 2016'

import re
import io
import urllib.request, urllib.error, urllib.parse

from qgis.PyQt.QtCore import QUrl, QEventLoop
from qgis.PyQt.QtNetwork import QNetworkRequest, QNetworkReply

from qgis.core import (
QgsApplication,
QgsNetworkAccessManager,
QgsMessageLog
)

# FIXME: ignored
DEFAULT_MAX_REDIRECTS = 4

class RequestsException(Exception):
pass

class RequestsExceptionTimeout(RequestsException):
pass

class RequestsExceptionConnectionError(RequestsException):
pass

class RequestsExceptionUserAbort(RequestsException):
pass

class Map(dict):
"""
Example:
m = Map({'first_name': 'Eduardo'}, last_name='Pool', age=24, sports=['Soccer'])
"""
def __init__(self, *args, **kwargs):
super(Map, self).__init__(*args, **kwargs)
for arg in args:
if isinstance(arg, dict):
for k, v in arg.items():
self[k] = v

if kwargs:
for k, v in kwargs.items():
self[k] = v

def __getattr__(self, attr):
return self.get(attr)

def __setattr__(self, key, value):
self.__setitem__(key, value)

def __setitem__(self, key, value):
super(Map, self).__setitem__(key, value)
self.__dict__.update({key: value})

def __delattr__(self, item):
self.__delitem__(item)

def __delitem__(self, key):
super(Map, self).__delitem__(key)
del self.__dict__[key]


class Response(Map):
pass

class NetworkAccessManager(object):
"""
This class mimicks httplib2 by using QgsNetworkAccessManager for all
network calls.

The return value is a tuple of (response, content), the first being and
instance of the Response class, the second being a string that contains
the response entity body.

Parameters
----------
debug : bool
verbose logging if True
exception_class : Exception
Custom exception class

Usage 1 (blocking mode)
-----
::
nam = NetworkAccessManager(authcgf)
try:
(response, content) = nam.request('http://www.example.com')
except RequestsException as e:
# Handle exception
pass

Usage 2 (Non blocking mode)
-------------------------
::
NOTE! if blocking mode returns immediatly
it's up to the caller to manage listeners in case
of non blocking mode

nam = NetworkAccessManager(authcgf)
try:
nam.request('http://www.example.com', blocking=False)
nam.reply.finished.connect(a_signal_listener)
except RequestsException as e:
# Handle exception
pass

Get response using method:
nam.httpResult() that return a dictionary with keys:
'status' - http code result come from reply.attribute(QNetworkRequest.HttpStatusCodeAttribute)
'status_code' - http code result come from reply.attribute(QNetworkRequest.HttpStatusCodeAttribute)
'status_message' - reply message string from reply.attribute(QNetworkRequest.HttpReasonPhraseAttribute)
'content' - bytearray returned from reply
'ok' - request success [True, False]
'headers' - Dicionary containing the reply header
'reason' - fomatted message string with reply.errorString()
'exception' - the exception returne dduring execution
"""

def __init__(self, authid=None, disable_ssl_certificate_validation=False, exception_class=None, debug=True):
self.disable_ssl_certificate_validation = disable_ssl_certificate_validation
self.authid = authid
self.reply = None
self.debug = debug
self.exception_class = exception_class
self.on_abort = False
self.blocking_mode = False
self.http_call_result = Response({
'status': 0,
'status_code': 0,
'status_message': '',
'content' : '',
'ok': False,
'headers': {},
'reason': '',
'exception': None,
})

def msg_log(self, msg):
if self.debug:
QgsMessageLog.logMessage(msg, "NetworkAccessManager")

def httpResult(self):
return self.http_call_result

def auth_manager(self):
return QgsApplication.authManager()

def request(self, url, method="GET", body=None, headers=None, redirections=DEFAULT_MAX_REDIRECTS, connection_type=None, blocking=True):
"""
Make a network request by calling QgsNetworkAccessManager.
redirections argument is ignored and is here only for httplib2 compatibility.
"""
self.msg_log(u'http_call request: {0}'.format(url))

self.blocking_mode = blocking
req = QNetworkRequest()
# Avoid double quoting form QUrl
url = urllib.parse.unquote(url)
req.setUrl(QUrl(url))
if headers is not None:
# This fixes a wierd error with compressed content not being correctly
# inflated.
# If you set the header on the QNetworkRequest you are basically telling
# QNetworkAccessManager "I know what I'm doing, please don't do any content
# encoding processing".
# See: https://bugs.webkit.org/show_bug.cgi?id=63696#c1
try:
del headers['Accept-Encoding']
except KeyError:
pass
for k, v in list(headers.items()):
self.msg_log("Setting header %s to %s" % (k, v))
req.setRawHeader(k.encode(), v.encode())
if self.authid:
self.msg_log("Update request w/ authid: {0}".format(self.authid))
self.auth_manager().updateNetworkRequest(req, self.authid)
if self.reply is not None and self.reply.isRunning():
self.reply.close()
if method.lower() == 'delete':
func = getattr(QgsNetworkAccessManager.instance(), 'deleteResource')
else:
func = getattr(QgsNetworkAccessManager.instance(), method.lower())
# Calling the server ...
# Let's log the whole call for debugging purposes:
self.msg_log("Sending %s request to %s" % (method.upper(), req.url().toString()))
self.on_abort = False
headers = {str(h): str(req.rawHeader(h)) for h in req.rawHeaderList()}
for k, v in list(headers.items()):
self.msg_log("%s: %s" % (k, v))
if method.lower() in ['post', 'put']:
if isinstance(body, io.IOBase):
body = body.read()
if isinstance(body, str):
body = body.encode()
if isinstance(body, dict):
body = str(body).encode(encoding='utf-8')
self.reply = func(req, body)
else:
self.reply = func(req)
if self.authid:
self.msg_log("Update reply w/ authid: {0}".format(self.authid))
self.auth_manager().updateNetworkReply(self.reply, self.authid)

# necessary to trap local timout manage by QgsNetworkAccessManager
# calling QgsNetworkAccessManager::abortRequest
QgsNetworkAccessManager.instance().requestTimedOut.connect(self.requestTimedOut)

self.reply.sslErrors.connect(self.sslErrors)
self.reply.finished.connect(self.replyFinished)
self.reply.downloadProgress.connect(self.downloadProgress)

# block if blocking mode otherwise return immediatly
# it's up to the caller to manage listeners in case of no blocking mode
if not self.blocking_mode:
return (None, None)

# Call and block
self.el = QEventLoop()
self.reply.finished.connect(self.el.quit)

# Catch all exceptions (and clean up requests)
try:
self.el.exec_(QEventLoop.ExcludeUserInputEvents)
except Exception as e:
raise e

if self.reply:
self.reply.finished.disconnect(self.el.quit)

# emit exception in case of error
if not self.http_call_result.ok:
if self.http_call_result.exception and not self.exception_class:
raise self.http_call_result.exception
else:
raise self.exception_class(self.http_call_result.reason)

return (self.http_call_result, self.http_call_result.content)

def downloadProgress(self, bytesReceived, bytesTotal):
"""Keep track of the download progress"""
#self.msg_log("downloadProgress %s of %s ..." % (bytesReceived, bytesTotal))
pass

def requestTimedOut(self, reply):
"""Trap the timeout. In Async mode requestTimedOut is called after replyFinished"""
# adapt http_call_result basing on receiving qgs timer timout signal
self.exception_class = RequestsExceptionTimeout
self.http_call_result.exception = RequestsExceptionTimeout("Timeout error")

def replyFinished(self):
err = self.reply.error()
httpStatus = self.reply.attribute(QNetworkRequest.HttpStatusCodeAttribute)
httpStatusMessage = self.reply.attribute(QNetworkRequest.HttpReasonPhraseAttribute)
self.http_call_result.status_code = httpStatus
self.http_call_result.status = httpStatus
self.http_call_result.status_message = httpStatusMessage
for k, v in self.reply.rawHeaderPairs():
self.http_call_result.headers[str(k.data(), encoding='utf-8')] = str(v.data(), encoding='utf-8')
self.http_call_result.headers[str(k.data(), encoding='utf-8').lower()] = str(v.data(), encoding='utf-8')

if err != QNetworkReply.NoError:
# handle error
# check if errorString is empty, if so, then set err string as
# reply dump
if re.match('(.)*server replied: $', self.reply.errorString()):
errString = self.reply.errorString() + self.http_call_result.content
else:
errString = self.reply.errorString()
# check if self.http_call_result.status_code is available (client abort
# does not produce http.status_code)
if self.http_call_result.status_code:
msg = "Network error #{0}: {1}".format(
self.http_call_result.status_code, errString)
else:
msg = "Network error: {0}".format(errString)

self.http_call_result.reason = msg
self.http_call_result.text = str(self.reply.readAll().data(), encoding='utf-8')
self.http_call_result.ok = False
self.msg_log(msg)
# set return exception
if err == QNetworkReply.TimeoutError:
self.http_call_result.exception = RequestsExceptionTimeout(msg)

elif err == QNetworkReply.ConnectionRefusedError:
self.http_call_result.exception = RequestsExceptionConnectionError(msg)

elif err == QNetworkReply.OperationCanceledError:
# request abort by calling NAM.abort() => cancelled by the user
if self.on_abort:
self.http_call_result.exception = RequestsExceptionUserAbort(msg)
else:
self.http_call_result.exception = RequestsException(msg)

else:
self.http_call_result.exception = RequestsException(msg)

# overload exception to the custom exception if available
if self.exception_class:
self.http_call_result.exception = self.exception_class(msg)

else:
# Handle redirections
redirectionUrl = self.reply.attribute(QNetworkRequest.RedirectionTargetAttribute)
if redirectionUrl is not None and redirectionUrl != self.reply.url():
if redirectionUrl.isRelative():
redirectionUrl = self.reply.url().resolved(redirectionUrl)

msg = "Redirected from '{}' to '{}'".format(
self.reply.url().toString(), redirectionUrl.toString())
self.msg_log(msg)

self.reply.deleteLater()
self.reply = None
self.request(redirectionUrl.toString())

# really end request
else:
msg = "Network success #{0}".format(self.reply.error())
self.http_call_result.reason = msg
self.msg_log(msg)

ba = self.reply.readAll()
self.http_call_result.content = bytes(ba)
self.http_call_result.text = str(ba.data(), encoding='utf-8')
self.http_call_result.ok = True

# Let's log the whole response for debugging purposes:
self.msg_log("Got response %s %s from %s" % \
(self.http_call_result.status_code,
self.http_call_result.status_message,
self.reply.url().toString()))
for k, v in list(self.http_call_result.headers.items()):
self.msg_log("%s: %s" % (k, v))
if len(self.http_call_result.content) < 1024:
self.msg_log("Payload :\n%s" % self.http_call_result.text)
else:
self.msg_log("Payload is > 1 KB ...")

# clean reply
if self.reply is not None:
if self.reply.isRunning():
self.reply.close()
self.msg_log("Deleting reply ...")
# Disconnect all slots
self.reply.sslErrors.disconnect(self.sslErrors)
self.reply.finished.disconnect(self.replyFinished)
self.reply.downloadProgress.disconnect(self.downloadProgress)
self.reply.deleteLater()
self.reply = None
else:
self.msg_log("Reply was already deleted ...")

def sslErrors(self, ssl_errors):
"""
Handle SSL errors, logging them if debug is on and ignoring them
if disable_ssl_certificate_validation is set.
"""
if ssl_errors:
for v in ssl_errors:
self.msg_log("SSL Error: %s" % v.errorString())
if self.disable_ssl_certificate_validation:
self.reply.ignoreSslErrors()

def abort(self):
"""
Handle request to cancel HTTP call
"""
if (self.reply and self.reply.isRunning()):
self.on_abort = True
self.reply.abort()

+ 11
- 2
osmtools_processing/isochrones.py View File

@@ -104,12 +104,17 @@ class IsochronesGeoAlg(GeoAlgorithm):
progress.setInfo('Initializing')

simplify = self.getParameterValue(self.IN_SIMPLIFY)
apiKey = self.getParameterValue(self.IN_KEY)
profile = self.getParameterValue(self.IN_PROFILE)
metric = self.getParameterValue(self.IN_METRIC)
ranges = self.getParameterValue(self.IN_RANGES)
ranges = list(map(int, ranges.split(',')))
client = Client(None, apiKey)
provider = {
'key': self.getParameterValue(self.IN_KEY),
'base_url': 'https://api.openrouteservice.org',
'timeout': 60,
}
client = Client(provider, provider['timeout'])

pointLayer = getObjectFromUri(self.getParameterValue(self.IN_POINTS))

# ORS understands WGS84 only, so we convert all points before sending
@@ -123,8 +128,12 @@ class IsochronesGeoAlg(GeoAlgorithm):

progress.setInfo('Processing each selected point')
responses = []
POINTS_REQ = 5
pointsInReq = 0
for feature in features(pointLayer):

# TODO: run 5 points per request!

feature.geometry().transform(transformer)
point = feature.geometry().asPoint()


+ 6
- 1
osmtools_processing/isochrones_join.py View File

@@ -107,7 +107,12 @@ class IsochronesJoinGeoAlg(GeoAlgorithm):
except ValueError: # a string, so we try a fieldname
distFromField = dist

client = Client(None, apiKey)
provider = {
'key': self.getParameterValue(self.IN_KEY),
'base_url': 'https://api.openrouteservice.org',
'timeout': 60,
}
client = Client(provider, provider['timeout'])
pointLayer = getObjectFromUri(self.getParameterValue(self.IN_POINTS))

# ORS understands WGS84 only, so we convert all points before sending

+ 28
- 0
utils/__init__.py View File

@@ -0,0 +1,28 @@
# -*- coding: utf-8 -*-
"""
/***************************************************************************
OSMtools
A QGIS plugin
QGIS client to query openrouteservice
-------------------
begin : 2017-02-01
git sha : $Format:%H$
copyright : (C) 2017 by Nils Nolde
email : nils.nolde@gmail.com
***************************************************************************/

This plugin provides access to the various APIs from OpenRouteService
(https://openrouteservice.org), developed and
maintained by GIScience team at University of Heidelberg, Germany. By using
this plugin you agree to the ORS terms of service
(https://openrouteservice.org/terms-of-service/).

/***************************************************************************
* *
* This program is free software; you can redistribute it and/or modify *
* it under the terms of the GNU General Public License as published by *
* the Free Software Foundation; either version 2 of the License, or *
* (at your option) any later version. *
* *
***************************************************************************/
"""

+ 70
- 0
utils/configmanager.py View File

@@ -0,0 +1,70 @@
# -*- coding: utf-8 -*-
"""
/***************************************************************************
OSMtools
A QGIS plugin
QGIS client to query openrouteservice
-------------------
begin : 2017-02-01
git sha : $Format:%H$
copyright : (C) 2017 by Nils Nolde
email : nils.nolde@gmail.com
***************************************************************************/
This plugin provides access to the various APIs from OpenRouteService
(https://openrouteservice.org), developed and
maintained by GIScience team at University of Heidelberg, Germany. By using
this plugin you agree to the ORS terms of service
(https://openrouteservice.org/terms-of-service/).
/***************************************************************************
* *
* This program is free software; you can redistribute it and/or modify *
* it under the terms of the GNU General Public License as published by *
* the Free Software Foundation; either version 2 of the License, or *
* (at your option) any later version. *
* *
***************************************************************************/
"""
import yaml
import os
# from OSMtools import CONFIG_PATH
CONFIG_PATH = './orstools.config.yml'
def read_config():
"""
Reads config.yml from file and returns the parsed dict.
:returns: Parsed settings dictionary.
:rtype: dict
"""
with open(CONFIG_PATH) as f:
doc = yaml.safe_load(f)
return doc
def write_config(new_config):
"""
Dumps new config
:param new_config: new provider settings after altering in dialog.
:type new_config: dict
"""
with open(CONFIG_PATH, 'w') as f:
yaml.safe_dump(new_config, f)
def write_env_var(key, value):
"""
Update quota env variables
:param key: environment variable to update.
:type key: str
:param value: value for env variable.
:type value: str
"""
os.environ[key] = value

+ 138
- 0
utils/convert.py View File

@@ -0,0 +1,138 @@
# -*- coding: utf-8 -*-
"""
/***************************************************************************
OSMtools
A QGIS plugin
QGIS client to query openrouteservice
-------------------
begin : 2017-02-01
git sha : $Format:%H$
copyright : (C) 2017 by Nils Nolde
email : nils.nolde@gmail.com
***************************************************************************/

This plugin provides access to the various APIs from OpenRouteService
(https://openrouteservice.org), developed and
maintained by GIScience team at University of Heidelberg, Germany. By using
this plugin you agree to the ORS terms of service
(https://openrouteservice.org/terms-of-service/).

/***************************************************************************
* *
* This program is free software; you can redistribute it and/or modify *
* it under the terms of the GNU General Public License as published by *
* the Free Software Foundation; either version 2 of the License, or *
* (at your option) any later version. *
* *
***************************************************************************/
"""


def pipe_list(arg):
"""Convert list of values to pipe-delimited string"""
if not _is_list(arg):
raise TypeError(
"Expected a list or tuple, "
"but got {}".format(type(arg).__name__))
return "|".join(map(str, arg))


def comma_list(arg):
"""Convert list to comma-separated string"""
if not _is_list(arg):
raise TypeError(
"Expected a list or tuple, "
"but got {}".format(type(arg).__name__))
return ",".join(map(str, arg))


def _checkBool(boolean):
"""Check whether passed boolean is a string"""
if boolean not in ["true", "false"]:
raise ValueError("Give boolean as string 'true' or 'false'.")

return


def _format_float(arg):
"""Formats a float value to be as short as possible.

Trims extraneous trailing zeros and period to give API
args the best possible chance of fitting within 2000 char
URL length restrictions.

For example:

format_float(40) -> "40"
format_float(40.0) -> "40"
format_float(40.1) -> "40.1"
format_float(40.001) -> "40.001"
format_float(40.0010) -> "40.001"

:param arg: The lat or lng float.
:type arg: float

:rtype: string
"""
return ("{}".format(round(float(arg), 6)).rstrip("0").rstrip("."))


def build_coords(arg):
"""Converts one or many lng/lat pair(s) to a comma-separated, pipe
delimited string. Coordinates will be rounded to 5 digits.

For example:

convert.build_coords([(151.2069902,-33.8674869),(2.352315,48.513158)])
# '151.20699,-33.86749|2.35232,48.51316'

:param arg: The lat/lon pair(s).
:type arg: list or tuple

:rtype: str
"""
if _is_list(arg):
return pipe_list(_concat_coords(arg))
else:
raise TypeError(
"Expected a list or tuple of lng/lat tuples or lists, "
"but got {}".format(type(arg).__name__))


def _concat_coords(arg):
"""Turn the passed coordinate tuple(s) in comma separated coordinate tuple(s).

:param arg: coordinate pair(s)
:type arg: list or tuple

:rtype: list of strings
"""
if all(_is_list(tup) for tup in arg):
# Check if arg is a list/tuple of lists/tuples
return [comma_list(map(_format_float, tup)) for tup in arg]
else:
return [comma_list(_format_float(coord) for coord in arg)]


def _is_list(arg):
"""Checks if arg is list-like."""
if isinstance(arg, dict):
return False
if isinstance(arg, str): # Python 3-only, as str has __iter__
return False
return (not _has_method(arg, "strip")
and _has_method(arg, "__getitem__")
or _has_method(arg, "__iter__"))


def _has_method(arg, method):
"""Returns true if the given object has a method with the given name.

:param arg: the object

:param method: the method name
:type method: string

:rtype: bool
"""
return hasattr(arg, method) and callable(getattr(arg, method))

+ 91
- 0
utils/exceptions.py View File

@@ -0,0 +1,91 @@
# -*- coding: utf-8 -*-
"""
/***************************************************************************
OSMtools
A QGIS plugin
QGIS client to query openrouteservice
-------------------
begin : 2017-02-01
git sha : $Format:%H$
copyright : (C) 2017 by Nils Nolde
email : nils.nolde@gmail.com
***************************************************************************/

This plugin provides access to the various APIs from OpenRouteService
(https://openrouteservice.org), developed and
maintained by GIScience team at University of Heidelberg, Germany. By using
this plugin you agree to the ORS terms of service
(https://openrouteservice.org/terms-of-service/).

/***************************************************************************
* *
* This program is free software; you can redistribute it and/or modify *
* it under the terms of the GNU General Public License as published by *
* the Free Software Foundation; either version 2 of the License, or *
* (at your option) any later version. *
* *
***************************************************************************/
"""

"""
Defines exceptions that are thrown by the ORS client.
"""


class ApiError(Exception):
"""Represents an exception returned by the remote API."""
def __init__(self, status, message=None):
self.status = status
self.message = message

def __str__(self):
if self.message is None:
return self.status
else:
return "{} ({})".format(self.status, self.message)


class InvalidKey(Exception):
"""only called for 403"""
def __init__(self, status, message):
self.status = status
self.message = message

def __str__(self):
if self.message is None:
return self.status
else:
return "{} ({})".format(self.status, self.message)


class OverQueryLimit(Exception):
"""Signifies that the request failed because the client exceeded its query rate limit."""

def __init__(self, status, message=None):
self.status = status
self.message = message

def __str__(self):
if self.message is None:
return self.status
else:
return "{} ({})".format(self.status, self.message)


class Timeout(Exception):
"""The request timed out."""
pass


class GenericServerError(Exception):
"""Anything else"""

def __init__(self, status, message=None):
self.status = status
self.message = message

def __str__(self):
if self.message is None:
return self.status
else:
return "{} ({})".format(self.status, self.message)

+ 51
- 0
utils/logger.py View File

@@ -0,0 +1,51 @@
# -*- coding: utf-8 -*-
"""
/***************************************************************************
OSMtools
A QGIS plugin
QGIS client to query openrouteservice
-------------------
begin : 2017-02-01
git sha : $Format:%H$
copyright : (C) 2017 by Nils Nolde
email : nils.nolde@gmail.com
***************************************************************************/

This plugin provides access to the various APIs from OpenRouteService
(https://openrouteservice.org), developed and
maintained by GIScience team at University of Heidelberg, Germany. By using
this plugin you agree to the ORS terms of service
(https://openrouteservice.org/terms-of-service/).

/***************************************************************************
* *
* This program is free software; you can redistribute it and/or modify *
* it under the terms of the GNU General Public License as published by *
* the Free Software Foundation; either version 2 of the License, or *
* (at your option) any later version. *
* *
***************************************************************************/
"""

from qgis.core import QgsMessageLog

def log(message, level_in=0):
"""
Writes to QGIS inbuilt logger accessible through panel.

:param message: logging message to write, error or URL.
:type message: str

:param level_in: integer representation of logging level.
:type level_in: int
"""
if level_in == 0:
level = QgsMessageLog.INFO
elif level_in == 1:
level = QgsMessageLog.WARNING
elif level_in == 2:
level = QgsMessageLog.CRITICAL
else:
level = QgsMessageLog.INFO

QgsMessageLog.logMessage(message, 'OSMtools', level)

+ 154
- 0
utils/maptools.py View File

@@ -0,0 +1,154 @@
# -*- coding: utf-8 -*-
"""
/***************************************************************************
OSMtools
A QGIS plugin
QGIS client to query openrouteservice
-------------------
begin : 2017-02-01
git sha : $Format:%H$
copyright : (C) 2017 by Nils Nolde
email : nils.nolde@gmail.com
***************************************************************************/
This plugin provides access to the various APIs from OpenRouteService
(https://openrouteservice.org), developed and
maintained by GIScience team at University of Heidelberg, Germany. By using
this plugin you agree to the ORS terms of service
(https://openrouteservice.org/terms-of-service/).
/***************************************************************************
* *
* This program is free software; you can redistribute it and/or modify *
* it under the terms of the GNU General Public License as published by *
* the Free Software Foundation; either version 2 of the License, or *
* (at your option) any later version. *
* *
***************************************************************************/
"""
from PyQt5.QtCore import pyqtSignal
from PyQt5.QtGui import QCursor, QPixmap, QColor
from PyQt5.QtWidgets import QApplication
from qgis.core import QgsWkbTypes
from qgis.gui import QgsMapToolEmitPoint, QgsRubberBand, QgsVertexMarker
from OSMtools import RESOURCE_PREFIX, DEFAULT_COLOR
from OSMtools.utils import transform
class PointTool(QgsMapToolEmitPoint):
"""Point Map tool to capture mapped coordinates."""
def __init__(self, canvas, button):
"""
:param canvas: current map canvas
:type: QgsMapCanvas
:param button: name of 'Map!' button pressed.
:type button: str
"""
QgsMapToolEmitPoint.__init__(self, canvas)
self.canvas = canvas
self.button = button
self.cursor = QCursor(QPixmap(RESOURCE_PREFIX + 'icon_locate.png').scaledToWidth(48), 24, 24)
canvasClicked = pyqtSignal(['QgsPointXY', 'QString'])
def canvasReleaseEvent(self, event):
#Get the click and emit a transformed point
# mapSettings() was only introduced in QGIS 2.4, keep compatibility
crsSrc = self.canvas.mapSettings().destinationCrs()
point_oldcrs = event.mapPoint()
xform = transform.transformToWGS(crsSrc)
point_newcrs = xform.transform(point_oldcrs)
QApplication.restoreOverrideCursor()
self.canvasClicked.emit(point_newcrs, self.button)
def activate(self):
QApplication.setOverrideCursor(self.cursor)
class LineTool(QgsMapToolEmitPoint):
"""Line Map tool to capture mapped lines."""
def __init__(self, canvas):
"""
:param canvas: current map canvas
:type canvas: QgsMapCanvas
"""
self.canvas = canvas
QgsMapToolEmitPoint.__init__(self, self.canvas)
self.rubberBand = QgsRubberBand(self.canvas, False)
self.rubberBand.setStrokeColor(QColor(DEFAULT_COLOR))
self.rubberBand.setWidth(3)
crsSrc = self.canvas.mapSettings().destinationCrs()
self.transformer = transform.transformToWGS(crsSrc)
self.previous_point = None
self.points = []
self.markers = []
self.reset()
def reset(self):
"""reset rubberband and captured points."""
self.points = []
# self.isEmittingPoint = False
self.rubberBand.reset(QgsWkbTypes.LineGeometry)
def add_marker(self, point):
"""
Adds a clicked marker to the map canvas.
:param point: point clicked by the user.
:type point: QgsPointXY
"""
new_marker = QgsVertexMarker(self.canvas)
new_marker.setCenter(point)
new_marker.setIconType(QgsVertexMarker.ICON_CROSS)
new_marker.setIconSize(10)
new_marker.setFillColor(QColor('#485bea'))
new_marker.setColor(QColor('#000000'))
self.markers.append(new_marker)
pointDrawn = pyqtSignal(["QgsPointXY", "int"])
def canvasReleaseEvent(self, e):
"""Add marker to canvas and shows line."""
new_point = self.toMapCoordinates(e.pos())
self.add_marker(new_point)
self.points.append(new_point)
self.pointDrawn.emit(self.transformer.transform(new_point), self.points.index(new_point))
self.showLine()
def showLine(self):
"""Builds rubberband from all points and adds it to the map canvas."""
self.rubberBand.reset(QgsWkbTypes.LineGeometry)
for point in self.points:
if point == self.points[-1]:
self.rubberBand.addPoint(point, True)
self.rubberBand.addPoint(point, False)
self.rubberBand.show()
doubleClicked = pyqtSignal(['int'])
def canvasDoubleClickEvent(self, e):
"""Ends line drawing and deletes rubberband and markers from map canvas."""
self.doubleClicked.emit(len(self.points))
self.canvas.scene().removeItem(self.rubberBand)
if self.markers:
for marker in self.markers:
self.canvas.scene().removeItem(marker)
def deactivate(self):
super(LineTool, self).deactivate()
self.deactivated.emit()

+ 49
- 0
utils/transform.py View File

@@ -0,0 +1,49 @@
# -*- coding: utf-8 -*-
"""
/***************************************************************************
OSMtools
A QGIS plugin
QGIS client to query openrouteservice
-------------------
begin : 2017-02-01
git sha : $Format:%H$
copyright : (C) 2017 by Nils Nolde
email : nils.nolde@gmail.com
***************************************************************************/

This plugin provides access to the various APIs from OpenRouteService
(https://openrouteservice.org), developed and
maintained by GIScience team at University of Heidelberg, Germany. By using
this plugin you agree to the ORS terms of service
(https://openrouteservice.org/terms-of-service/).

/***************************************************************************
* *
* This program is free software; you can redistribute it and/or modify *
* it under the terms of the GNU General Public License as published by *
* the Free Software Foundation; either version 2 of the License, or *
* (at your option) any later version. *
* *
***************************************************************************/
"""

from qgis.core import (QgsCoordinateReferenceSystem,
QgsCoordinateTransform,
QgsProject
)


def transformToWGS(old_crs):
"""
Returns a transformer to WGS84

:param old_crs: CRS to transfrom from
:type old_crs: QgsCoordinateReferenceSystem

:returns: transformer to use in various modules.
:rtype: QgsCoordinateTransform
"""
outCrs = QgsCoordinateReferenceSystem(4326)
xformer = QgsCoordinateTransform(old_crs, outCrs, QgsProject.instance())

return xformer

Loading…
Cancel
Save