Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly
Appearance settings

Commit 8f4c5ef

Browse filesBrowse files
Jon Wayne Parrottlandrito
Jon Wayne Parrott
authored and
landrito
committed
Add google.api.core.retry with base retry functionality (googleapis#3819)
Add google.api.core.retry with base retry functionality Additionally: * Add google.api.core.exceptions.RetryError * Add google.api.core.helpers package * Add google.api.core.helpers.datetime_helpers module
1 parent cd60801 commit 8f4c5ef
Copy full SHA for 8f4c5ef

File tree

Expand file treeCollapse file tree

7 files changed

+344
-0
lines changed
Open diff view settings
Filter options
Expand file treeCollapse file tree

7 files changed

+344
-0
lines changed
Open diff view settings
Collapse file

‎core/google/api/core/exceptions.py‎

Copy file name to clipboardExpand all lines: core/google/api/core/exceptions.py
+23Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,29 @@ class GoogleAPIError(Exception):
4040
pass
4141

4242

43+
@six.python_2_unicode_compatible
44+
class RetryError(GoogleAPIError):
45+
"""Raised when a function has exhausted all of its available retries.
46+
47+
Args:
48+
message (str): The exception message.
49+
cause (Exception): The last exception raised when retring the
50+
function.
51+
"""
52+
def __init__(self, message, cause):
53+
super(RetryError, self).__init__(message)
54+
self.message = message
55+
self._cause = cause
56+
57+
@property
58+
def cause(self):
59+
"""The last exception raised when retrying the function."""
60+
return self._cause
61+
62+
def __str__(self):
63+
return '{}, last exception: {}'.format(self.message, self.cause)
64+
65+
4366
class _GoogleAPICallErrorMeta(type):
4467
"""Metaclass for registering GoogleAPICallError subclasses."""
4568
def __new__(mcs, name, bases, class_dict):
Collapse file

‎core/google/api/core/helpers/__init__.py‎

Copy file name to clipboardExpand all lines: core/google/api/core/helpers/__init__.py
Whitespace-only changes.
Collapse file

‎core/google/api/core/helpers/datetime_helpers.py‎

Copy file name to clipboard
+22Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
# Copyright 2017 Google Inc.
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
"""Helpers for :mod:`datetime`."""
16+
17+
import datetime
18+
19+
20+
def utcnow():
21+
"""A :meth:`datetime.datetime.utcnow()` alias to allow mocking in tests."""
22+
return datetime.datetime.utcnow()
Collapse file

‎core/google/api/core/retry.py‎

Copy file name to clipboard
+148Lines changed: 148 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,148 @@
1+
# Copyright 2017 Google Inc.
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
"""Helpers for retrying functions with exponential back-off."""
16+
17+
import datetime
18+
import logging
19+
import random
20+
import time
21+
22+
import six
23+
24+
from google.api.core import exceptions
25+
from google.api.core.helpers import datetime_helpers
26+
27+
_LOGGER = logging.getLogger(__name__)
28+
_DEFAULT_MAX_JITTER = 0.2
29+
30+
31+
def if_exception_type(*exception_types):
32+
"""Creates a predicate to check if the exception is of a given type.
33+
34+
Args:
35+
exception_types (Sequence[type]): The exception types to check for.
36+
37+
Returns:
38+
Callable[Exception]: A predicate that returns True if the provided
39+
exception is of the given type(s).
40+
"""
41+
def inner(exception):
42+
"""Bound predicate for checking an exception type."""
43+
return isinstance(exception, exception_types)
44+
return inner
45+
46+
47+
# pylint: disable=invalid-name
48+
# Pylint sees this as a constant, but it is also an alias that should be
49+
# considered a function.
50+
if_transient_error = if_exception_type((
51+
exceptions.InternalServerError,
52+
exceptions.TooManyRequests))
53+
"""A predicate that checks if an exception is a transient API error.
54+
55+
The following server errors are considered transient:
56+
57+
- :class:`google.api.core.exceptions.InternalServerError` - HTTP 500, gRPC
58+
``INTERNAL(13)`` and its subclasses.
59+
- :class:`google.api.core.exceptions.TooManyRequests` - HTTP 429
60+
- :class:`google.api.core.exceptions.ResourceExhausted` - gRPC
61+
``RESOURCE_EXHAUSTED(8)``
62+
"""
63+
# pylint: enable=invalid-name
64+
65+
66+
def exponential_sleep_generator(
67+
initial, maximum, multiplier=2, jitter=_DEFAULT_MAX_JITTER):
68+
"""Generates sleep intervals based on the exponential back-off algorithm.
69+
70+
This implements the `Truncated Exponential Back-off`_ algorithm.
71+
72+
.. _Truncated Exponential Back-off:
73+
https://cloud.google.com/storage/docs/exponential-backoff
74+
75+
Args:
76+
initial (float): The minimum about of time to delay. This must
77+
be greater than 0.
78+
maximum (float): The maximum about of time to delay.
79+
multiplier (float): The multiplier applied to the delay.
80+
jitter (float): The maximum about of randomness to apply to the delay.
81+
82+
Yields:
83+
float: successive sleep intervals.
84+
"""
85+
delay = initial
86+
while True:
87+
yield delay
88+
delay = min(
89+
delay * multiplier + random.uniform(0, jitter), maximum)
90+
91+
92+
def retry_target(target, predicate, sleep_generator, deadline):
93+
"""Call a function and retry if it fails.
94+
95+
This is the lowest-level retry helper. Generally, you'll use the
96+
higher-level retry helper :class:`Retry`.
97+
98+
Args:
99+
target(Callable): The function to call and retry. This must be a
100+
nullary function - apply arguments with `functools.partial`.
101+
predicate (Callable[Exception]): A callable used to determine if an
102+
exception raised by the target should be considered retryable.
103+
It should return True to retry or False otherwise.
104+
sleep_generator (Iterator[float]): An infinite iterator that determines
105+
how long to sleep between retries.
106+
deadline (float): How long to keep retrying the target.
107+
108+
Returns:
109+
Any: the return value of the target function.
110+
111+
Raises:
112+
google.api.core.RetryError: If the deadline is exceeded while retrying.
113+
ValueError: If the sleep generator stops yielding values.
114+
Exception: If the target raises a method that isn't retryable.
115+
"""
116+
if deadline is not None:
117+
deadline_datetime = (
118+
datetime_helpers.utcnow() + datetime.timedelta(seconds=deadline))
119+
else:
120+
deadline_datetime = None
121+
122+
last_exc = None
123+
124+
for sleep in sleep_generator:
125+
try:
126+
return target()
127+
128+
# pylint: disable=broad-except
129+
# This function explicitly must deal with broad exceptions.
130+
except Exception as exc:
131+
if not predicate(exc):
132+
raise
133+
last_exc = exc
134+
135+
now = datetime_helpers.utcnow()
136+
if deadline_datetime is not None and deadline_datetime < now:
137+
six.raise_from(
138+
exceptions.RetryError(
139+
'Deadline of {:.1f}s exceeded while calling {}'.format(
140+
deadline, target),
141+
last_exc),
142+
last_exc)
143+
144+
_LOGGER.debug('Retrying due to {}, sleeping {:.1f}s ...'.format(
145+
last_exc, sleep))
146+
time.sleep(sleep)
147+
148+
raise ValueError('Sleep generator stopped yielding sleep values.')
Collapse file

‎core/tests/unit/api_core/helpers/__init__.py‎

Copy file name to clipboardExpand all lines: core/tests/unit/api_core/helpers/__init__.py
Whitespace-only changes.
Collapse file

‎core/tests/unit/api_core/helpers/test_datetime_helpers.py‎

Copy file name to clipboard
+22Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
# Copyright 2017, Google Inc.
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
import datetime
16+
17+
from google.api.core.helpers import datetime_helpers
18+
19+
20+
def test_utcnow():
21+
result = datetime_helpers.utcnow()
22+
assert isinstance(result, datetime.datetime)
Collapse file

‎core/tests/unit/api_core/test_retry.py‎

Copy file name to clipboard
+129Lines changed: 129 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,129 @@
1+
# Copyright 2017 Google Inc.
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
import datetime
16+
import itertools
17+
18+
import mock
19+
import pytest
20+
21+
from google.api.core import exceptions
22+
from google.api.core import retry
23+
24+
25+
def test_if_exception_type():
26+
predicate = retry.if_exception_type(ValueError)
27+
28+
assert predicate(ValueError())
29+
assert not predicate(TypeError())
30+
31+
32+
def test_if_exception_type_multiple():
33+
predicate = retry.if_exception_type(ValueError, TypeError)
34+
35+
assert predicate(ValueError())
36+
assert predicate(TypeError())
37+
assert not predicate(RuntimeError())
38+
39+
40+
def test_if_transient_error():
41+
assert retry.if_transient_error(exceptions.InternalServerError(''))
42+
assert retry.if_transient_error(exceptions.TooManyRequests(''))
43+
assert not retry.if_transient_error(exceptions.InvalidArgument(''))
44+
45+
46+
def test_exponential_sleep_generator_base_2():
47+
gen = retry.exponential_sleep_generator(
48+
1, 60, 2, jitter=0.0)
49+
50+
result = list(itertools.islice(gen, 8))
51+
assert result == [1, 2, 4, 8, 16, 32, 60, 60]
52+
53+
54+
@mock.patch('random.uniform')
55+
def test_exponential_sleep_generator_jitter(uniform):
56+
uniform.return_value = 1
57+
gen = retry.exponential_sleep_generator(
58+
1, 60, 2, jitter=2.2)
59+
60+
result = list(itertools.islice(gen, 7))
61+
assert result == [1, 3, 7, 15, 31, 60, 60]
62+
uniform.assert_called_with(0.0, 2.2)
63+
64+
65+
@mock.patch('time.sleep')
66+
@mock.patch(
67+
'google.api.core.helpers.datetime_helpers.utcnow',
68+
return_value=datetime.datetime.min)
69+
def test_retry_target_success(utcnow, sleep):
70+
predicate = retry.if_exception_type(ValueError)
71+
call_count = [0]
72+
73+
def target():
74+
call_count[0] += 1
75+
if call_count[0] < 3:
76+
raise ValueError()
77+
return 42
78+
79+
result = retry.retry_target(target, predicate, range(10), None)
80+
81+
assert result == 42
82+
assert call_count[0] == 3
83+
sleep.assert_has_calls([mock.call(0), mock.call(1)])
84+
85+
86+
@mock.patch('time.sleep')
87+
@mock.patch(
88+
'google.api.core.helpers.datetime_helpers.utcnow',
89+
return_value=datetime.datetime.min)
90+
def test_retry_target_non_retryable_error(utcnow, sleep):
91+
predicate = retry.if_exception_type(ValueError)
92+
exception = TypeError()
93+
target = mock.Mock(side_effect=exception)
94+
95+
with pytest.raises(TypeError) as exc_info:
96+
retry.retry_target(target, predicate, range(10), None)
97+
98+
assert exc_info.value == exception
99+
sleep.assert_not_called()
100+
101+
102+
@mock.patch('time.sleep')
103+
@mock.patch(
104+
'google.api.core.helpers.datetime_helpers.utcnow')
105+
def test_retry_target_deadline_exceeded(utcnow, sleep):
106+
predicate = retry.if_exception_type(ValueError)
107+
exception = ValueError('meep')
108+
target = mock.Mock(side_effect=exception)
109+
# Setup the timeline so that the first call takes 5 seconds but the second
110+
# call takes 6, which puts the retry over the deadline.
111+
utcnow.side_effect = [
112+
# The first call to utcnow establishes the start of the timeline.
113+
datetime.datetime.min,
114+
datetime.datetime.min + datetime.timedelta(seconds=5),
115+
datetime.datetime.min + datetime.timedelta(seconds=11)]
116+
117+
with pytest.raises(exceptions.RetryError) as exc_info:
118+
retry.retry_target(target, predicate, range(10), deadline=10)
119+
120+
assert exc_info.value.cause == exception
121+
assert exc_info.match('Deadline of 10.0s exceeded')
122+
assert exc_info.match('last exception: meep')
123+
assert target.call_count == 2
124+
125+
126+
def test_retry_target_bad_sleep_generator():
127+
with pytest.raises(ValueError, match='Sleep generator'):
128+
retry.retry_target(
129+
mock.sentinel.target, mock.sentinel.predicate, [], None)

0 commit comments

Comments
0 (0)
Morty Proxy This is a proxified and sanitized view of the page, visit original site.