From 42b7bb2a1162a4c8ed18ebf03dee0b9970e117c5 Mon Sep 17 00:00:00 2001 From: Takashi Matsuo Date: Mon, 29 Jun 2020 23:32:03 +0000 Subject: [PATCH] fix(monitoring): also retry upon DeadlineExceeded fixes #4180 --- monitoring/api/v3/alerts-client/snippets_test.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/monitoring/api/v3/alerts-client/snippets_test.py b/monitoring/api/v3/alerts-client/snippets_test.py index b69a1b5fd38..550a8dc9759 100644 --- a/monitoring/api/v3/alerts-client/snippets_test.py +++ b/monitoring/api/v3/alerts-client/snippets_test.py @@ -19,6 +19,7 @@ import time from google.api_core.exceptions import Aborted +from google.api_core.exceptions import DeadlineExceeded from google.api_core.exceptions import NotFound from google.api_core.exceptions import ServiceUnavailable from google.cloud import monitoring_v3 @@ -38,12 +39,13 @@ def random_name(length): [random.choice(string.ascii_lowercase) for i in range(length)]) -def retry_if_aborted(exception): - return isinstance(exception, (Aborted, ServiceUnavailable)) +def retry_on_exceptions(exception): + return isinstance( + exception, (Aborted, ServiceUnavailable, DeadlineExceeded)) def delay_on_aborted(err, *args): - if retry_if_aborted(err[1]): + if retry_on_exceptions(err[1]): # add randomness for avoiding continuous conflict time.sleep(5 + (random.randint(0, 9) * 0.1)) return True @@ -64,7 +66,8 @@ def __init__(self): def __enter__(self): @retry(wait_exponential_multiplier=1000, wait_exponential_max=10000, - stop_max_attempt_number=10, retry_on_exception=retry_if_aborted) + stop_max_attempt_number=10, + retry_on_exception=retry_on_exceptions) def setup(): # Create a policy. policy = monitoring_v3.types.alert_pb2.AlertPolicy() @@ -89,7 +92,8 @@ def setup(): def __exit__(self, type, value, traceback): # Delete the policy and channel we created. @retry(wait_exponential_multiplier=1000, wait_exponential_max=10000, - stop_max_attempt_number=10, retry_on_exception=retry_if_aborted) + stop_max_attempt_number=10, + retry_on_exception=retry_on_exceptions) def teardown(): try: self.alert_policy_client.delete_alert_policy(