Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly

Appearance settings

Commit 14364a5

Browse filesBrowse files
gkevinzhenggcf-owl-bot[bot]parthea
authored
test: Added cleanup of old sink storage buckets (#991)
* test: Added cleanup of old sink storage buckets * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md * add list_buckets threshold --------- Co-authored-by: Owl Bot <gcf-owl-bot[bot]@users.noreply.github.com> Co-authored-by: Anthonios Partheniou <partheniou@google.com>
1 parent f4fb25a commit 14364a5
Copy full SHA for 14364a5

File tree

Expand file treeCollapse file tree

1 file changed

+23
-6
lines changed
Filter options
Expand file treeCollapse file tree

1 file changed

+23
-6
lines changed

‎samples/snippets/export_test.py

Copy file name to clipboardExpand all lines: samples/snippets/export_test.py
+23-6Lines changed: 23 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@
1919
import time
2020

2121
import backoff
22-
from google.cloud import logging
22+
from google.cloud import logging, storage
2323
import pytest
2424

2525
import export
@@ -34,6 +34,10 @@
3434
# old sink, in seconds
3535
CLEANUP_THRESHOLD = 7200 # 2 hours
3636

37+
# Max buckets to delete at a time, to mitigate operation timeout
38+
# issues. To turn off in the future, set to None.
39+
MAX_BUCKETS = 1500
40+
3741

3842
def _random_id():
3943
return "".join(
@@ -46,8 +50,8 @@ def _create_sink_name():
4650

4751

4852
@backoff.on_exception(backoff.expo, Exception, max_time=60, raise_on_giveup=False)
49-
def _delete_sink(sink):
50-
sink.delete()
53+
def _delete_object(obj):
54+
obj.delete()
5155

5256

5357
# Runs once for entire test suite
@@ -62,7 +66,20 @@ def cleanup_old_sinks():
6266
if match:
6367
sink_timestamp = int(match.group(1))
6468
if TIMESTAMP - sink_timestamp > CLEANUP_THRESHOLD:
65-
_delete_sink(sink)
69+
_delete_object(sink)
70+
71+
storage_client = storage.Client()
72+
73+
# See _sink_storage_setup in usage_guide.py for details about how
74+
# sinks are named.
75+
test_bucket_name_regex = r"^sink\-storage\-(\d+)$"
76+
for bucket in storage_client.list_buckets(max_results=MAX_BUCKETS):
77+
match = re.match(test_bucket_name_regex, bucket.name)
78+
if match:
79+
# Bucket timestamp is int(time.time() * 1000)
80+
bucket_timestamp = int(match.group(1))
81+
if TIMESTAMP - bucket_timestamp // 1000 > CLEANUP_THRESHOLD:
82+
_delete_object(bucket)
6683

6784

6885
@pytest.fixture
@@ -79,7 +96,7 @@ def example_sink(cleanup_old_sinks):
7996

8097
yield sink
8198

82-
_delete_sink(sink)
99+
_delete_object(sink)
83100

84101

85102
def test_list(example_sink, capsys):
@@ -99,7 +116,7 @@ def test_create(capsys):
99116
export.create_sink(sink_name, BUCKET, TEST_SINK_FILTER)
100117
# Clean-up the temporary sink.
101118
finally:
102-
_delete_sink(logging.Client().sink(sink_name))
119+
_delete_object(logging.Client().sink(sink_name))
103120

104121
out, _ = capsys.readouterr()
105122
assert sink_name in out

0 commit comments

Comments
0 (0)
Morty Proxy This is a proxified and sanitized view of the page, visit original site.