From 49825685c2f5b9d8faf223b4b62c39286e6d61e7 Mon Sep 17 00:00:00 2001 From: Kevin Zheng Date: Mon, 21 Apr 2025 15:46:20 +0000 Subject: [PATCH 1/3] test: Added cleanup of old sink storage buckets --- samples/snippets/export_test.py | 25 +++++++++++++++++++------ 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/samples/snippets/export_test.py b/samples/snippets/export_test.py index c21fab2d..d4fe311a 100644 --- a/samples/snippets/export_test.py +++ b/samples/snippets/export_test.py @@ -19,7 +19,7 @@ import time import backoff -from google.cloud import logging +from google.cloud import logging, storage import pytest import export @@ -46,8 +46,8 @@ def _create_sink_name(): @backoff.on_exception(backoff.expo, Exception, max_time=60, raise_on_giveup=False) -def _delete_sink(sink): - sink.delete() +def _delete_object(obj): + obj.delete() # Runs once for entire test suite @@ -62,7 +62,20 @@ def cleanup_old_sinks(): if match: sink_timestamp = int(match.group(1)) if TIMESTAMP - sink_timestamp > CLEANUP_THRESHOLD: - _delete_sink(sink) + _delete_object(sink) + + storage_client = storage.Client() + + # See _sink_storage_setup in usage_guide.py for details about how + # sinks are named. + test_bucket_name_regex = r"^sink\-storage\-(\d+)$" + for bucket in storage_client.list_buckets(): + match = re.match(test_bucket_name_regex, bucket.name) + if match: + # Bucket timestamp is int(time.time() * 1000) + bucket_timestamp = int(match.group(1)) + if TIMESTAMP - bucket_timestamp//1000 > CLEANUP_THRESHOLD: + _delete_object(bucket) @pytest.fixture @@ -79,7 +92,7 @@ def example_sink(cleanup_old_sinks): yield sink - _delete_sink(sink) + _delete_object(sink) def test_list(example_sink, capsys): @@ -99,7 +112,7 @@ def test_create(capsys): export.create_sink(sink_name, BUCKET, TEST_SINK_FILTER) # Clean-up the temporary sink. finally: - _delete_sink(logging.Client().sink(sink_name)) + _delete_object(logging.Client().sink(sink_name)) out, _ = capsys.readouterr() assert sink_name in out From 8e36f878c5771d332e0c3622e5d91974d6953b97 Mon Sep 17 00:00:00 2001 From: Owl Bot Date: Mon, 21 Apr 2025 15:51:13 +0000 Subject: [PATCH 2/3] =?UTF-8?q?=F0=9F=A6=89=20Updates=20from=20OwlBot=20po?= =?UTF-8?q?st-processor?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md --- samples/snippets/export_test.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/samples/snippets/export_test.py b/samples/snippets/export_test.py index d4fe311a..4f748ba9 100644 --- a/samples/snippets/export_test.py +++ b/samples/snippets/export_test.py @@ -63,7 +63,7 @@ def cleanup_old_sinks(): sink_timestamp = int(match.group(1)) if TIMESTAMP - sink_timestamp > CLEANUP_THRESHOLD: _delete_object(sink) - + storage_client = storage.Client() # See _sink_storage_setup in usage_guide.py for details about how @@ -74,7 +74,7 @@ def cleanup_old_sinks(): if match: # Bucket timestamp is int(time.time() * 1000) bucket_timestamp = int(match.group(1)) - if TIMESTAMP - bucket_timestamp//1000 > CLEANUP_THRESHOLD: + if TIMESTAMP - bucket_timestamp // 1000 > CLEANUP_THRESHOLD: _delete_object(bucket) From fe6044cc63d16caf729d453cb5e2c7fe7da9ffe8 Mon Sep 17 00:00:00 2001 From: Kevin Zheng Date: Wed, 21 May 2025 14:32:18 +0000 Subject: [PATCH 3/3] add list_buckets threshold --- samples/snippets/export_test.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/samples/snippets/export_test.py b/samples/snippets/export_test.py index 4f748ba9..845359e0 100644 --- a/samples/snippets/export_test.py +++ b/samples/snippets/export_test.py @@ -34,6 +34,10 @@ # old sink, in seconds CLEANUP_THRESHOLD = 7200 # 2 hours +# Max buckets to delete at a time, to mitigate operation timeout +# issues. To turn off in the future, set to None. +MAX_BUCKETS = 1500 + def _random_id(): return "".join( @@ -69,7 +73,7 @@ def cleanup_old_sinks(): # See _sink_storage_setup in usage_guide.py for details about how # sinks are named. test_bucket_name_regex = r"^sink\-storage\-(\d+)$" - for bucket in storage_client.list_buckets(): + for bucket in storage_client.list_buckets(max_results=MAX_BUCKETS): match = re.match(test_bucket_name_regex, bucket.name) if match: # Bucket timestamp is int(time.time() * 1000)