From 44b5084e1a5847be40609adde50b048f59dd5056 Mon Sep 17 00:00:00 2001 From: Leah Cole Date: Fri, 6 Mar 2020 13:15:54 -0800 Subject: [PATCH 1/4] chore: update all dependencies --- dlp/requirements.txt | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/dlp/requirements.txt b/dlp/requirements.txt index dccabd5de6b..ca423e2c176 100644 --- a/dlp/requirements.txt +++ b/dlp/requirements.txt @@ -1,5 +1,5 @@ -google-cloud-dlp==0.12.1 -google-cloud-storage==1.19.1 -google-cloud-pubsub==1.0.0 -google-cloud-datastore==1.9.0 -google-cloud-bigquery==1.20.0 +google-cloud-dlp==0.13.0 +google-cloud-storage==1.26.0 +google-cloud-pubsub==1.3.1 +google-cloud-datastore==1.11.0 +google-cloud-bigquery==1.24.0 From 48d34db7982acff30b603baa781aaac83b56fd4f Mon Sep 17 00:00:00 2001 From: Leah Cole Date: Fri, 6 Mar 2020 16:57:32 -0800 Subject: [PATCH 2/4] fix: look for LRO id instead of waiting for finish --- dlp/inspect_content.py | 2 +- dlp/inspect_content_test.py | 9 +++++++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/dlp/inspect_content.py b/dlp/inspect_content.py index 336f798128b..b1e97ae3147 100644 --- a/dlp/inspect_content.py +++ b/dlp/inspect_content.py @@ -473,7 +473,7 @@ def inspect_gcs_file( } operation = dlp.create_dlp_job(parent, inspect_job=inspect_job) - + print("Inspection operation started: {}".format(operation.name)) # Create a Pub/Sub client and find the subscription. The subscription is # expected to already be listening to the topic. subscriber = google.cloud.pubsub.SubscriberClient() diff --git a/dlp/inspect_content_test.py b/dlp/inspect_content_test.py index d444c22e1f0..d75ad7e9246 100644 --- a/dlp/inspect_content_test.py +++ b/dlp/inspect_content_test.py @@ -20,6 +20,7 @@ import google.api_core.exceptions import google.cloud.bigquery import google.cloud.datastore +import google.cloud.dlp_v2 import google.cloud.exceptions import google.cloud.pubsub import google.cloud.storage @@ -290,7 +291,6 @@ def test_inspect_image_file(capsys): assert "Info type: PHONE_NUMBER" in out -@flaky def test_inspect_gcs_file(bucket, topic_id, subscription_id, capsys): inspect_content.inspect_gcs_file( GCLOUD_PROJECT, @@ -303,7 +303,12 @@ def test_inspect_gcs_file(bucket, topic_id, subscription_id, capsys): ) out, _ = capsys.readouterr() - assert "Info type: EMAIL_ADDRESS" in out + assert "Inspection operation started" in out + # Cancel the operation + operation_id = out.split("Inspection operation started: ")[1].split("\n")[0] + print(operation_id) + client = google.cloud.dlp_v2.DlpServiceClient() + client.cancel_dlp_job(operation_id) @flaky From f788c105e8a3c1d56b8dd59e091304c81e9bcb58 Mon Sep 17 00:00:00 2001 From: Leah Cole Date: Fri, 6 Mar 2020 17:03:29 -0800 Subject: [PATCH 3/4] ran black --- dlp/inspect_content_test.py | 41 ++++++++++++++++++++++--------------- 1 file changed, 24 insertions(+), 17 deletions(-) diff --git a/dlp/inspect_content_test.py b/dlp/inspect_content_test.py index d75ad7e9246..152f4de14df 100644 --- a/dlp/inspect_content_test.py +++ b/dlp/inspect_content_test.py @@ -95,9 +95,7 @@ def subscription_id(topic_id): # Subscribes to a topic. subscriber = google.cloud.pubsub.SubscriberClient() topic_path = subscriber.topic_path(GCLOUD_PROJECT, topic_id) - subscription_path = subscriber.subscription_path( - GCLOUD_PROJECT, SUBSCRIPTION_ID - ) + subscription_path = subscriber.subscription_path(GCLOUD_PROJECT, SUBSCRIPTION_ID) try: subscriber.create_subscription(subscription_path, topic_path) except google.api_core.exceptions.AlreadyExists: @@ -311,7 +309,6 @@ def test_inspect_gcs_file(bucket, topic_id, subscription_id, capsys): client.cancel_dlp_job(operation_id) -@flaky def test_inspect_gcs_file_with_custom_info_types( bucket, topic_id, subscription_id, capsys ): @@ -331,14 +328,16 @@ def test_inspect_gcs_file_with_custom_info_types( ) out, _ = capsys.readouterr() - assert "Info type: CUSTOM_DICTIONARY_0" in out - assert "Info type: CUSTOM_REGEX_0" in out + assert "Inspection operation started" in out + # Cancel the operation + operation_id = out.split("Inspection operation started: ")[1].split("\n")[0] + print(operation_id) + client = google.cloud.dlp_v2.DlpServiceClient() + client.cancel_dlp_job(operation_id) -@flaky -def test_inspect_gcs_file_no_results( - bucket, topic_id, subscription_id, capsys -): + +def test_inspect_gcs_file_no_results(bucket, topic_id, subscription_id, capsys): inspect_content.inspect_gcs_file( GCLOUD_PROJECT, bucket.name, @@ -350,7 +349,13 @@ def test_inspect_gcs_file_no_results( ) out, _ = capsys.readouterr() - assert "No findings" in out + + assert "Inspection operation started" in out + # Cancel the operation + operation_id = out.split("Inspection operation started: ")[1].split("\n")[0] + print(operation_id) + client = google.cloud.dlp_v2.DlpServiceClient() + client.cancel_dlp_job(operation_id) @pytest.mark.skip(reason="nondeterministically failing") @@ -368,7 +373,6 @@ def test_inspect_gcs_image_file(bucket, topic_id, subscription_id, capsys): assert "Info type: EMAIL_ADDRESS" in out -@flaky def test_inspect_gcs_multiple_files(bucket, topic_id, subscription_id, capsys): inspect_content.inspect_gcs_file( GCLOUD_PROJECT, @@ -380,14 +384,17 @@ def test_inspect_gcs_multiple_files(bucket, topic_id, subscription_id, capsys): ) out, _ = capsys.readouterr() - assert "Info type: EMAIL_ADDRESS" in out - assert "Info type: PHONE_NUMBER" in out + + assert "Inspection operation started" in out + # Cancel the operation + operation_id = out.split("Inspection operation started: ")[1].split("\n")[0] + print(operation_id) + client = google.cloud.dlp_v2.DlpServiceClient() + client.cancel_dlp_job(operation_id) @flaky -def test_inspect_datastore( - datastore_project, topic_id, subscription_id, capsys -): +def test_inspect_datastore(datastore_project, topic_id, subscription_id, capsys): @eventually_consistent.call def _(): inspect_content.inspect_datastore( From f743686fcf0a47584c04521104d246ec94007fe9 Mon Sep 17 00:00:00 2001 From: Leah Cole Date: Mon, 9 Mar 2020 13:10:13 -0700 Subject: [PATCH 4/4] add eventually consistent to datastore --- dlp/inspect_content_test.py | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/dlp/inspect_content_test.py b/dlp/inspect_content_test.py index 152f4de14df..ad493ecce71 100644 --- a/dlp/inspect_content_test.py +++ b/dlp/inspect_content_test.py @@ -414,17 +414,19 @@ def _(): def test_inspect_datastore_no_results( datastore_project, topic_id, subscription_id, capsys ): - inspect_content.inspect_datastore( - GCLOUD_PROJECT, - datastore_project, - DATASTORE_KIND, - topic_id, - subscription_id, - ["PHONE_NUMBER"], - ) + @eventually_consistent.call + def _(): + inspect_content.inspect_datastore( + GCLOUD_PROJECT, + datastore_project, + DATASTORE_KIND, + topic_id, + subscription_id, + ["PHONE_NUMBER"], + ) - out, _ = capsys.readouterr() - assert "No findings" in out + out, _ = capsys.readouterr() + assert "No findings" in out @pytest.mark.skip(reason="unknown issue")