Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly
Appearance settings

Commit ecdaa93

Browse filesBrowse files
d-googparthea
d-goog
and
parthea
authored
feat(storagetransfer): Add POSIX & Manifest Samples (GoogleCloudPlatform#8091)
* feat: add transfer manifest samples * feat: POSIX samples * fix: `manifest_file` * fix: `transfer_between_posix` call * fix: typo `sink_bucket` -> `destination_bucket` * Update storagetransfer/conftest.py Co-authored-by: Anthonios Partheniou <partheniou@google.com> * doc: fix doc for POSIX to POSIX Co-authored-by: Anthonios Partheniou <partheniou@google.com>
1 parent 13596b8 commit ecdaa93
Copy full SHA for ecdaa93
Expand file treeCollapse file tree

9 files changed

+665
-0
lines changed

‎storagetransfer/conftest.py

Copy file name to clipboardExpand all lines: storagetransfer/conftest.py
+55Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -165,3 +165,58 @@ def destination_bucket(gcs_bucket: storage.Bucket, sts_service_account: str):
165165
gcs_bucket.set_iam_policy(policy)
166166

167167
yield gcs_bucket
168+
169+
170+
@pytest.fixture(scope='module')
171+
def intermediate_bucket(gcs_bucket: storage.Bucket, sts_service_account: str):
172+
"""
173+
Yields and auto-cleans up a GCS bucket preconfigured with necessary
174+
STS service account write perms
175+
"""
176+
177+
# Setup policy for STS
178+
member: str = f"serviceAccount:{sts_service_account}"
179+
objectViewer = "roles/storage.objectViewer"
180+
bucketReader = "roles/storage.legacyBucketReader"
181+
bucketWriter = "roles/storage.legacyBucketWriter"
182+
183+
# Prepare policy
184+
policy = gcs_bucket.get_iam_policy(requested_policy_version=3)
185+
policy.bindings.append({"role": objectViewer, "members": {member}})
186+
policy.bindings.append({"role": bucketReader, "members": {member}})
187+
policy.bindings.append({"role": bucketWriter, "members": {member}})
188+
189+
# Set policy
190+
gcs_bucket.set_iam_policy(policy)
191+
192+
yield gcs_bucket
193+
194+
195+
@pytest.fixture(scope='module')
196+
def agent_pool_name():
197+
"""
198+
Yields a source agent pool name
199+
"""
200+
201+
# use default agent
202+
yield ''
203+
204+
205+
@pytest.fixture(scope='module')
206+
def posix_root_directory():
207+
"""
208+
Yields a POSIX root directory
209+
"""
210+
211+
# use arbitrary path
212+
yield '/my-posix-root/'
213+
214+
215+
@pytest.fixture(scope='module')
216+
def manifest_file(source_bucket: storage.Bucket):
217+
"""
218+
Yields a transfer manifest file name
219+
"""
220+
221+
# use arbitrary path and name
222+
yield f'gs://{source_bucket.name}/test-manifest.csv'

‎storagetransfer/manifest_request.py

Copy file name to clipboard
+111Lines changed: 111 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,111 @@
1+
#!/usr/bin/env python
2+
3+
# Copyright 2022 Google LLC
4+
#
5+
# Licensed under the Apache License, Version 2.0 (the "License");
6+
# you may not use this file except in compliance with the License.
7+
# You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing, software
12+
# distributed under the License is distributed on an "AS IS" BASIS,
13+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
# See the License for the specific language governing permissions and
15+
# limitations under the License.
16+
17+
"""
18+
Command-line sample that creates a transfer from a POSIX file system to a
19+
GCS bucket using a manifest file.
20+
"""
21+
22+
23+
import argparse
24+
25+
# [START storagetransfer_manifest_request]
26+
from google.cloud import storage_transfer
27+
28+
29+
def create_transfer_with_manifest(
30+
project_id: str, description: str, source_agent_pool_name: str,
31+
root_directory: str, sink_bucket: str, manifest_location: str):
32+
"""Create a transfer from a POSIX file system to a GCS bucket using
33+
a manifest file."""
34+
35+
client = storage_transfer.StorageTransferServiceClient()
36+
37+
# The ID of the Google Cloud Platform Project that owns the job
38+
# project_id = 'my-project-id'
39+
40+
# A useful description for your transfer job
41+
# description = 'My transfer job'
42+
43+
# The agent pool associated with the POSIX data source.
44+
# Defaults to 'projects/{project_id}/agentPools/transfer_service_default'
45+
# source_agent_pool_name = 'projects/my-project/agentPools/my-agent'
46+
47+
# The root directory path on the source filesystem
48+
# root_directory = '/directory/to/transfer/source'
49+
50+
# Google Cloud Storage destination bucket name
51+
# sink_bucket = 'my-gcs-destination-bucket'
52+
53+
# Transfer manifest location. Must be a `gs:` URL
54+
# manifest_location = 'gs://my-bucket/sample_manifest.csv'
55+
56+
transfer_job_request = storage_transfer.CreateTransferJobRequest({
57+
'transfer_job': {
58+
'project_id': project_id,
59+
'description': description,
60+
'status': storage_transfer.TransferJob.Status.ENABLED,
61+
'transfer_spec': {
62+
'source_agent_pool_name': source_agent_pool_name,
63+
'posix_data_source': {
64+
'root_directory': root_directory,
65+
},
66+
'gcs_data_sink': {
67+
'bucket_name': sink_bucket,
68+
},
69+
'transfer_manifest': {
70+
'location': manifest_location
71+
}
72+
}
73+
}
74+
})
75+
76+
result = client.create_transfer_job(transfer_job_request)
77+
print(f'Created transferJob: {result.name}')
78+
79+
80+
# [END storagetransfer_manifest_request]
81+
82+
if __name__ == "__main__":
83+
parser = argparse.ArgumentParser(description=__doc__)
84+
parser.add_argument(
85+
'--project-id',
86+
help='The ID of the Google Cloud Platform Project that owns the job',
87+
required=True)
88+
parser.add_argument(
89+
'--description',
90+
help='A useful description for your transfer job',
91+
required=True)
92+
parser.add_argument(
93+
'--source-agent-pool-name',
94+
help='The agent pool associated with the POSIX data source',
95+
required=True)
96+
parser.add_argument(
97+
'--root-directory',
98+
help='The root directory path on the source filesystem',
99+
required=True)
100+
parser.add_argument(
101+
'--sink-bucket',
102+
help='Google Cloud Storage destination bucket name',
103+
required=True)
104+
parser.add_argument(
105+
'--manifest-location',
106+
help='Transfer manifest location. Must be a `gs:` URL',
107+
required=True)
108+
109+
args = parser.parse_args()
110+
111+
create_transfer_with_manifest(**vars(args))

‎storagetransfer/manifest_request_test.py

Copy file name to clipboard
+39Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
# Copyright 2022 Google LLC
2+
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
import backoff
16+
from google.api_core.exceptions import RetryError
17+
from google.cloud.storage import Bucket
18+
19+
import manifest_request
20+
21+
22+
@backoff.on_exception(backoff.expo, (RetryError,), max_time=60)
23+
def test_manifest_request(
24+
capsys, project_id: str, job_description_unique: str,
25+
agent_pool_name: str, posix_root_directory: str,
26+
destination_bucket: Bucket, manifest_file: str):
27+
28+
manifest_request.create_transfer_with_manifest(
29+
project_id=project_id,
30+
description=job_description_unique,
31+
source_agent_pool_name=agent_pool_name,
32+
root_directory=posix_root_directory,
33+
sink_bucket=destination_bucket.name,
34+
manifest_location=manifest_file
35+
)
36+
37+
out, _ = capsys.readouterr()
38+
39+
assert "Created transferJob" in out

‎storagetransfer/posix_download.py

Copy file name to clipboard
+108Lines changed: 108 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,108 @@
1+
#!/usr/bin/env python
2+
3+
# Copyright 2022 Google LLC
4+
#
5+
# Licensed under the Apache License, Version 2.0 (the "License");
6+
# you may not use this file except in compliance with the License.
7+
# You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing, software
12+
# distributed under the License is distributed on an "AS IS" BASIS,
13+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
# See the License for the specific language governing permissions and
15+
# limitations under the License.
16+
17+
"""
18+
Command-line sample that creates a transfer from a GCS bucket to a POSIX file
19+
system.
20+
"""
21+
22+
23+
import argparse
24+
25+
# [START storagetransfer_download_to_posix]
26+
from google.cloud import storage_transfer
27+
28+
29+
def download_from_gcs(
30+
project_id: str, description: str, sink_agent_pool_name: str,
31+
root_directory: str, source_bucket: str, gcs_source_path: str):
32+
"""Create a transfer from a GCS bucket to a POSIX file system."""
33+
34+
client = storage_transfer.StorageTransferServiceClient()
35+
36+
# The ID of the Google Cloud Platform Project that owns the job
37+
# project_id = 'my-project-id'
38+
39+
# A useful description for your transfer job
40+
# description = 'My transfer job'
41+
42+
# The agent pool associated with the POSIX data sink.
43+
# Defaults to 'projects/{project_id}/agentPools/transfer_service_default'
44+
# sink_agent_pool_name = 'projects/my-project/agentPools/my-agent'
45+
46+
# The root directory path on the destination filesystem
47+
# root_directory = '/directory/to/transfer/sink'
48+
49+
# Google Cloud Storage source bucket name
50+
# source_bucket = 'my-gcs-source-bucket'
51+
52+
# An optional path on the Google Cloud Storage bucket to download from
53+
# gcs_source_path = 'foo/bar/'
54+
55+
transfer_job_request = storage_transfer.CreateTransferJobRequest({
56+
'transfer_job': {
57+
'project_id': project_id,
58+
'description': description,
59+
'status': storage_transfer.TransferJob.Status.ENABLED,
60+
'transfer_spec': {
61+
'sink_agent_pool_name': sink_agent_pool_name,
62+
'posix_data_sink': {
63+
'root_directory': root_directory,
64+
},
65+
'gcs_data_source': {
66+
'bucket_name': source_bucket,
67+
'path': gcs_source_path,
68+
},
69+
}
70+
}
71+
})
72+
73+
result = client.create_transfer_job(transfer_job_request)
74+
print(f'Created transferJob: {result.name}')
75+
76+
77+
# [END storagetransfer_download_to_posix]
78+
79+
if __name__ == "__main__":
80+
parser = argparse.ArgumentParser(description=__doc__)
81+
parser.add_argument(
82+
'--project-id',
83+
help='The ID of the Google Cloud Platform Project that owns the job',
84+
required=True)
85+
parser.add_argument(
86+
'--description',
87+
help='A useful description for your transfer job',
88+
required=True)
89+
parser.add_argument(
90+
'--sink-agent-pool-name',
91+
help='The agent pool associated with the POSIX data sink',
92+
required=True)
93+
parser.add_argument(
94+
'--root-directory',
95+
help='The root directory path on the destination filesystem',
96+
required=True)
97+
parser.add_argument(
98+
'--source-bucket',
99+
help='Google Cloud Storage source bucket name',
100+
required=True)
101+
parser.add_argument(
102+
'--gcs-source-path',
103+
help='A path on the Google Cloud Storage bucket to download from',
104+
required=True)
105+
106+
args = parser.parse_args()
107+
108+
download_from_gcs(**vars(args))

‎storagetransfer/posix_download_test.py

Copy file name to clipboard
+39Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
# Copyright 2022 Google LLC
2+
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
import backoff
16+
from google.api_core.exceptions import RetryError
17+
from google.cloud.storage import Bucket
18+
19+
import posix_download
20+
21+
22+
@backoff.on_exception(backoff.expo, (RetryError,), max_time=60)
23+
def test_posix_download(
24+
capsys, project_id: str, job_description_unique: str,
25+
agent_pool_name: str, posix_root_directory: str,
26+
source_bucket: Bucket):
27+
28+
posix_download.download_from_gcs(
29+
project_id=project_id,
30+
description=job_description_unique,
31+
sink_agent_pool_name=agent_pool_name,
32+
root_directory=posix_root_directory,
33+
source_bucket=source_bucket.name,
34+
gcs_source_path=posix_root_directory
35+
)
36+
37+
out, _ = capsys.readouterr()
38+
39+
assert "Created transferJob" in out

0 commit comments

Comments
0 (0)
Morty Proxy This is a proxified and sanitized view of the page, visit original site.