Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly
Appearance settings

Commit ad12664

Browse filesBrowse files
busunkim96busunkim96
authored
Merge branch 'main' into functions-billing-tests
2 parents 8c5aa74 + 6a0d788 commit ad12664
Copy full SHA for ad12664

File tree

Expand file treeCollapse file tree

6 files changed

+171
-0
lines changed
Filter options
Expand file treeCollapse file tree

6 files changed

+171
-0
lines changed

‎dataflow/custom-containers/ubuntu/Dockerfile

Copy file name to clipboard
+34Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
# Copyright 2022 Google LLC
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
FROM ubuntu:latest
16+
17+
# Set the entrypoint to Apache Beam SDK worker launcher.
18+
COPY --from=apache/beam_python3.8_sdk:2.36.0 /opt/apache/beam /opt/apache/beam
19+
ENTRYPOINT [ "/opt/apache/beam/boot" ]
20+
21+
# Install Python with pip, dev tools, distutils, and a C++ compiler.
22+
COPY requirements.txt .
23+
RUN apt-get update \
24+
&& apt-get install -y --no-install-recommends \
25+
ca-certificates curl g++ python3.8-dev python3-distutils \
26+
&& rm -rf /var/lib/apt/lists/* \
27+
&& update-alternatives --install /usr/bin/python python /usr/bin/python3.8 10 \
28+
&& curl https://bootstrap.pypa.io/get-pip.py | python \
29+
# Install the requirements.
30+
&& pip install --no-cache-dir -r requirements.txt \
31+
&& pip check
32+
33+
# Copy the pipeline source files.
34+
COPY main.py ./

‎dataflow/custom-containers/ubuntu/e2e_test.py

Copy file name to clipboard
+54Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,54 @@
1+
#!/usr/bin/env python
2+
3+
# Copyright 2022 Google LLC
4+
#
5+
# Licensed under the Apache License, Version 2.0 (the "License");
6+
# you may not use this file except in compliance with the License.
7+
# You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing, software
12+
# distributed under the License is distributed on an "AS IS" BASIS,
13+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
# See the License for the specific language governing permissions and
15+
# limitations under the License.
16+
17+
import subprocess
18+
19+
try:
20+
# `conftest` cannot be imported when running in `nox`, but we still
21+
# try to import it for the autocomplete when writing the tests.
22+
from conftest import Utils
23+
except ModuleNotFoundError:
24+
Utils = None
25+
import pytest
26+
27+
NAME = "dataflow/custom-containers/minimal"
28+
29+
30+
@pytest.fixture(scope="session")
31+
def bucket_name(utils: Utils) -> str:
32+
yield from utils.storage_bucket(NAME)
33+
34+
35+
@pytest.fixture(scope="session")
36+
def container_image(utils: Utils) -> str:
37+
yield from utils.cloud_build_submit(image_name=NAME)
38+
39+
40+
def test_tensorflow_minimal(
41+
utils: Utils, bucket_name: str, container_image: str
42+
) -> None:
43+
subprocess.check_call(
44+
[
45+
"python",
46+
"main.py",
47+
"--runner=DataflowRunner",
48+
f"--project={utils.project}",
49+
f"--region={utils.region}",
50+
f"--temp_location=gs://{bucket_name}",
51+
f"--sdk_container_image={container_image}",
52+
"--experiment=use_runner_v2",
53+
]
54+
)

‎dataflow/custom-containers/ubuntu/main.py

Copy file name to clipboard
+34Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
#!/usr/bin/env python
2+
3+
# Copyright 2022 Google LLC
4+
#
5+
# Licensed under the Apache License, Version 2.0 (the "License");
6+
# you may not use this file except in compliance with the License.
7+
# You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing, software
12+
# distributed under the License is distributed on an "AS IS" BASIS,
13+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
# See the License for the specific language governing permissions and
15+
# limitations under the License.
16+
17+
import logging
18+
import platform
19+
20+
import apache_beam as beam
21+
22+
23+
def run() -> None:
24+
with beam.Pipeline() as pipeline:
25+
(
26+
pipeline
27+
| "Create data" >> beam.Create(["Hello", "World!", platform.platform()])
28+
| "Print" >> beam.Map(logging.info)
29+
)
30+
31+
32+
if __name__ == "__main__":
33+
logging.getLogger().setLevel(logging.INFO)
34+
run()

‎dataflow/custom-containers/ubuntu/noxfile_config.py

Copy file name to clipboard
+45Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,45 @@
1+
# Copyright 2022 Google LLC
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
# Default TEST_CONFIG_OVERRIDE for python repos.
16+
17+
# You can copy this file into your directory, then it will be imported from
18+
# the noxfile.py.
19+
20+
# The source of truth:
21+
# https://github.com/GoogleCloudPlatform/python-docs-samples/blob/main/noxfile_config.py
22+
23+
TEST_CONFIG_OVERRIDE = {
24+
# You can opt out from the test for specific Python versions.
25+
# > ℹ️ We're opting out of all Python versions except 3.8.
26+
# > The Python version used is defined by the Dockerfile, so it's redundant
27+
# > to run multiple tests since they would all be running the same Dockerfile.
28+
"ignored_versions": ["2.7", "3.6", "3.7", "3.9", "3.10"],
29+
# Old samples are opted out of enforcing Python type hints
30+
# All new samples should feature them
31+
"enforce_type_hints": True,
32+
# An envvar key for determining the project id to use. Change it
33+
# to 'BUILD_SPECIFIC_GCLOUD_PROJECT' if you want to opt in using a
34+
# build specific Cloud project. You can also use your own string
35+
# to use your own Cloud project.
36+
"gcloud_project_env": "GOOGLE_CLOUD_PROJECT",
37+
# 'gcloud_project_env': 'BUILD_SPECIFIC_GCLOUD_PROJECT',
38+
# If you need to use a specific version of pip,
39+
# change pip_version_override to the string representation
40+
# of the version number, for example, "20.2.4"
41+
"pip_version_override": None,
42+
# A dictionary you want to inject into your test. Don't put any
43+
# secrets here. These values will override predefined values.
44+
"envs": {},
45+
}

‎dataflow/custom-containers/ubuntu/requirements-test.txt

Copy file name to clipboard
+3Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
google-cloud-storage==1.43.0
2+
pytest-xdist==2.5.0
3+
pytest==6.2.4

‎dataflow/custom-containers/ubuntu/requirements.txt

Copy file name to clipboard
+1Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
apache-beam[gcp]==2.36.0

0 commit comments

Comments
0 (0)
Morty Proxy This is a proxified and sanitized view of the page, visit original site.