Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly
Appearance settings

Commit b4098b6

Browse filesBrowse files
tswasttswast
authored
Add Composer samples that explicitly call python2 (GoogleCloudPlatform#1711)
* Add Composer samples that explicitly call python2 * Use new unit_testing module for Python 2 DAG samples.
1 parent 90ab014 commit b4098b6
Copy full SHA for b4098b6

File tree

Expand file treeCollapse file tree

5 files changed

+179
-0
lines changed
Filter options
Expand file treeCollapse file tree

5 files changed

+179
-0
lines changed

‎composer/data/python2_script.py

Copy file name to clipboard
+15Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
# Copyright 2018 Google LLC
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# https://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
print 'Output from Python 2.' # noqa

‎composer/workflows/bashoperator_python2.py

Copy file name to clipboard
+44Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
# Copyright 2018 Google LLC
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# https://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
# [START composer_bashoperator_python2]
16+
import datetime
17+
18+
from airflow import models
19+
from airflow.operators import bash_operator
20+
21+
22+
yesterday = datetime.datetime.combine(
23+
datetime.datetime.today() - datetime.timedelta(1),
24+
datetime.datetime.min.time())
25+
26+
27+
default_dag_args = {
28+
# Setting start date as yesterday starts the DAG immediately when it is
29+
# detected in the Cloud Storage bucket.
30+
'start_date': yesterday,
31+
}
32+
33+
with models.DAG(
34+
'composer_sample_bashoperator_python2',
35+
schedule_interval=datetime.timedelta(days=1),
36+
default_args=default_dag_args) as dag:
37+
38+
run_python2 = bash_operator.BashOperator(
39+
task_id='run_python2',
40+
# This example runs a Python script from the data folder to prevent
41+
# Airflow from attempting to parse the script as a DAG.
42+
bash_command='python2 /home/airflow/gcs/data/python2_script.py',
43+
)
44+
# [END composer_bashoperator_python2]

‎composer/workflows/bashoperator_python2_test.py

Copy file name to clipboard
+26Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
# Copyright 2018 Google LLC
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# https://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
from . import unit_testing
16+
17+
18+
def test_dag_import():
19+
"""Test that the DAG file can be successfully imported.
20+
21+
This tests that the DAG can be parsed, but does not run it in an Airflow
22+
environment. This is a recommended sanity check by the official Airflow
23+
docs: https://airflow.incubator.apache.org/tutorial.html#testing
24+
"""
25+
from . import bashoperator_python2 as module
26+
unit_testing.assert_has_valid_dag(module)

‎composer/workflows/pythonvirtualenvoperator_python2.py

Copy file name to clipboard
+63Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,63 @@
1+
# Copyright 2018 Google LLC
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# https://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
# [START composer_pythonvirtualenvoperator_python2]
16+
import datetime
17+
18+
from airflow import models
19+
from airflow.operators import python_operator
20+
21+
22+
def python2_function():
23+
"""A function which has not been converted to Python 3."""
24+
# Use the global variable virtualenv_string_args to pass in values when the
25+
# Python version differs from that used by the Airflow process.
26+
global virtualenv_string_args
27+
28+
# Imports must happen within the function when run with the
29+
# PythonVirtualenvOperator.
30+
import cStringIO
31+
import logging
32+
33+
arg0 = virtualenv_string_args[0]
34+
buffer = cStringIO.StringIO()
35+
buffer.write('Wrote an ASCII string to buffer:\n')
36+
buffer.write(arg0)
37+
logging.info(buffer.getvalue())
38+
39+
40+
yesterday = datetime.datetime.combine(
41+
datetime.datetime.today() - datetime.timedelta(1),
42+
datetime.datetime.min.time())
43+
44+
45+
default_dag_args = {
46+
# Setting start date as yesterday starts the DAG immediately when it is
47+
# detected in the Cloud Storage bucket.
48+
'start_date': yesterday,
49+
}
50+
51+
with models.DAG(
52+
'composer_sample_pythonvirtualenvoperator_python2',
53+
schedule_interval=datetime.timedelta(days=1),
54+
default_args=default_dag_args) as dag:
55+
56+
# Use the PythonVirtualenvOperator to select an explicit python_version.
57+
run_python2 = python_operator.PythonVirtualenvOperator(
58+
task_id='run_python2',
59+
python_callable=python2_function,
60+
python_version='2',
61+
string_args=['An example input string'],
62+
)
63+
# [END composer_pythonvirtualenvoperator_python2]

‎composer/workflows/pythonvirtualenvoperator_python2_test.py

Copy file name to clipboard
+31Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
# Copyright 2018 Google LLC
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# https://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
import sys
16+
17+
import pytest
18+
19+
from . import unit_testing
20+
21+
22+
@pytest.mark.skipif(sys.version_info >= (3, 0), reason="requires Python 2")
23+
def test_dag_import():
24+
"""Test that the DAG file can be successfully imported.
25+
26+
This tests that the DAG can be parsed, but does not run it in an Airflow
27+
environment. This is a recommended sanity check by the official Airflow
28+
docs: https://airflow.incubator.apache.org/tutorial.html#testing
29+
"""
30+
from . import pythonvirtualenvoperator_python2 as module
31+
unit_testing.assert_has_valid_dag(module)

0 commit comments

Comments
0 (0)
Morty Proxy This is a proxified and sanitized view of the page, visit original site.