Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly
Appearance settings

Commit 36c4c80

Browse filesBrowse files
tswasttswast
authored
Composer: Move trigger response DAG to GitHub. (GoogleCloudPlatform#1645)
* Composer: Move trigger response DAG to GitHub. Sample originally was posted at https://cloud.google.com/composer/docs/how-to/using/triggering-with-gcf#wzxhzdk15gcs_response_dagpywzxhzdk16 * Composer: Add snippet to get the client ID for a composer env.
1 parent 790fd5f commit 36c4c80
Copy full SHA for 36c4c80

File tree

Expand file treeCollapse file tree

5 files changed

+168
-1
lines changed
Filter options
Expand file treeCollapse file tree

5 files changed

+168
-1
lines changed

‎composer/rest/get_client_id.py

Copy file name to clipboard
+70Lines changed: 70 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,70 @@
1+
# Copyright 2018 Google LLC
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# https://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
"""Get the client ID associated with a Cloud Composer environment."""
16+
17+
import argparse
18+
19+
20+
def get_client_id(project_id, location, composer_environment):
21+
# [START composer_get_environment_client_id]
22+
import google.auth
23+
import google.auth.transport.requests
24+
import requests
25+
import six.moves.urllib.parse
26+
27+
# Authenticate with Google Cloud.
28+
# See: https://cloud.google.com/docs/authentication/getting-started
29+
credentials, _ = google.auth.default(
30+
scopes=['https://www.googleapis.com/auth/cloud-platform'])
31+
authed_session = google.auth.transport.requests.AuthorizedSession(
32+
credentials)
33+
34+
# project_id = 'YOUR_PROJECT_ID'
35+
# location = 'us-central1'
36+
# composer_environment = 'YOUR_COMPOSER_ENVIRONMENT_NAME'
37+
38+
environment_url = (
39+
'https://composer.googleapis.com/v1beta1/projects/{}/locations/{}'
40+
'/environments/{}').format(project_id, location, composer_environment)
41+
composer_response = authed_session.request('GET', environment_url)
42+
environment_data = composer_response.json()
43+
airflow_uri = environment_data['config']['airflowUri']
44+
45+
# The Composer environment response does not include the IAP client ID.
46+
# Make a second, unauthenticated HTTP request to the web server to get the
47+
# redirect URI.
48+
redirect_response = requests.get(airflow_uri, allow_redirects=False)
49+
redirect_location = redirect_response.headers['location']
50+
51+
# Extract the client_id query parameter from the redirect.
52+
parsed = six.moves.urllib.parse.urlparse(redirect_location)
53+
query_string = six.moves.urllib.parse.parse_qs(parsed.query)
54+
print(query_string['client_id'][0])
55+
# [END composer_get_environment_client_id]
56+
57+
58+
if __name__ == '__main__':
59+
parser = argparse.ArgumentParser(
60+
description=__doc__,
61+
formatter_class=argparse.RawDescriptionHelpFormatter)
62+
parser.add_argument('project_id', help='Your Project ID.')
63+
parser.add_argument(
64+
'location', help='Region of the Cloud Composer environent.')
65+
parser.add_argument(
66+
'composer_environment', help='Name of the Cloud Composer environent.')
67+
68+
args = parser.parse_args()
69+
get_client_id(
70+
args.project_id, args.location, args.composer_environment)

‎composer/rest/get_client_id_test.py

Copy file name to clipboard
+28Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
# Copyright 2018 Google LLC
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# https://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
import os
16+
17+
from .get_client_id import get_client_id
18+
19+
20+
PROJECT = os.environ['GOOGLE_CLOUD_PROJECT']
21+
COMPOSER_LOCATION = os.environ['COMPOSER_LOCATION']
22+
COMPOSER_ENVIRONMENT = os.environ['COMPOSER_ENVIRONMENT']
23+
24+
25+
def test_get_client_id(capsys):
26+
get_client_id(PROJECT, COMPOSER_LOCATION, COMPOSER_ENVIRONMENT)
27+
out, _ = capsys.readouterr()
28+
assert '.apps.googleusercontent.com' in out

‎composer/rest/requirements.txt

Copy file name to clipboard
+2-1Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,2 +1,3 @@
11
google-auth==1.4.1
2-
requests==2.18.4
2+
requests==2.18.4
3+
six==1.11.0

‎composer/workflows/trigger_response_dag.py

Copy file name to clipboard
+45Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,45 @@
1+
# Copyright 2018 Google LLC
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# https://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
"""DAG running in response to a Cloud Storage bucket change."""
16+
17+
# [START composer_trigger_response_dag]
18+
import datetime
19+
20+
import airflow
21+
from airflow.operators import bash_operator
22+
23+
24+
default_args = {
25+
'owner': 'Composer Example',
26+
'depends_on_past': False,
27+
'email': [''],
28+
'email_on_failure': False,
29+
'email_on_retry': False,
30+
'retries': 1,
31+
'retry_delay': datetime.timedelta(minutes=5),
32+
'start_date': datetime.datetime(2017, 1, 1),
33+
}
34+
35+
with airflow.DAG(
36+
'composer_sample_trigger_response_dag',
37+
default_args=default_args,
38+
# Not scheduled, trigger only
39+
schedule_interval=None) as dag:
40+
41+
# Print the dag_run's configuration, which includes information about the
42+
# Cloud Storage object change.
43+
print_gcs_info = bash_operator.BashOperator(
44+
task_id='print_gcs_info', bash_command='echo {{ dag_run.conf }}')
45+
# [END composer_trigger_response_dag]

‎composer/workflows/trigger_response_dag_test.py

Copy file name to clipboard
+23Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
# Copyright 2018 Google LLC
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# https://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
16+
def test_dag_import():
17+
"""Test that the DAG file can be successfully imported.
18+
19+
This tests that the DAG can be parsed, but does not run it in an Airflow
20+
environment. This is a recommended sanity check by the official Airflow
21+
docs: https://airflow.incubator.apache.org/tutorial.html#testing
22+
"""
23+
from . import trigger_response_dag # noqa

0 commit comments

Comments
0 (0)
Morty Proxy This is a proxified and sanitized view of the page, visit original site.