Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly
Appearance settings

Commit ecf241a

Browse filesBrowse files
vi3k6i5vi3k6i5
authored
docs: add docs build kokoro config for django spanner (#644)
* docs: added trampoline v2 file and docs config for kokoro * fix: change library_type to INTEGRATION * docs: add kokoro docker build for docs * fix:repo metadata * fix: updated PROJECT_ROOT for django spanner docs build using kokoro * fix: remove sudo apt update because sudo is not allowed * fix: add build_nox.sh as original build.sh file is used by kokoro workers to run parallelized tests * refactor: renamed build_nox.sh to build-nox.sh * refactor: change build-nox to nox.sh * refactor: change build-nox to nox.sh * refactor: change kokoro build to run different test for docs and different test for workers * refactor: change kokoro build to run different test for docs and different test for workers * refactor: add log msg for debugging * refactor: remove log msg added for debugging * fix: correct docs build command * refactor: remove unused nox file * feat: add .trampolinerc file * fix: correct repo_metadata release_level to beta
1 parent fe4a062 commit ecf241a
Copy full SHA for ecf241a

10 files changed

+890
-21
lines changed

‎.kokoro/build.sh

Copy file name to clipboardExpand all lines: .kokoro/build.sh
+37-21Lines changed: 37 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,11 @@
1515

1616
set -eo pipefail
1717

18-
cd github/python-spanner-django
18+
if [[ -z "${PROJECT_ROOT:-}" ]]; then
19+
PROJECT_ROOT="github/python-spanner-django"
20+
fi
21+
22+
cd "${PROJECT_ROOT}"
1923

2024
# Disable buffering, so that the logs stream through.
2125
export PYTHONUNBUFFERED=1
@@ -27,26 +31,38 @@ export GOOGLE_CLOUD_PROJECT=$(cat "${KOKORO_GFILE_DIR}/project-id.json")
2731

2832
export RUNNING_SPANNER_BACKEND_TESTS=1
2933

30-
pip3 install .
31-
export DJANGO_TESTS_DIR="django_tests_dir"
32-
mkdir -p $DJANGO_TESTS_DIR && git clone --depth 1 --single-branch --branch "spanner/stable/2.2.x" https://github.com/c24t/django.git $DJANGO_TESTS_DIR/django
33-
34-
# Install dependencies for Django tests.
35-
sudo apt-get update
36-
apt-get install -y libffi-dev libjpeg-dev zlib1g-dev libmemcached-dev
37-
cd $DJANGO_TESTS_DIR/django && pip3 install -e . && pip3 install -r tests/requirements/py3.txt; cd ../../
38-
39-
# Hardcode the max number of workers since Spanner has a very low
40-
# QPS for administrative RPCs of 5QPS (averaged every 100 seconds)
41-
if [[ $KOKORO_JOB_NAME == *"continuous"* ]]
34+
if [[ $KOKORO_JOB_NAME == *"docs"* ]]
4235
then
43-
# Disable continuous build as it creates too many Spanner instances
44-
# ("Quota exceeded for quota metric 'Instance create requests' and
45-
# limit 'Instance create requests per minute' of service
46-
# 'spanner.googleapis.com').
47-
export DJANGO_WORKER_COUNT=0
36+
echo "Running docs generation."
37+
# Remove old nox
38+
python3 -m pip uninstall --yes --quiet nox-automation
39+
40+
# Install nox
41+
python3 -m pip install --upgrade --quiet nox
42+
# Generate docs.
43+
python3 -m nox -s docs docfx
4844
else
49-
export DJANGO_WORKER_COUNT=5
50-
fi
45+
pip3 install .
46+
export DJANGO_TESTS_DIR="django_tests_dir"
47+
mkdir -p $DJANGO_TESTS_DIR && git clone --depth 1 --single-branch --branch "spanner/stable/2.2.x" https://github.com/c24t/django.git $DJANGO_TESTS_DIR/django
48+
49+
# Install dependencies for Django tests.
50+
sudo apt-get update
51+
apt-get install -y libffi-dev libjpeg-dev zlib1g-dev libmemcached-dev
52+
cd $DJANGO_TESTS_DIR/django && pip3 install -e . && pip3 install -r tests/requirements/py3.txt; cd ../../
5153

52-
python3 ./run_testing_worker.py
54+
# Hardcode the max number of workers since Spanner has a very low
55+
# QPS for administrative RPCs of 5QPS (averaged every 100 seconds)
56+
if [[ $KOKORO_JOB_NAME == *"continuous"* ]]
57+
then
58+
# Disable continuous build as it creates too many Spanner instances
59+
# ("Quota exceeded for quota metric 'Instance create requests' and
60+
# limit 'Instance create requests per minute' of service
61+
# 'spanner.googleapis.com').
62+
export DJANGO_WORKER_COUNT=0
63+
else
64+
export DJANGO_WORKER_COUNT=5
65+
fi
66+
67+
python3 ./run_testing_worker.py
68+
fi

‎.kokoro/docker/docs/Dockerfile

Copy file name to clipboard
+98Lines changed: 98 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,98 @@
1+
# Copyright 2020 Google LLC
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
from ubuntu:20.04
16+
17+
ENV DEBIAN_FRONTEND noninteractive
18+
19+
# Ensure local Python is preferred over distribution Python.
20+
ENV PATH /usr/local/bin:$PATH
21+
22+
# Install dependencies.
23+
RUN apt-get update \
24+
&& apt-get install -y --no-install-recommends \
25+
apt-transport-https \
26+
build-essential \
27+
ca-certificates \
28+
curl \
29+
dirmngr \
30+
git \
31+
gpg-agent \
32+
graphviz \
33+
libbz2-dev \
34+
libdb5.3-dev \
35+
libexpat1-dev \
36+
libffi-dev \
37+
liblzma-dev \
38+
libreadline-dev \
39+
libsnappy-dev \
40+
libssl-dev \
41+
libsqlite3-dev \
42+
portaudio19-dev \
43+
redis-server \
44+
software-properties-common \
45+
ssh \
46+
sudo \
47+
tcl \
48+
tcl-dev \
49+
tk \
50+
tk-dev \
51+
uuid-dev \
52+
wget \
53+
zlib1g-dev \
54+
&& add-apt-repository universe \
55+
&& apt-get update \
56+
&& apt-get -y install jq \
57+
&& apt-get clean autoclean \
58+
&& apt-get autoremove -y \
59+
&& rm -rf /var/lib/apt/lists/* \
60+
&& rm -f /var/cache/apt/archives/*.deb
61+
62+
63+
COPY fetch_gpg_keys.sh /tmp
64+
# Install the desired versions of Python.
65+
RUN set -ex \
66+
&& export GNUPGHOME="$(mktemp -d)" \
67+
&& echo "disable-ipv6" >> "${GNUPGHOME}/dirmngr.conf" \
68+
&& /tmp/fetch_gpg_keys.sh \
69+
&& for PYTHON_VERSION in 3.7.8 3.8.5; do \
70+
wget --no-check-certificate -O python-${PYTHON_VERSION}.tar.xz "https://www.python.org/ftp/python/${PYTHON_VERSION%%[a-z]*}/Python-$PYTHON_VERSION.tar.xz" \
71+
&& wget --no-check-certificate -O python-${PYTHON_VERSION}.tar.xz.asc "https://www.python.org/ftp/python/${PYTHON_VERSION%%[a-z]*}/Python-$PYTHON_VERSION.tar.xz.asc" \
72+
&& gpg --batch --verify python-${PYTHON_VERSION}.tar.xz.asc python-${PYTHON_VERSION}.tar.xz \
73+
&& rm -r python-${PYTHON_VERSION}.tar.xz.asc \
74+
&& mkdir -p /usr/src/python-${PYTHON_VERSION} \
75+
&& tar -xJC /usr/src/python-${PYTHON_VERSION} --strip-components=1 -f python-${PYTHON_VERSION}.tar.xz \
76+
&& rm python-${PYTHON_VERSION}.tar.xz \
77+
&& cd /usr/src/python-${PYTHON_VERSION} \
78+
&& ./configure \
79+
--enable-shared \
80+
# This works only on Python 2.7 and throws a warning on every other
81+
# version, but seems otherwise harmless.
82+
--enable-unicode=ucs4 \
83+
--with-system-ffi \
84+
--without-ensurepip \
85+
&& make -j$(nproc) \
86+
&& make install \
87+
&& ldconfig \
88+
; done \
89+
&& rm -rf "${GNUPGHOME}" \
90+
&& rm -rf /usr/src/python* \
91+
&& rm -rf ~/.cache/
92+
93+
RUN wget -O /tmp/get-pip.py 'https://bootstrap.pypa.io/get-pip.py' \
94+
&& python3.7 /tmp/get-pip.py \
95+
&& python3.8 /tmp/get-pip.py \
96+
&& rm /tmp/get-pip.py
97+
98+
CMD ["python3.7"]

‎.kokoro/docker/docs/fetch_gpg_keys.sh

Copy file name to clipboard
+45Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,45 @@
1+
#!/bin/bash
2+
# Copyright 2020 Google LLC
3+
#
4+
# Licensed under the Apache License, Version 2.0 (the "License");
5+
# you may not use this file except in compliance with the License.
6+
# You may obtain a copy of the License at
7+
#
8+
# http://www.apache.org/licenses/LICENSE-2.0
9+
#
10+
# Unless required by applicable law or agreed to in writing, software
11+
# distributed under the License is distributed on an "AS IS" BASIS,
12+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
# See the License for the specific language governing permissions and
14+
# limitations under the License.
15+
16+
# A script to fetch gpg keys with retry.
17+
# Avoid jinja parsing the file.
18+
#
19+
20+
function retry {
21+
if [[ "${#}" -le 1 ]]; then
22+
echo "Usage: ${0} retry_count commands.."
23+
exit 1
24+
fi
25+
local retries=${1}
26+
local command="${@:2}"
27+
until [[ "${retries}" -le 0 ]]; do
28+
$command && return 0
29+
if [[ $? -ne 0 ]]; then
30+
echo "command failed, retrying"
31+
((retries--))
32+
fi
33+
done
34+
return 1
35+
}
36+
37+
# 3.6.9, 3.7.5 (Ned Deily)
38+
retry 3 gpg --keyserver ha.pool.sks-keyservers.net --recv-keys \
39+
0D96DF4D4110E5C43FBFB17F2D347EA6AA65421D
40+
41+
# 3.8.0 (Łukasz Langa)
42+
retry 3 gpg --keyserver ha.pool.sks-keyservers.net --recv-keys \
43+
E3FF2839C048B25C084DEBE9B26995E310250568
44+
45+
#

‎.kokoro/docs/common.cfg

Copy file name to clipboard
+65Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,65 @@
1+
# Format: //devtools/kokoro/config/proto/build.proto
2+
3+
# Build logs will be here
4+
action {
5+
define_artifacts {
6+
regex: "**/*sponge_log.xml"
7+
}
8+
}
9+
10+
# Download trampoline resources.
11+
gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/trampoline"
12+
13+
# Use the trampoline script to run in docker.
14+
build_file: "python-spanner-django/.kokoro/trampoline_v2.sh"
15+
16+
# Configure the docker image for kokoro-trampoline.
17+
env_vars: {
18+
key: "TRAMPOLINE_IMAGE"
19+
value: "gcr.io/cloud-devrel-kokoro-resources/python-lib-docs"
20+
}
21+
env_vars: {
22+
key: "TRAMPOLINE_BUILD_FILE"
23+
value: "github/python-spanner-django/.kokoro/publish-docs.sh"
24+
}
25+
26+
env_vars: {
27+
key: "STAGING_BUCKET"
28+
value: "docs-staging"
29+
}
30+
31+
env_vars: {
32+
key: "V2_STAGING_BUCKET"
33+
value: "docs-staging-v2"
34+
}
35+
36+
# It will upload the docker image after successful builds.
37+
env_vars: {
38+
key: "TRAMPOLINE_IMAGE_UPLOAD"
39+
value: "true"
40+
}
41+
42+
# It will always build the docker image.
43+
env_vars: {
44+
key: "TRAMPOLINE_DOCKERFILE"
45+
value: ".kokoro/docker/docs/Dockerfile"
46+
}
47+
48+
# Fetch the token needed for reporting release status to GitHub
49+
before_action {
50+
fetch_keystore {
51+
keystore_resource {
52+
keystore_config_id: 73713
53+
keyname: "yoshi-automation-github-key"
54+
}
55+
}
56+
}
57+
58+
before_action {
59+
fetch_keystore {
60+
keystore_resource {
61+
keystore_config_id: 73713
62+
keyname: "docuploader_service_account"
63+
}
64+
}
65+
}

‎.kokoro/docs/docs-presubmit.cfg

Copy file name to clipboard
+28Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
# Format: //devtools/kokoro/config/proto/build.proto
2+
3+
env_vars: {
4+
key: "STAGING_BUCKET"
5+
value: "gcloud-python-test"
6+
}
7+
8+
env_vars: {
9+
key: "V2_STAGING_BUCKET"
10+
value: "gcloud-python-test"
11+
}
12+
13+
# We only upload the image in the main `docs` build.
14+
env_vars: {
15+
key: "TRAMPOLINE_IMAGE_UPLOAD"
16+
value: "false"
17+
}
18+
19+
env_vars: {
20+
key: "TRAMPOLINE_BUILD_FILE"
21+
value: "github/python-spanner-django/.kokoro/build.sh"
22+
}
23+
24+
# Only run this nox session.
25+
env_vars: {
26+
key: "NOX_SESSION"
27+
value: "docs docfx"
28+
}

‎.kokoro/docs/docs.cfg

Copy file name to clipboard
+1Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
# Format: //devtools/kokoro/config/proto/build.proto

‎.kokoro/publish-docs.sh

Copy file name to clipboard
+64Lines changed: 64 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,64 @@
1+
#!/bin/bash
2+
# Copyright 2020 Google LLC
3+
#
4+
# Licensed under the Apache License, Version 2.0 (the "License");
5+
# you may not use this file except in compliance with the License.
6+
# You may obtain a copy of the License at
7+
#
8+
# https://www.apache.org/licenses/LICENSE-2.0
9+
#
10+
# Unless required by applicable law or agreed to in writing, software
11+
# distributed under the License is distributed on an "AS IS" BASIS,
12+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
# See the License for the specific language governing permissions and
14+
# limitations under the License.
15+
16+
set -eo pipefail
17+
18+
# Disable buffering, so that the logs stream through.
19+
export PYTHONUNBUFFERED=1
20+
21+
export PATH="${HOME}/.local/bin:${PATH}"
22+
23+
# Install nox
24+
python3 -m pip install --user --upgrade --quiet nox
25+
python3 -m nox --version
26+
27+
# build docs
28+
nox -s docs
29+
30+
python3 -m pip install --user gcp-docuploader
31+
32+
# create metadata
33+
python3 -m docuploader create-metadata \
34+
--name=$(jq --raw-output '.name // empty' .repo-metadata.json) \
35+
--version=$(python3 setup.py --version) \
36+
--language=$(jq --raw-output '.language // empty' .repo-metadata.json) \
37+
--distribution-name=$(python3 setup.py --name) \
38+
--product-page=$(jq --raw-output '.product_documentation // empty' .repo-metadata.json) \
39+
--github-repository=$(jq --raw-output '.repo // empty' .repo-metadata.json) \
40+
--issue-tracker=$(jq --raw-output '.issue_tracker // empty' .repo-metadata.json)
41+
42+
cat docs.metadata
43+
44+
# upload docs
45+
python3 -m docuploader upload docs/_build/html --metadata-file docs.metadata --staging-bucket "${STAGING_BUCKET}"
46+
47+
48+
# docfx yaml files
49+
nox -s docfx
50+
51+
# create metadata.
52+
python3 -m docuploader create-metadata \
53+
--name=$(jq --raw-output '.name // empty' .repo-metadata.json) \
54+
--version=$(python3 setup.py --version) \
55+
--language=$(jq --raw-output '.language // empty' .repo-metadata.json) \
56+
--distribution-name=$(python3 setup.py --name) \
57+
--product-page=$(jq --raw-output '.product_documentation // empty' .repo-metadata.json) \
58+
--github-repository=$(jq --raw-output '.repo // empty' .repo-metadata.json) \
59+
--issue-tracker=$(jq --raw-output '.issue_tracker // empty' .repo-metadata.json)
60+
61+
cat docs.metadata
62+
63+
# upload docs
64+
python3 -m docuploader upload docs/_build/html/docfx_yaml --metadata-file docs.metadata --destination-prefix docfx --staging-bucket "${V2_STAGING_BUCKET}"

0 commit comments

Comments
0 (0)
Morty Proxy This is a proxified and sanitized view of the page, visit original site.