diff --git a/.github/.OwlBot.lock.yaml b/.github/.OwlBot.lock.yaml new file mode 100644 index 000000000..b5c26ed01 --- /dev/null +++ b/.github/.OwlBot.lock.yaml @@ -0,0 +1,3 @@ +docker: + image: gcr.io/repo-automation-bots/owlbot-python:latest + digest: sha256:457583330eec64daa02aeb7a72a04d33e7be2428f646671ce4045dcbc0191b1e diff --git a/.github/.OwlBot.yaml b/.github/.OwlBot.yaml new file mode 100644 index 000000000..63a2aab54 --- /dev/null +++ b/.github/.OwlBot.yaml @@ -0,0 +1,26 @@ +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +docker: + image: gcr.io/repo-automation-bots/owlbot-python:latest + +deep-remove-regex: + - /owl-bot-staging + +deep-copy-regex: + - source: /google/logging/(v.*)/.*-py/(.*) + dest: /owl-bot-staging/$1/$2 + +begin-after-commit-hash: 130ce904e5d546c312943d10f48799590f9c0f66 + diff --git a/.github/header-checker-lint.yml b/.github/header-checker-lint.yml index fc281c05b..6fe78aa79 100644 --- a/.github/header-checker-lint.yml +++ b/.github/header-checker-lint.yml @@ -1,6 +1,6 @@ {"allowedCopyrightHolders": ["Google LLC"], "allowedLicenses": ["Apache-2.0", "MIT", "BSD-3"], - "ignoreFiles": ["**/requirements.txt", "**/requirements-test.txt"], + "ignoreFiles": ["**/requirements.txt", "**/requirements-test.txt", "**/__init__.py", "samples/**/constraints.txt", "samples/**/constraints-test.txt"], "sourceFileExtensions": [ "ts", "js", diff --git a/.kokoro/release.sh b/.kokoro/release.sh index 2ef944a00..f8994b034 100755 --- a/.kokoro/release.sh +++ b/.kokoro/release.sh @@ -26,7 +26,7 @@ python3 -m pip install --upgrade twine wheel setuptools export PYTHONUNBUFFERED=1 # Move into the package, build the distribution and upload. -TWINE_PASSWORD=$(cat "${KOKORO_KEYSTORE_DIR}/73713_google_cloud_pypi_password") +TWINE_PASSWORD=$(cat "${KOKORO_GFILE_DIR}/secret_manager/google-cloud-pypi-token") cd github/python-logging python3 setup.py sdist bdist_wheel -twine upload --username gcloudpypi --password "${TWINE_PASSWORD}" dist/* +twine upload --username __token__ --password "${TWINE_PASSWORD}" dist/* diff --git a/.kokoro/release/common.cfg b/.kokoro/release/common.cfg index 9fedb82bb..e0012bf9c 100644 --- a/.kokoro/release/common.cfg +++ b/.kokoro/release/common.cfg @@ -23,18 +23,8 @@ env_vars: { value: "github/python-logging/.kokoro/release.sh" } -# Fetch PyPI password -before_action { - fetch_keystore { - keystore_resource { - keystore_config_id: 73713 - keyname: "google_cloud_pypi_password" - } - } -} - # Tokens needed to report release status back to GitHub env_vars: { key: "SECRET_MANAGER_KEYS" - value: "releasetool-publish-reporter-app,releasetool-publish-reporter-googleapis-installation,releasetool-publish-reporter-pem" -} \ No newline at end of file + value: "releasetool-publish-reporter-app,releasetool-publish-reporter-googleapis-installation,releasetool-publish-reporter-pem,google-cloud-pypi-token" +} diff --git a/.kokoro/samples/python3.6/periodic-head.cfg b/.kokoro/samples/python3.6/periodic-head.cfg new file mode 100644 index 000000000..f9cfcd33e --- /dev/null +++ b/.kokoro/samples/python3.6/periodic-head.cfg @@ -0,0 +1,11 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +env_vars: { + key: "INSTALL_LIBRARY_FROM_SOURCE" + value: "True" +} + +env_vars: { + key: "TRAMPOLINE_BUILD_FILE" + value: "github/python-pubsub/.kokoro/test-samples-against-head.sh" +} diff --git a/.kokoro/samples/python3.7/periodic-head.cfg b/.kokoro/samples/python3.7/periodic-head.cfg new file mode 100644 index 000000000..f9cfcd33e --- /dev/null +++ b/.kokoro/samples/python3.7/periodic-head.cfg @@ -0,0 +1,11 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +env_vars: { + key: "INSTALL_LIBRARY_FROM_SOURCE" + value: "True" +} + +env_vars: { + key: "TRAMPOLINE_BUILD_FILE" + value: "github/python-pubsub/.kokoro/test-samples-against-head.sh" +} diff --git a/.kokoro/samples/python3.8/periodic-head.cfg b/.kokoro/samples/python3.8/periodic-head.cfg new file mode 100644 index 000000000..f9cfcd33e --- /dev/null +++ b/.kokoro/samples/python3.8/periodic-head.cfg @@ -0,0 +1,11 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +env_vars: { + key: "INSTALL_LIBRARY_FROM_SOURCE" + value: "True" +} + +env_vars: { + key: "TRAMPOLINE_BUILD_FILE" + value: "github/python-pubsub/.kokoro/test-samples-against-head.sh" +} diff --git a/.kokoro/test-samples-against-head.sh b/.kokoro/test-samples-against-head.sh new file mode 100755 index 000000000..635a5ace0 --- /dev/null +++ b/.kokoro/test-samples-against-head.sh @@ -0,0 +1,28 @@ +#!/bin/bash +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# A customized test runner for samples. +# +# For periodic builds, you can specify this file for testing against head. + +# `-e` enables the script to automatically fail when a command fails +# `-o pipefail` sets the exit code to the rightmost comment to exit with a non-zero +set -eo pipefail +# Enables `**` to include files nested inside sub-folders +shopt -s globstar + +cd github/python-logging + +exec .kokoro/test-samples-impl.sh diff --git a/.kokoro/test-samples-impl.sh b/.kokoro/test-samples-impl.sh new file mode 100755 index 000000000..cf5de74c1 --- /dev/null +++ b/.kokoro/test-samples-impl.sh @@ -0,0 +1,102 @@ +#!/bin/bash +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +# `-e` enables the script to automatically fail when a command fails +# `-o pipefail` sets the exit code to the rightmost comment to exit with a non-zero +set -eo pipefail +# Enables `**` to include files nested inside sub-folders +shopt -s globstar + +# Exit early if samples directory doesn't exist +if [ ! -d "./samples" ]; then + echo "No tests run. `./samples` not found" + exit 0 +fi + +# Disable buffering, so that the logs stream through. +export PYTHONUNBUFFERED=1 + +# Debug: show build environment +env | grep KOKORO + +# Install nox +python3.6 -m pip install --upgrade --quiet nox + +# Use secrets acessor service account to get secrets +if [[ -f "${KOKORO_GFILE_DIR}/secrets_viewer_service_account.json" ]]; then + gcloud auth activate-service-account \ + --key-file="${KOKORO_GFILE_DIR}/secrets_viewer_service_account.json" \ + --project="cloud-devrel-kokoro-resources" +fi + +# This script will create 3 files: +# - testing/test-env.sh +# - testing/service-account.json +# - testing/client-secrets.json +./scripts/decrypt-secrets.sh + +source ./testing/test-env.sh +export GOOGLE_APPLICATION_CREDENTIALS=$(pwd)/testing/service-account.json + +# For cloud-run session, we activate the service account for gcloud sdk. +gcloud auth activate-service-account \ + --key-file "${GOOGLE_APPLICATION_CREDENTIALS}" + +export GOOGLE_CLIENT_SECRETS=$(pwd)/testing/client-secrets.json + +echo -e "\n******************** TESTING PROJECTS ********************" + +# Switch to 'fail at end' to allow all tests to complete before exiting. +set +e +# Use RTN to return a non-zero value if the test fails. +RTN=0 +ROOT=$(pwd) +# Find all requirements.txt in the samples directory (may break on whitespace). +for file in samples/**/requirements.txt; do + cd "$ROOT" + # Navigate to the project folder. + file=$(dirname "$file") + cd "$file" + + echo "------------------------------------------------------------" + echo "- testing $file" + echo "------------------------------------------------------------" + + # Use nox to execute the tests for the project. + python3.6 -m nox -s "$RUN_TESTS_SESSION" + EXIT=$? + + # If this is a periodic build, send the test log to the FlakyBot. + # See https://github.com/googleapis/repo-automation-bots/tree/master/packages/flakybot. + if [[ $KOKORO_BUILD_ARTIFACTS_SUBDIR = *"periodic"* ]]; then + chmod +x $KOKORO_GFILE_DIR/linux_amd64/flakybot + $KOKORO_GFILE_DIR/linux_amd64/flakybot + fi + + if [[ $EXIT -ne 0 ]]; then + RTN=1 + echo -e "\n Testing failed: Nox returned a non-zero exit code. \n" + else + echo -e "\n Testing completed.\n" + fi + +done +cd "$ROOT" + +# Workaround for Kokoro permissions issue: delete secrets +rm testing/{test-env.sh,client-secrets.json,service-account.json} + +exit "$RTN" diff --git a/.kokoro/test-samples.sh b/.kokoro/test-samples.sh index e75891832..0f5f8d400 100755 --- a/.kokoro/test-samples.sh +++ b/.kokoro/test-samples.sh @@ -13,6 +13,10 @@ # See the License for the specific language governing permissions and # limitations under the License. +# The default test runner for samples. +# +# For periodic builds, we rewinds the repo to the latest release, and +# run test-samples-impl.sh. # `-e` enables the script to automatically fail when a command fails # `-o pipefail` sets the exit code to the rightmost comment to exit with a non-zero @@ -24,87 +28,19 @@ cd github/python-logging # Run periodic samples tests at latest release if [[ $KOKORO_BUILD_ARTIFACTS_SUBDIR = *"periodic"* ]]; then + # preserving the test runner implementation. + cp .kokoro/test-samples-impl.sh "${TMPDIR}/test-samples-impl.sh" + echo "--- IMPORTANT IMPORTANT IMPORTANT ---" + echo "Now we rewind the repo back to the latest release..." LATEST_RELEASE=$(git describe --abbrev=0 --tags) git checkout $LATEST_RELEASE -fi - -# Exit early if samples directory doesn't exist -if [ ! -d "./samples" ]; then - echo "No tests run. `./samples` not found" - exit 0 -fi - -# Disable buffering, so that the logs stream through. -export PYTHONUNBUFFERED=1 - -# Debug: show build environment -env | grep KOKORO - -# Install nox -python3.6 -m pip install --upgrade --quiet nox - -# Use secrets acessor service account to get secrets -if [[ -f "${KOKORO_GFILE_DIR}/secrets_viewer_service_account.json" ]]; then - gcloud auth activate-service-account \ - --key-file="${KOKORO_GFILE_DIR}/secrets_viewer_service_account.json" \ - --project="cloud-devrel-kokoro-resources" -fi - -# This script will create 3 files: -# - testing/test-env.sh -# - testing/service-account.json -# - testing/client-secrets.json -./scripts/decrypt-secrets.sh - -source ./testing/test-env.sh -export GOOGLE_APPLICATION_CREDENTIALS=$(pwd)/testing/service-account.json - -# For cloud-run session, we activate the service account for gcloud sdk. -gcloud auth activate-service-account \ - --key-file "${GOOGLE_APPLICATION_CREDENTIALS}" - -export GOOGLE_CLIENT_SECRETS=$(pwd)/testing/client-secrets.json - -echo -e "\n******************** TESTING PROJECTS ********************" - -# Switch to 'fail at end' to allow all tests to complete before exiting. -set +e -# Use RTN to return a non-zero value if the test fails. -RTN=0 -ROOT=$(pwd) -# Find all requirements.txt in the samples directory (may break on whitespace). -for file in samples/**/requirements.txt; do - cd "$ROOT" - # Navigate to the project folder. - file=$(dirname "$file") - cd "$file" - - echo "------------------------------------------------------------" - echo "- testing $file" - echo "------------------------------------------------------------" - - # Use nox to execute the tests for the project. - python3.6 -m nox -s "$RUN_TESTS_SESSION" - EXIT=$? - - # If this is a periodic build, send the test log to the FlakyBot. - # See https://github.com/googleapis/repo-automation-bots/tree/master/packages/flakybot. - if [[ $KOKORO_BUILD_ARTIFACTS_SUBDIR = *"periodic"* ]]; then - chmod +x $KOKORO_GFILE_DIR/linux_amd64/flakybot - $KOKORO_GFILE_DIR/linux_amd64/flakybot + echo "The current head is: " + echo $(git rev-parse --verify HEAD) + echo "--- IMPORTANT IMPORTANT IMPORTANT ---" + # move back the test runner implementation if there's no file. + if [ ! -f .kokoro/test-samples-impl.sh ]; then + cp "${TMPDIR}/test-samples-impl.sh" .kokoro/test-samples-impl.sh fi +fi - if [[ $EXIT -ne 0 ]]; then - RTN=1 - echo -e "\n Testing failed: Nox returned a non-zero exit code. \n" - else - echo -e "\n Testing completed.\n" - fi - -done -cd "$ROOT" - -# Workaround for Kokoro permissions issue: delete secrets -rm testing/{test-env.sh,client-secrets.json,service-account.json} - -exit "$RTN" +exec .kokoro/test-samples-impl.sh diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index a9024b15d..1bbd78783 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,3 +1,17 @@ +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# # See https://pre-commit.com for more information # See https://pre-commit.com/hooks.html for more hooks repos: @@ -12,6 +26,6 @@ repos: hooks: - id: black - repo: https://gitlab.com/pycqa/flake8 - rev: 3.8.4 + rev: 3.9.1 hooks: - id: flake8 diff --git a/.repo-metadata.json b/.repo-metadata.json index 30541e78b..911d58dca 100644 --- a/.repo-metadata.json +++ b/.repo-metadata.json @@ -6,6 +6,7 @@ "issue_tracker": "https://issuetracker.google.com/savedsearches/559764", "release_level": "ga", "language": "python", + "library_type": "GAPIC_COMBO", "repo": "googleapis/python-logging", "distribution_name": "google-cloud-logging", "api_id": "logging.googleapis.com", diff --git a/CHANGELOG.md b/CHANGELOG.md index e7822c4d5..1828b3e69 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,27 @@ [1]: https://pypi.org/project/google-cloud-logging/#history +## [2.4.0](https://www.github.com/googleapis/python-logging/compare/v2.3.1...v2.4.0) (2021-05-12) + + +### Features + +* allow custom labels with standard library logging ([#264](https://www.github.com/googleapis/python-logging/issues/264)) ([fe4de39](https://www.github.com/googleapis/python-logging/commit/fe4de39a87581a9e9f2cee62462ae2f26176194f)) +* Improve source location overrides ([#258](https://www.github.com/googleapis/python-logging/issues/258)) ([6b10b74](https://www.github.com/googleapis/python-logging/commit/6b10b74e2bf65ea406b10585a4c24078348483d2)) +* record source locations ([#254](https://www.github.com/googleapis/python-logging/issues/254)) ([a5c2f8e](https://www.github.com/googleapis/python-logging/commit/a5c2f8e948bb116cbce313f063643aec02d06a84)) +* support span inference ([#267](https://www.github.com/googleapis/python-logging/issues/267)) ([fcd26eb](https://www.github.com/googleapis/python-logging/commit/fcd26eb0ff4f97c097ca33b2d212d8f83e56686e)) +* use standard output logs on serverless environments ([#228](https://www.github.com/googleapis/python-logging/issues/228)) ([a78f577](https://www.github.com/googleapis/python-logging/commit/a78f577bda17d758551237be84182035ed7b9cce)) + + +### Bug Fixes + +* changed region format on serverless ([#291](https://www.github.com/googleapis/python-logging/issues/291)) ([8872d6f](https://www.github.com/googleapis/python-logging/commit/8872d6f6b2bb979adffad0b054fa40306b68cfc0)) +* changed region format on serverless ([#291](https://www.github.com/googleapis/python-logging/issues/291)) ([360d3d2](https://www.github.com/googleapis/python-logging/commit/360d3d23db7709b7c3946c092ef373f888f47c3d)) +* **deps:** fix minimum required version of google-api-core ([#244](https://www.github.com/googleapis/python-logging/issues/244)) ([874fdfa](https://www.github.com/googleapis/python-logging/commit/874fdfa809063c2bfb33e59aded553e098601876)) +* **deps:** fix minimum required version of google-api-core ([#244](https://www.github.com/googleapis/python-logging/issues/244)) ([37d33fc](https://www.github.com/googleapis/python-logging/commit/37d33fcd8402b973377486a572c04ba6d4029b58)) +* improve API compatibility for next release ([#292](https://www.github.com/googleapis/python-logging/issues/292)) ([1f9517d](https://www.github.com/googleapis/python-logging/commit/1f9517da7302e19198e598d452df58238d4e6306)) +* remove noisy logs ([#290](https://www.github.com/googleapis/python-logging/issues/290)) ([bdf8273](https://www.github.com/googleapis/python-logging/commit/bdf827358de5935f736ecd73ab10b2d861daf690)) + ### [2.3.1](https://www.github.com/googleapis/python-logging/compare/v2.3.0...v2.3.1) (2021-03-24) diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index f6ddd7268..4604493b6 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -160,21 +160,7 @@ Running System Tests auth settings and change some configuration in your project to run all the tests. -- System tests will be run against an actual project and - so you'll need to provide some environment variables to facilitate - authentication to your project: - - - ``GOOGLE_APPLICATION_CREDENTIALS``: The path to a JSON key file; - Such a file can be downloaded directly from the developer's console by clicking - "Generate new JSON key". See private key - `docs `__ - for more details. - -- Once you have downloaded your json keys, set the environment variable - ``GOOGLE_APPLICATION_CREDENTIALS`` to the absolute path of the json file:: - - $ export GOOGLE_APPLICATION_CREDENTIALS="/Users//path/to/app_credentials.json" - +- System tests will be run against an actual project. You should use local credentials from gcloud when possible. See `Best practices for application authentication `__. Some tests require a service account. For those tests see `Authenticating as a service account `__. ************* Test Coverage diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 000000000..8b58ae9c0 --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,7 @@ +# Security Policy + +To report a security issue, please use [g.co/vulnz](https://g.co/vulnz). + +The Google Security Team will respond within 5 working days of your report on g.co/vulnz. + +We use g.co/vulnz for our intake, and do coordination and disclosure here using GitHub Security Advisory to privately discuss and fix the issue. diff --git a/docs/_static/custom.css b/docs/_static/custom.css index bcd37bbd3..b0a295464 100644 --- a/docs/_static/custom.css +++ b/docs/_static/custom.css @@ -1,9 +1,20 @@ div#python2-eol { border-color: red; border-width: medium; -} +} /* Ensure minimum width for 'Parameters' / 'Returns' column */ dl.field-list > dt { min-width: 100px } + +/* Insert space between methods for readability */ +dl.method { + padding-top: 10px; + padding-bottom: 10px +} + +/* Insert empty space between classes */ +dl.class { + padding-bottom: 50px +} diff --git a/docs/conf.py b/docs/conf.py index 6da1e2e79..8e1d46bc7 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -1,4 +1,17 @@ # -*- coding: utf-8 -*- +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. # # google-cloud-logging documentation build configuration file # diff --git a/google/cloud/logging/handlers/__init__.py b/google/cloud/logging/handlers/__init__.py index 29ed8f0d1..e27f8e673 100644 --- a/google/cloud/logging/handlers/__init__.py +++ b/google/cloud/logging/handlers/__init__.py @@ -16,12 +16,16 @@ from google.cloud.logging_v2.handlers.app_engine import AppEngineHandler from google.cloud.logging_v2.handlers.container_engine import ContainerEngineHandler +from google.cloud.logging_v2.handlers.structured_log import StructuredLogHandler +from google.cloud.logging_v2.handlers.handlers import CloudLoggingFilter from google.cloud.logging_v2.handlers.handlers import CloudLoggingHandler from google.cloud.logging_v2.handlers.handlers import setup_logging __all__ = [ "AppEngineHandler", + "CloudLoggingFilter", "CloudLoggingHandler", "ContainerEngineHandler", + "StructuredLogHandler", "setup_logging", ] diff --git a/google/cloud/logging_v2/client.py b/google/cloud/logging_v2/client.py index 17d853401..51d93355c 100644 --- a/google/cloud/logging_v2/client.py +++ b/google/cloud/logging_v2/client.py @@ -16,6 +16,7 @@ import logging import os +import sys try: from google.cloud.logging_v2 import _gapic @@ -36,6 +37,7 @@ from google.cloud.logging_v2.handlers import CloudLoggingHandler from google.cloud.logging_v2.handlers import AppEngineHandler from google.cloud.logging_v2.handlers import ContainerEngineHandler +from google.cloud.logging_v2.handlers import StructuredLogHandler from google.cloud.logging_v2.handlers import setup_logging from google.cloud.logging_v2.handlers.handlers import EXCLUDED_LOGGER_DEFAULTS from google.cloud.logging_v2.resource import Resource @@ -53,6 +55,7 @@ _GAE_RESOURCE_TYPE = "gae_app" _GKE_RESOURCE_TYPE = "k8s_container" _GCF_RESOURCE_TYPE = "cloud_function" +_RUN_RESOURCE_TYPE = "cloud_run_revision" class Client(ClientWithProject): @@ -347,18 +350,22 @@ def get_default_handler(self, **kw): """ monitored_resource = kw.pop("resource", detect_resource(self.project)) - if ( - isinstance(monitored_resource, Resource) - and monitored_resource.type == _GAE_RESOURCE_TYPE - ): - return AppEngineHandler(self, **kw) - elif ( - isinstance(monitored_resource, Resource) - and monitored_resource.type == _GKE_RESOURCE_TYPE - ): - return ContainerEngineHandler(**kw) - else: - return CloudLoggingHandler(self, resource=monitored_resource, **kw) + if isinstance(monitored_resource, Resource): + if monitored_resource.type == _GAE_RESOURCE_TYPE: + return AppEngineHandler(self, **kw) + elif monitored_resource.type == _GKE_RESOURCE_TYPE: + return ContainerEngineHandler(**kw) + elif ( + monitored_resource.type == _GCF_RESOURCE_TYPE + and sys.version_info[0] == 3 + and sys.version_info[1] >= 8 + ): + # Cloud Functions with runtimes > 3.8 supports structured logs on standard out + # 3.7 should use the standard CloudLoggingHandler, which sends logs over the network. + return StructuredLogHandler(**kw, project_id=self.project) + elif monitored_resource.type == _RUN_RESOURCE_TYPE: + return StructuredLogHandler(**kw, project_id=self.project) + return CloudLoggingHandler(self, resource=monitored_resource, **kw) def setup_logging( self, *, log_level=logging.INFO, excluded_loggers=EXCLUDED_LOGGER_DEFAULTS, **kw diff --git a/google/cloud/logging_v2/handlers/__init__.py b/google/cloud/logging_v2/handlers/__init__.py index 29ed8f0d1..a1ed08b5e 100644 --- a/google/cloud/logging_v2/handlers/__init__.py +++ b/google/cloud/logging_v2/handlers/__init__.py @@ -16,12 +16,16 @@ from google.cloud.logging_v2.handlers.app_engine import AppEngineHandler from google.cloud.logging_v2.handlers.container_engine import ContainerEngineHandler +from google.cloud.logging_v2.handlers.structured_log import StructuredLogHandler from google.cloud.logging_v2.handlers.handlers import CloudLoggingHandler +from google.cloud.logging_v2.handlers.handlers import CloudLoggingFilter from google.cloud.logging_v2.handlers.handlers import setup_logging __all__ = [ "AppEngineHandler", + "CloudLoggingFilter", "CloudLoggingHandler", "ContainerEngineHandler", + "StructuredLogHandler", "setup_logging", ] diff --git a/google/cloud/logging_v2/handlers/_helpers.py b/google/cloud/logging_v2/handlers/_helpers.py index 88eba07a6..931b7a2f5 100644 --- a/google/cloud/logging_v2/handlers/_helpers.py +++ b/google/cloud/logging_v2/handlers/_helpers.py @@ -16,6 +16,7 @@ import math import json +import re try: import flask @@ -55,12 +56,13 @@ def get_request_data_from_flask(): """Get http_request and trace data from flask request headers. Returns: - Tuple[Optional[dict], Optional[str]]: - Data related to the current http request and the trace_id for the - request. Both fields will be None if a flask request isn't found. + Tuple[Optional[dict], Optional[str], Optional[str]]: + Data related to the current http request, trace_id, and span_id for + the request. All fields will be None if a django request isn't + found. """ if flask is None or not flask.request: - return None, None + return None, None, None # build http_request http_request = { @@ -73,27 +75,26 @@ def get_request_data_from_flask(): "protocol": flask.request.environ.get(_PROTOCOL_HEADER), } - # find trace id - trace_id = None + # find trace id and span id header = flask.request.headers.get(_FLASK_TRACE_HEADER) - if header: - trace_id = header.split("/", 1)[0] + trace_id, span_id = _parse_trace_span(header) - return http_request, trace_id + return http_request, trace_id, span_id def get_request_data_from_django(): """Get http_request and trace data from django request headers. Returns: - Tuple[Optional[dict], Optional[str]]: - Data related to the current http request and the trace_id for the - request. Both fields will be None if a django request isn't found. + Tuple[Optional[dict], Optional[str], Optional[str]]: + Data related to the current http request, trace_id, and span_id for + the request. All fields will be None if a django request isn't + found. """ request = _get_django_request() if request is None: - return None, None + return None, None, None # convert content_length to int if it exists content_length = None @@ -101,6 +102,7 @@ def get_request_data_from_django(): content_length = int(request.META.get(_DJANGO_CONTENT_LENGTH)) except (ValueError, TypeError): content_length = None + # build http_request http_request = { "requestMethod": request.method, @@ -112,13 +114,35 @@ def get_request_data_from_django(): "protocol": request.META.get(_PROTOCOL_HEADER), } - # find trace id - trace_id = None + # find trace id and span id header = request.META.get(_DJANGO_TRACE_HEADER) - if header: - trace_id = header.split("/", 1)[0] + trace_id, span_id = _parse_trace_span(header) + + return http_request, trace_id, span_id - return http_request, trace_id + +def _parse_trace_span(header): + """Given an X_CLOUD_TRACE header, extract the trace and span ids. + + Args: + header (str): the string extracted from the X_CLOUD_TRACE header + Returns: + Tuple[Optional[dict], Optional[str]]: + The trace_id and span_id extracted from the header + Each field will be None if not found. + """ + trace_id = None + span_id = None + if header: + try: + split_header = header.split("/", 1) + trace_id = split_header[0] + header_suffix = split_header[1] + # the span is the set of alphanumeric characters after the / + span_id = re.findall(r"^\w+", header_suffix)[0] + except IndexError: + pass + return trace_id, span_id def get_request_data(): @@ -126,9 +150,10 @@ def get_request_data(): frameworks (currently supported: Flask and Django). Returns: - Tuple[Optional[dict], Optional[str]]: - Data related to the current http request and the trace_id for the - request. Both fields will be None if a supported web request isn't found. + Tuple[Optional[dict], Optional[str], Optional[str]]: + Data related to the current http request, trace_id, and span_id for + the request. All fields will be None if a django request isn't + found. """ checkers = ( get_request_data_from_django, @@ -136,8 +161,8 @@ def get_request_data(): ) for checker in checkers: - http_request, trace_id = checker() + http_request, trace_id, span_id = checker() if http_request is not None: - return http_request, trace_id + return http_request, trace_id, span_id - return None, None + return None, None, None diff --git a/google/cloud/logging_v2/handlers/_monitored_resources.py b/google/cloud/logging_v2/handlers/_monitored_resources.py index ad1de4d2b..e257f08e4 100644 --- a/google/cloud/logging_v2/handlers/_monitored_resources.py +++ b/google/cloud/logging_v2/handlers/_monitored_resources.py @@ -73,7 +73,7 @@ def _create_functions_resource(): labels={ "project_id": project, "function_name": function_name, - "region": region if region else "", + "region": region.split("/")[-1] if region else "", }, ) return resource @@ -131,7 +131,7 @@ def _create_cloud_run_resource(): "project_id": project, "service_name": os.environ.get(_CLOUD_RUN_SERVICE_ID, ""), "revision_name": os.environ.get(_CLOUD_RUN_REVISION_ID, ""), - "location": region if region else "", + "location": region.split("/")[-1] if region else "", "configuration_name": os.environ.get(_CLOUD_RUN_CONFIGURATION_ID, ""), }, ) diff --git a/google/cloud/logging_v2/handlers/app_engine.py b/google/cloud/logging_v2/handlers/app_engine.py index bc7daa9d0..874a9d608 100644 --- a/google/cloud/logging_v2/handlers/app_engine.py +++ b/google/cloud/logging_v2/handlers/app_engine.py @@ -90,7 +90,7 @@ def get_gae_labels(self): """ gae_labels = {} - _, trace_id = get_request_data() + _, trace_id, _ = get_request_data() if trace_id is not None: gae_labels[_TRACE_ID_LABEL] = trace_id @@ -107,7 +107,7 @@ def emit(self, record): record (logging.LogRecord): The record to be logged. """ message = super(AppEngineHandler, self).format(record) - inferred_http, inferred_trace = get_request_data() + inferred_http, inferred_trace, _ = get_request_data() if inferred_trace is not None: inferred_trace = f"projects/{self.project_id}/traces/{inferred_trace}" # allow user overrides diff --git a/google/cloud/logging_v2/handlers/handlers.py b/google/cloud/logging_v2/handlers/handlers.py index ffcc03ae2..b3b787fe2 100644 --- a/google/cloud/logging_v2/handlers/handlers.py +++ b/google/cloud/logging_v2/handlers/handlers.py @@ -14,20 +14,100 @@ """Python :mod:`logging` handlers for Cloud Logging.""" +import json import logging - from google.cloud.logging_v2.logger import _GLOBAL_RESOURCE from google.cloud.logging_v2.handlers.transports import BackgroundThreadTransport from google.cloud.logging_v2.handlers._monitored_resources import detect_resource +from google.cloud.logging_v2.handlers._helpers import get_request_data DEFAULT_LOGGER_NAME = "python" -EXCLUDED_LOGGER_DEFAULTS = ("google.cloud", "google.auth", "google_auth_httplib2") +"""Exclude internal logs from propagating through handlers""" +EXCLUDED_LOGGER_DEFAULTS = ( + "google.cloud", + "google.auth", + "google_auth_httplib2", + "google.api_core.bidi", + "werkzeug", +) _CLEAR_HANDLER_RESOURCE_TYPES = ("gae_app", "cloud_function") +class CloudLoggingFilter(logging.Filter): + """Python standard ``logging`` Filter class to add Cloud Logging + information to each LogRecord. + + When attached to a LogHandler, each incoming log will be modified + to include new Cloud Logging relevant data. This data can be manually + overwritten using the `extras` argument when writing logs. + """ + + # The subset of http_request fields have been tested to work consistently across GCP environments + # https://cloud.google.com/logging/docs/reference/v2/rest/v2/LogEntry#httprequest + _supported_http_fields = ("requestMethod", "requestUrl", "userAgent", "protocol") + + def __init__(self, project=None, default_labels=None): + self.project = project + self.default_labels = default_labels if default_labels else {} + + @staticmethod + def _infer_source_location(record): + """Helper function to infer source location data from a LogRecord. + Will default to record.source_location if already set + """ + if hasattr(record, "source_location"): + return record.source_location + else: + name_map = [ + ("line", "lineno"), + ("file", "pathname"), + ("function", "funcName"), + ] + output = {} + for (gcp_name, std_lib_name) in name_map: + value = getattr(record, std_lib_name, None) + if value is not None: + output[gcp_name] = value + return output if output else None + + def filter(self, record): + """ + Add new Cloud Logging data to each LogRecord as it comes in + """ + user_labels = getattr(record, "labels", {}) + # infer request data from the environment + inferred_http, inferred_trace, inferred_span = get_request_data() + if inferred_http is not None: + # filter inferred_http to include only well-supported fields + inferred_http = { + k: v + for (k, v) in inferred_http.items() + if k in self._supported_http_fields and v is not None + } + if inferred_trace is not None and self.project is not None: + # add full path for detected trace + inferred_trace = f"projects/{self.project}/traces/{inferred_trace}" + # set new record values + record._resource = getattr(record, "resource", None) + record._trace = getattr(record, "trace", inferred_trace) or None + record._span_id = getattr(record, "span_id", inferred_span) or None + record._http_request = getattr(record, "http_request", inferred_http) + record._source_location = CloudLoggingFilter._infer_source_location(record) + record._labels = {**self.default_labels, **user_labels} or None + # create string representations for structured logging + record._trace_str = record._trace or "" + record._span_id_str = record._span_id or "" + record._http_request_str = json.dumps(record._http_request or {}) + record._source_location_str = json.dumps(record._source_location or {}) + record._labels_str = json.dumps(record._labels or {}) + # break quotes for parsing through structured logging + record._msg_str = str(record.msg).replace('"', '\\"') if record.msg else "" + return True + + class CloudLoggingHandler(logging.StreamHandler): """Handler that directly makes Cloud Logging API calls. @@ -83,8 +163,7 @@ def __init__( option is :class:`.SyncTransport`. resource (~logging_v2.resource.Resource): Resource for this Handler. Defaults to ``global``. - labels (Optional[dict]): Monitored resource of the entry, defaults - to the global resource type. + labels (Optional[dict]): Additional labels to attach to logs. stream (Optional[IO]): Stream to be used by the handler. """ super(CloudLoggingHandler, self).__init__(stream) @@ -94,6 +173,9 @@ def __init__( self.project_id = client.project self.resource = resource self.labels = labels + # add extra keys to log record + log_filter = CloudLoggingFilter(project=self.project_id, default_labels=labels) + self.addFilter(log_filter) def emit(self, record): """Actually log the specified logging record. @@ -106,25 +188,16 @@ def emit(self, record): record (logging.LogRecord): The record to be logged. """ message = super(CloudLoggingHandler, self).format(record) - trace_id = getattr(record, "trace", None) - span_id = getattr(record, "span_id", None) - http_request = getattr(record, "http_request", None) - resource = getattr(record, "resource", self.resource) - user_labels = getattr(record, "labels", {}) - # merge labels - total_labels = self.labels if self.labels is not None else {} - total_labels.update(user_labels) - if len(total_labels) == 0: - total_labels = None # send off request self.transport.send( record, message, - resource=resource, - labels=(total_labels if total_labels else None), - trace=trace_id, - span_id=span_id, - http_request=http_request, + resource=(record._resource or self.resource), + labels=record._labels, + trace=record._trace, + span_id=record._span_id, + http_request=record._http_request, + source_location=record._source_location, ) @@ -169,6 +242,6 @@ def setup_logging( logger.setLevel(log_level) logger.addHandler(handler) for logger_name in all_excluded_loggers: + # prevent excluded loggers from propagating logs to handler logger = logging.getLogger(logger_name) logger.propagate = False - logger.addHandler(logging.StreamHandler()) diff --git a/google/cloud/logging_v2/handlers/structured_log.py b/google/cloud/logging_v2/handlers/structured_log.py new file mode 100644 index 000000000..0edb5c39e --- /dev/null +++ b/google/cloud/logging_v2/handlers/structured_log.py @@ -0,0 +1,64 @@ +# Copyright 2021 Google LLC All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Logging handler for printing formatted structured logs to standard output. +""" + +import logging.handlers + +from google.cloud.logging_v2.handlers.handlers import CloudLoggingFilter + +GCP_FORMAT = ( + '{"message": "%(_msg_str)s", ' + '"severity": "%(levelname)s", ' + '"logging.googleapis.com/labels": %(_labels_str)s, ' + '"logging.googleapis.com/trace": "%(_trace_str)s", ' + '"logging.googleapis.com/spanId": "%(_span_id_str)s", ' + '"logging.googleapis.com/sourceLocation": %(_source_location_str)s, ' + '"httpRequest": %(_http_request_str)s }' +) + + +class StructuredLogHandler(logging.StreamHandler): + """Handler to format logs into the Cloud Logging structured log format, + and write them to standard output + """ + + def __init__(self, *, labels=None, stream=None, project_id=None): + """ + Args: + labels (Optional[dict]): Additional labels to attach to logs. + stream (Optional[IO]): Stream to be used by the handler. + project (Optional[str]): Project Id associated with the logs. + """ + super(StructuredLogHandler, self).__init__(stream=stream) + self.project_id = project_id + + # add extra keys to log record + log_filter = CloudLoggingFilter(project=project_id, default_labels=labels) + self.addFilter(log_filter) + + # make logs appear in GCP structured logging format + self.formatter = logging.Formatter(GCP_FORMAT) + + def format(self, record): + """Format the message into structured log JSON. + Args: + record (logging.LogRecord): The log record. + Returns: + str: A JSON string formatted for GKE fluentd. + """ + + payload = self.formatter.format(record) + return payload diff --git a/logging-v2-py.tar.gz b/logging-v2-py.tar.gz new file mode 100644 index 000000000..e69de29bb diff --git a/noxfile.py b/noxfile.py index 1183ca5fb..493d67e6f 100644 --- a/noxfile.py +++ b/noxfile.py @@ -18,6 +18,7 @@ from __future__ import absolute_import import os +import pathlib import shutil import nox @@ -30,6 +31,8 @@ SYSTEM_TEST_PYTHON_VERSIONS = ["3.8"] UNIT_TEST_PYTHON_VERSIONS = ["3.6", "3.7", "3.8", "3.9"] +CURRENT_DIRECTORY = pathlib.Path(__file__).parent.absolute() + # 'docfx' is excluded since it only needs to run in 'docs-presubmit' nox.options.sessions = [ "unit", @@ -41,6 +44,9 @@ "docs", ] +# Error if a python version is missing +nox.options.error_on_missing_interpreters = True + @nox.session(python=DEFAULT_PYTHON_VERSION) def lint(session): @@ -56,16 +62,9 @@ def lint(session): session.run("flake8", "google", "tests") -@nox.session(python="3.6") +@nox.session(python=DEFAULT_PYTHON_VERSION) def blacken(session): - """Run black. - - Format code to uniform standard. - - This currently uses Python 3.6 due to the automated Kokoro run of synthtool. - That run uses an image that doesn't have 3.6 installed. Before updating this - check the state of the `gcp_ubuntu_config` we use for that Kokoro run. - """ + """Run black. Format code to uniform standard.""" session.install(BLACK_VERSION) session.run( "black", *BLACK_PATHS, @@ -81,11 +80,24 @@ def lint_setup_py(session): def default(session): # Install all test dependencies, then install this package in-place. - session.install("asyncmock", "pytest-asyncio") - session.install("mock", "pytest", "pytest-cov", "flask", "webob", "django") + constraints_path = str( + CURRENT_DIRECTORY / "testing" / f"constraints-{session.python}.txt" + ) + session.install("asyncmock", "pytest-asyncio", "-c", constraints_path) - session.install("-e", ".") + session.install( + "mock", + "pytest", + "pytest-cov", + "flask", + "webob", + "django", + "-c", + constraints_path, + ) + + session.install("-e", ".", "-c", constraints_path) # Run py.test against the unit tests. session.run( @@ -112,15 +124,15 @@ def unit(session): @nox.session(python=SYSTEM_TEST_PYTHON_VERSIONS) def system(session): """Run the system test suite.""" + constraints_path = str( + CURRENT_DIRECTORY / "testing" / f"constraints-{session.python}.txt" + ) system_test_path = os.path.join("tests", "system.py") system_test_folder_path = os.path.join("tests", "system") # Check the value of `RUN_SYSTEM_TESTS` env var. It defaults to true. if os.environ.get("RUN_SYSTEM_TESTS", "true") == "false": session.skip("RUN_SYSTEM_TESTS is set to false, skipping") - # Sanity check: Only run tests if the environment variable is set. - if not os.environ.get("GOOGLE_APPLICATION_CREDENTIALS", ""): - session.skip("Credentials must be set via environment variable") # Install pyopenssl for mTLS testing. if os.environ.get("GOOGLE_API_USE_CLIENT_CERTIFICATE", "false") == "true": session.install("pyopenssl") @@ -144,8 +156,10 @@ def system(session): "google-cloud-pubsub", "google-cloud-storage", "google-cloud-testutils", + "-c", + constraints_path, ) - session.install("-e", ".") + session.install("-e", ".", "-c", constraints_path) # Run py.test against the system tests. if system_test_exists: @@ -206,9 +220,7 @@ def docfx(session): """Build the docfx yaml files for this library.""" session.install("-e", ".") - # sphinx-docfx-yaml supports up to sphinx version 1.5.5. - # https://github.com/docascode/sphinx-docfx-yaml/issues/97 - session.install("sphinx==1.5.5", "alabaster", "recommonmark", "sphinx-docfx-yaml") + session.install("sphinx", "alabaster", "recommonmark", "gcp-sphinx-docfx-yaml") shutil.rmtree(os.path.join("docs", "_build"), ignore_errors=True) session.run( diff --git a/synth.py b/owlbot.py similarity index 69% rename from synth.py rename to owlbot.py index 7f7008a39..f012b1191 100644 --- a/synth.py +++ b/owlbot.py @@ -17,38 +17,33 @@ from synthtool import gcp from synthtool.languages import python -gapic = gcp.GAPICBazel() common = gcp.CommonTemplates() -# ---------------------------------------------------------------------------- -# Generate logging GAPIC layer -# ---------------------------------------------------------------------------- -library = gapic.py_library( - service="logging", - version="v2", - bazel_target="//google/logging/v2:logging-v2-py", - include_protos=True, -) +default_version = "v2" -s.move( - library, - excludes=[ - "setup.py", - "README.rst", - "google/cloud/logging/__init__.py", # generated types are hidden from users - "google/cloud/logging_v2/__init__.py", - "docs/index.rst", - "docs/logging_v2", # Don't include gapic library docs. Users should use the hand-written layer instead - "scripts/fixup_logging_v2_keywords.py", # don't include script since it only works for generated layer - ], -) +for library in s.get_staging_dirs(default_version): + if library.name == "v2": + # Fix generated unit tests + s.replace( + library / "tests/unit/gapic/logging_v2/test_logging_service_v2.py", + "MonitoredResource\(\s*type_", + "MonitoredResource(type" + ) -# Fix generated unit tests -s.replace( - "tests/unit/gapic/logging_v2/test_logging_service_v2.py", - "MonitoredResource\(\s*type_", - "MonitoredResource(type" -) + s.move( + library, + excludes=[ + "setup.py", + "README.rst", + "google/cloud/logging/__init__.py", # generated types are hidden from users + "google/cloud/logging_v2/__init__.py", + "docs/index.rst", + "docs/logging_v2", # Don't include gapic library docs. Users should use the hand-written layer instead + "scripts/fixup_logging_v2_keywords.py", # don't include script since it only works for generated layer + ], + ) + +s.remove_staging_dirs() # ---------------------------------------------------------------------------- # Add templated files diff --git a/renovate.json b/renovate.json index f08bc22c9..c04895563 100644 --- a/renovate.json +++ b/renovate.json @@ -2,5 +2,8 @@ "extends": [ "config:base", ":preserveSemverRanges" ], - "ignorePaths": [".pre-commit-config.yaml"] + "ignorePaths": [".pre-commit-config.yaml"], + "pip_requirements": { + "fileMatch": ["requirements-test.txt", "samples/[\\S/]*constraints.txt", "samples/[\\S/]*constraints-test.txt"] + } } diff --git a/samples/snippets/noxfile.py b/samples/snippets/noxfile.py index 97bf7da80..956cdf4f9 100644 --- a/samples/snippets/noxfile.py +++ b/samples/snippets/noxfile.py @@ -172,10 +172,16 @@ def blacken(session: nox.sessions.Session) -> None: def _session_tests(session: nox.sessions.Session, post_install: Callable = None) -> None: """Runs py.test for a particular project.""" if os.path.exists("requirements.txt"): - session.install("-r", "requirements.txt") + if os.path.exists("constraints.txt"): + session.install("-r", "requirements.txt", "-c", "constraints.txt") + else: + session.install("-r", "requirements.txt") if os.path.exists("requirements-test.txt"): - session.install("-r", "requirements-test.txt") + if os.path.exists("constraints-test.txt"): + session.install("-r", "requirements-test.txt", "-c", "constraints-test.txt") + else: + session.install("-r", "requirements-test.txt") if INSTALL_LIBRARY_FROM_SOURCE: session.install("-e", _get_repo_root()) diff --git a/samples/snippets/requirements-test.txt b/samples/snippets/requirements-test.txt index d0029c6de..766a8035d 100644 --- a/samples/snippets/requirements-test.txt +++ b/samples/snippets/requirements-test.txt @@ -1,2 +1,2 @@ backoff==1.10.0 -pytest==6.0.1 +pytest==6.2.4 diff --git a/samples/snippets/requirements.txt b/samples/snippets/requirements.txt index 0fb2ce1fa..489b0453d 100644 --- a/samples/snippets/requirements.txt +++ b/samples/snippets/requirements.txt @@ -1,4 +1,4 @@ -google-cloud-logging==2.3.0 -google-cloud-bigquery==2.12.0 -google-cloud-storage==1.36.2 -google-cloud-pubsub==2.4.0 +google-cloud-logging==2.3.1 +google-cloud-bigquery==2.16.0 +google-cloud-storage==1.38.0 +google-cloud-pubsub==2.4.2 diff --git a/setup.py b/setup.py index 326162660..36426fb60 100644 --- a/setup.py +++ b/setup.py @@ -22,14 +22,14 @@ name = "google-cloud-logging" description = "Stackdriver Logging API client library" -version = "2.3.1" +version = "2.4.0" # Should be one of: # 'Development Status :: 3 - Alpha' # 'Development Status :: 4 - Beta' # 'Development Status :: 5 - Production/Stable' release_status = "Development Status :: 5 - Production/Stable" dependencies = [ - "google-api-core[grpc] >= 1.22.0, < 2.0.0dev", + "google-api-core[grpc] >= 1.22.2, < 2.0.0dev", "google-cloud-core >= 1.4.1, < 2.0dev", "proto-plus >= 1.11.0", ] diff --git a/synth.metadata b/synth.metadata deleted file mode 100644 index b32612bb0..000000000 --- a/synth.metadata +++ /dev/null @@ -1,155 +0,0 @@ -{ - "sources": [ - { - "git": { - "name": ".", - "remote": "https://github.com/googleapis/python-logging.git", - "sha": "7246e7b18d75fe252928d93576fcbb4f3d4be1f2" - } - }, - { - "git": { - "name": "googleapis", - "remote": "https://github.com/googleapis/googleapis.git", - "sha": "149a3a84c29c9b8189576c7442ccb6dcf6a8f95b", - "internalRef": "364411656" - } - }, - { - "git": { - "name": "synthtool", - "remote": "https://github.com/googleapis/synthtool.git", - "sha": "2c54c473779ea731128cea61a3a6c975a08a5378" - } - }, - { - "git": { - "name": "synthtool", - "remote": "https://github.com/googleapis/synthtool.git", - "sha": "2c54c473779ea731128cea61a3a6c975a08a5378" - } - } - ], - "destinations": [ - { - "client": { - "source": "googleapis", - "apiName": "logging", - "apiVersion": "v2", - "language": "python", - "generator": "bazel" - } - } - ], - "generatedFiles": [ - ".coveragerc", - ".flake8", - ".github/CONTRIBUTING.md", - ".github/ISSUE_TEMPLATE/bug_report.md", - ".github/ISSUE_TEMPLATE/feature_request.md", - ".github/ISSUE_TEMPLATE/support_request.md", - ".github/PULL_REQUEST_TEMPLATE.md", - ".github/header-checker-lint.yml", - ".github/release-please.yml", - ".github/snippet-bot.yml", - ".gitignore", - ".kokoro/build.sh", - ".kokoro/continuous/common.cfg", - ".kokoro/continuous/continuous.cfg", - ".kokoro/docker/docs/Dockerfile", - ".kokoro/docker/docs/fetch_gpg_keys.sh", - ".kokoro/docs/common.cfg", - ".kokoro/docs/docs-presubmit.cfg", - ".kokoro/docs/docs.cfg", - ".kokoro/populate-secrets.sh", - ".kokoro/presubmit/common.cfg", - ".kokoro/presubmit/presubmit.cfg", - ".kokoro/publish-docs.sh", - ".kokoro/release.sh", - ".kokoro/release/common.cfg", - ".kokoro/release/release.cfg", - ".kokoro/samples/lint/common.cfg", - ".kokoro/samples/lint/continuous.cfg", - ".kokoro/samples/lint/periodic.cfg", - ".kokoro/samples/lint/presubmit.cfg", - ".kokoro/samples/python3.6/common.cfg", - ".kokoro/samples/python3.6/continuous.cfg", - ".kokoro/samples/python3.6/periodic.cfg", - ".kokoro/samples/python3.6/presubmit.cfg", - ".kokoro/samples/python3.7/common.cfg", - ".kokoro/samples/python3.7/continuous.cfg", - ".kokoro/samples/python3.7/periodic.cfg", - ".kokoro/samples/python3.7/presubmit.cfg", - ".kokoro/samples/python3.8/common.cfg", - ".kokoro/samples/python3.8/continuous.cfg", - ".kokoro/samples/python3.8/periodic.cfg", - ".kokoro/samples/python3.8/presubmit.cfg", - ".kokoro/test-samples.sh", - ".kokoro/trampoline.sh", - ".kokoro/trampoline_v2.sh", - ".pre-commit-config.yaml", - ".trampolinerc", - "CODE_OF_CONDUCT.md", - "CONTRIBUTING.rst", - "LICENSE", - "MANIFEST.in", - "docs/_static/custom.css", - "docs/_templates/layout.html", - "docs/conf.py", - "google/cloud/logging/py.typed", - "google/cloud/logging_v2/proto/log_entry.proto", - "google/cloud/logging_v2/proto/logging.proto", - "google/cloud/logging_v2/proto/logging_config.proto", - "google/cloud/logging_v2/proto/logging_metrics.proto", - "google/cloud/logging_v2/py.typed", - "google/cloud/logging_v2/services/__init__.py", - "google/cloud/logging_v2/services/config_service_v2/__init__.py", - "google/cloud/logging_v2/services/config_service_v2/async_client.py", - "google/cloud/logging_v2/services/config_service_v2/client.py", - "google/cloud/logging_v2/services/config_service_v2/pagers.py", - "google/cloud/logging_v2/services/config_service_v2/transports/__init__.py", - "google/cloud/logging_v2/services/config_service_v2/transports/base.py", - "google/cloud/logging_v2/services/config_service_v2/transports/grpc.py", - "google/cloud/logging_v2/services/config_service_v2/transports/grpc_asyncio.py", - "google/cloud/logging_v2/services/logging_service_v2/__init__.py", - "google/cloud/logging_v2/services/logging_service_v2/async_client.py", - "google/cloud/logging_v2/services/logging_service_v2/client.py", - "google/cloud/logging_v2/services/logging_service_v2/pagers.py", - "google/cloud/logging_v2/services/logging_service_v2/transports/__init__.py", - "google/cloud/logging_v2/services/logging_service_v2/transports/base.py", - "google/cloud/logging_v2/services/logging_service_v2/transports/grpc.py", - "google/cloud/logging_v2/services/logging_service_v2/transports/grpc_asyncio.py", - "google/cloud/logging_v2/services/metrics_service_v2/__init__.py", - "google/cloud/logging_v2/services/metrics_service_v2/async_client.py", - "google/cloud/logging_v2/services/metrics_service_v2/client.py", - "google/cloud/logging_v2/services/metrics_service_v2/pagers.py", - "google/cloud/logging_v2/services/metrics_service_v2/transports/__init__.py", - "google/cloud/logging_v2/services/metrics_service_v2/transports/base.py", - "google/cloud/logging_v2/services/metrics_service_v2/transports/grpc.py", - "google/cloud/logging_v2/services/metrics_service_v2/transports/grpc_asyncio.py", - "google/cloud/logging_v2/types/__init__.py", - "google/cloud/logging_v2/types/log_entry.py", - "google/cloud/logging_v2/types/logging.py", - "google/cloud/logging_v2/types/logging_config.py", - "google/cloud/logging_v2/types/logging_metrics.py", - "mypy.ini", - "noxfile.py", - "renovate.json", - "samples/AUTHORING_GUIDE.md", - "samples/CONTRIBUTING.md", - "samples/snippets/noxfile.py", - "scripts/decrypt-secrets.sh", - "scripts/readme-gen/readme_gen.py", - "scripts/readme-gen/templates/README.tmpl.rst", - "scripts/readme-gen/templates/auth.tmpl.rst", - "scripts/readme-gen/templates/auth_api_key.tmpl.rst", - "scripts/readme-gen/templates/install_deps.tmpl.rst", - "scripts/readme-gen/templates/install_portaudio.tmpl.rst", - "setup.cfg", - "testing/.gitignore", - "tests/unit/gapic/logging_v2/__init__.py", - "tests/unit/gapic/logging_v2/test_config_service_v2.py", - "tests/unit/gapic/logging_v2/test_logging_service_v2.py", - "tests/unit/gapic/logging_v2/test_metrics_service_v2.py" - ] -} \ No newline at end of file diff --git a/testing/constraints-3.6.txt b/testing/constraints-3.6.txt index 0e0bdeb0b..ae89ab4a1 100644 --- a/testing/constraints-3.6.txt +++ b/testing/constraints-3.6.txt @@ -5,6 +5,6 @@ # # e.g., if setup.py has "foo >= 1.14.0, < 2.0.0dev", # Then this file should have foo==1.14.0 -google-api-core==1.22.0 +google-api-core==1.22.2 google-cloud-core==1.4.1 -proto-plus==1.11.0 \ No newline at end of file +proto-plus==1.11.0 diff --git a/tests/environment b/tests/environment index 1962721db..30d6a8083 160000 --- a/tests/environment +++ b/tests/environment @@ -1 +1 @@ -Subproject commit 1962721db8aa382bb1f658921979a1c183bf2d1a +Subproject commit 30d6a80838a1cae6fb3945f41f3e1d90e815c0c9 diff --git a/tests/system/test_system.py b/tests/system/test_system.py index e6f5aa7cf..cc6d03804 100644 --- a/tests/system/test_system.py +++ b/tests/system/test_system.py @@ -333,10 +333,12 @@ def test_handlers_w_extras(self): cloud_logger = logging.getLogger(LOGGER_NAME) cloud_logger.addHandler(handler) expected_request = {"requestUrl": "localhost"} + expected_source = {"file": "test.py"} extra = { "trace": "123", "span_id": "456", "http_request": expected_request, + "source_location": expected_source, "resource": Resource(type="cloudiot_device", labels={}), "labels": {"test-label": "manual"}, } diff --git a/tests/unit/handlers/test__helpers.py b/tests/unit/handlers/test__helpers.py index fd17f6ffd..d26e700e8 100644 --- a/tests/unit/handlers/test__helpers.py +++ b/tests/unit/handlers/test__helpers.py @@ -17,8 +17,10 @@ import mock _FLASK_TRACE_ID = "flask-id" +_FLASK_SPAN_ID = "span0flask" _FLASK_HTTP_REQUEST = {"requestUrl": "https://flask.palletsprojects.com/en/1.1.x/"} _DJANGO_TRACE_ID = "django-id" +_DJANGO_SPAN_ID = "span0django" _DJANGO_HTTP_REQUEST = {"requestUrl": "https://www.djangoproject.com/"} @@ -44,15 +46,17 @@ def index(): def test_no_context_header(self): app = self.create_app() with app.test_request_context(path="/", headers={}): - http_request, trace_id = self._call_fut() + http_request, trace_id, span_id = self._call_fut() self.assertIsNone(trace_id) + self.assertIsNone(span_id) self.assertEqual(http_request["requestMethod"], "GET") def test_valid_context_header(self): flask_trace_header = "X_CLOUD_TRACE_CONTEXT" expected_trace_id = _FLASK_TRACE_ID - flask_trace_id = expected_trace_id + "/testspanid" + expected_span_id = _FLASK_SPAN_ID + flask_trace_id = f"{expected_trace_id}/{expected_span_id}" app = self.create_app() context = app.test_request_context( @@ -60,9 +64,10 @@ def test_valid_context_header(self): ) with context: - http_request, trace_id = self._call_fut() + http_request, trace_id, span_id = self._call_fut() self.assertEqual(trace_id, expected_trace_id) + self.assertEqual(span_id, expected_span_id) self.assertEqual(http_request["requestMethod"], "GET") def test_http_request_populated(self): @@ -70,37 +75,28 @@ def test_http_request_populated(self): expected_agent = "Mozilla/5.0" expected_referrer = "self" expected_ip = "10.1.2.3" - body_content = "test" headers = { "User-Agent": expected_agent, "Referer": expected_referrer, } app = self.create_app() - with app.test_client() as c: - c.put( - path=expected_path, - data=body_content, - environ_base={"REMOTE_ADDR": expected_ip}, - headers=headers, - ) - http_request, trace_id = self._call_fut() + with app.test_request_context( + expected_path, headers=headers, environ_base={"REMOTE_ADDR": expected_ip} + ): + http_request, *_ = self._call_fut() - self.assertEqual(http_request["requestMethod"], "PUT") + self.assertEqual(http_request["requestMethod"], "GET") self.assertEqual(http_request["requestUrl"], expected_path) self.assertEqual(http_request["userAgent"], expected_agent) - self.assertEqual(http_request["referer"], expected_referrer) - self.assertEqual(http_request["remoteIp"], expected_ip) - self.assertEqual(http_request["requestSize"], len(body_content)) self.assertEqual(http_request["protocol"], "HTTP/1.1") def test_http_request_sparse(self): expected_path = "http://testserver/123" app = self.create_app() - with app.test_client() as c: - c.put(path=expected_path) - http_request, trace_id = self._call_fut() - self.assertEqual(http_request["requestMethod"], "PUT") + with app.test_request_context(expected_path): + http_request, *_ = self._call_fut() + self.assertEqual(http_request["requestMethod"], "GET") self.assertEqual(http_request["requestUrl"], expected_path) self.assertEqual(http_request["protocol"], "HTTP/1.1") @@ -135,17 +131,20 @@ def test_no_context_header(self): middleware = request.RequestMiddleware(None) middleware.process_request(django_request) - http_request, trace_id = self._call_fut() + http_request, trace_id, span_id = self._call_fut() + self.assertEqual(http_request["requestMethod"], "GET") self.assertIsNone(trace_id) + self.assertIsNone(span_id) def test_valid_context_header(self): from django.test import RequestFactory from google.cloud.logging_v2.handlers.middleware import request django_trace_header = "HTTP_X_CLOUD_TRACE_CONTEXT" - expected_trace_id = "testtraceiddjango" - django_trace_id = expected_trace_id + "/testspanid" + expected_span_id = _DJANGO_SPAN_ID + expected_trace_id = _DJANGO_TRACE_ID + django_trace_id = f"{expected_trace_id}/{expected_span_id}" django_request = RequestFactory().get( "/", **{django_trace_header: django_trace_id} @@ -153,9 +152,10 @@ def test_valid_context_header(self): middleware = request.RequestMiddleware(None) middleware.process_request(django_request) - http_request, trace_id = self._call_fut() + http_request, trace_id, span_id = self._call_fut() self.assertEqual(trace_id, expected_trace_id) + self.assertEqual(span_id, expected_span_id) self.assertEqual(http_request["requestMethod"], "GET") def test_http_request_populated(self): @@ -178,13 +178,10 @@ def test_http_request_populated(self): middleware = request.RequestMiddleware(None) middleware.process_request(django_request) - http_request, trace_id = self._call_fut() + http_request, *_ = self._call_fut() self.assertEqual(http_request["requestMethod"], "PUT") self.assertEqual(http_request["requestUrl"], expected_path) self.assertEqual(http_request["userAgent"], expected_agent) - self.assertEqual(http_request["referer"], expected_referrer) - self.assertEqual(http_request["remoteIp"], "127.0.0.1") - self.assertEqual(http_request["requestSize"], len(body_content)) self.assertEqual(http_request["protocol"], "HTTP/1.1") def test_http_request_sparse(self): @@ -195,10 +192,9 @@ def test_http_request_sparse(self): django_request = RequestFactory().put(expected_path) middleware = request.RequestMiddleware(None) middleware.process_request(django_request) - http_request, trace_id = self._call_fut() + http_request, *_ = self._call_fut() self.assertEqual(http_request["requestMethod"], "PUT") self.assertEqual(http_request["requestUrl"], expected_path) - self.assertEqual(http_request["remoteIp"], "127.0.0.1") self.assertEqual(http_request["protocol"], "HTTP/1.1") @@ -226,8 +222,8 @@ def _helper(self, django_return, flask_return): return django_mock, flask_mock, result def test_from_django(self): - django_expected = (_DJANGO_HTTP_REQUEST, _DJANGO_TRACE_ID) - flask_expected = (None, None) + django_expected = (_DJANGO_HTTP_REQUEST, _DJANGO_TRACE_ID, _DJANGO_SPAN_ID) + flask_expected = (None, None, None) django_mock, flask_mock, output = self._helper(django_expected, flask_expected) self.assertEqual(output, django_expected) @@ -235,8 +231,8 @@ def test_from_django(self): flask_mock.assert_not_called() def test_from_flask(self): - django_expected = (None, None) - flask_expected = (_FLASK_HTTP_REQUEST, _FLASK_TRACE_ID) + django_expected = (None, None, None) + flask_expected = (_FLASK_HTTP_REQUEST, _FLASK_TRACE_ID, _FLASK_SPAN_ID) django_mock, flask_mock, output = self._helper(django_expected, flask_expected) self.assertEqual(output, flask_expected) @@ -245,8 +241,8 @@ def test_from_flask(self): flask_mock.assert_called_once_with() def test_from_django_and_flask(self): - django_expected = (_DJANGO_HTTP_REQUEST, _DJANGO_TRACE_ID) - flask_expected = (_FLASK_HTTP_REQUEST, _FLASK_TRACE_ID) + django_expected = (_DJANGO_HTTP_REQUEST, _DJANGO_TRACE_ID, _DJANGO_SPAN_ID) + flask_expected = (_FLASK_HTTP_REQUEST, _FLASK_TRACE_ID, _FLASK_SPAN_ID) django_mock, flask_mock, output = self._helper(django_expected, flask_expected) @@ -257,19 +253,19 @@ def test_from_django_and_flask(self): flask_mock.assert_not_called() def test_missing_http_request(self): - flask_expected = (None, _FLASK_TRACE_ID) - django_expected = (None, _DJANGO_TRACE_ID) + flask_expected = (None, _FLASK_TRACE_ID, _FLASK_SPAN_ID) + django_expected = (None, _DJANGO_TRACE_ID, _DJANGO_TRACE_ID) django_mock, flask_mock, output = self._helper(django_expected, flask_expected) # function only returns trace if http_request data is present - self.assertEqual(output, (None, None)) + self.assertEqual(output, (None, None, None)) django_mock.assert_called_once_with() flask_mock.assert_called_once_with() def test_missing_trace_id(self): - flask_expected = (_FLASK_HTTP_REQUEST, None) - django_expected = (None, _DJANGO_TRACE_ID) + flask_expected = (_FLASK_HTTP_REQUEST, None, None) + django_expected = (None, _DJANGO_TRACE_ID, _DJANGO_SPAN_ID) django_mock, flask_mock, output = self._helper(django_expected, flask_expected) # trace_id is optional @@ -279,14 +275,77 @@ def test_missing_trace_id(self): flask_mock.assert_called_once_with() def test_missing_both(self): - flask_expected = (None, None) - django_expected = (None, None) + flask_expected = (None, None, None) + django_expected = (None, None, None) django_mock, flask_mock, output = self._helper(django_expected, flask_expected) - self.assertEqual(output, (None, None)) + self.assertEqual(output, (None, None, None)) django_mock.assert_called_once_with() flask_mock.assert_called_once_with() def test_wo_libraries(self): output = self._call_fut() - self.assertEqual(output, (None, None)) + self.assertEqual(output, (None, None, None)) + + +class Test__parse_trace_span(unittest.TestCase): + @staticmethod + def _call_fut(header): + from google.cloud.logging_v2.handlers import _helpers + + return _helpers._parse_trace_span(header) + + def test_empty_header(self): + header = "" + trace_id, span_id = self._call_fut(header) + self.assertEqual(trace_id, None) + self.assertEqual(span_id, None) + + def test_no_span(self): + header = "12345" + trace_id, span_id = self._call_fut(header) + self.assertEqual(trace_id, header) + self.assertEqual(span_id, None) + + def test_no_trace(self): + header = "/12345" + trace_id, span_id = self._call_fut(header) + self.assertEqual(trace_id, "") + self.assertEqual(span_id, "12345") + + def test_with_span(self): + expected_trace = "12345" + expected_span = "67890" + header = f"{expected_trace}/{expected_span}" + trace_id, span_id = self._call_fut(header) + self.assertEqual(trace_id, expected_trace) + self.assertEqual(span_id, expected_span) + + def test_with_extra_characters(self): + expected_trace = "12345" + expected_span = "67890" + header = f"{expected_trace}/{expected_span};o=0" + trace_id, span_id = self._call_fut(header) + self.assertEqual(trace_id, expected_trace) + self.assertEqual(span_id, expected_span) + + def test_with_unicode_span(self): + """ + Spans are expected to be alphanumeric + """ + expected_trace = "12345" + header = f"{expected_trace}/😀123" + trace_id, span_id = self._call_fut(header) + self.assertEqual(trace_id, expected_trace) + self.assertEqual(span_id, None) + + def test_with_unicode_trace(self): + """ + Spans are expected to be alphanumeric + """ + expected_trace = "12😀345" + expected_span = "67890" + header = f"{expected_trace}/{expected_span}" + trace_id, span_id = self._call_fut(header) + self.assertEqual(trace_id, expected_trace) + self.assertEqual(span_id, expected_span) diff --git a/tests/unit/handlers/test_app_engine.py b/tests/unit/handlers/test_app_engine.py index 65e804573..c726c8496 100644 --- a/tests/unit/handlers/test_app_engine.py +++ b/tests/unit/handlers/test_app_engine.py @@ -97,7 +97,7 @@ def test_emit(self): expected_trace_id = f"projects/{self.PROJECT}/traces/{trace_id}" get_request_patch = mock.patch( "google.cloud.logging_v2.handlers.app_engine.get_request_data", - return_value=(expected_http_request, trace_id), + return_value=(expected_http_request, trace_id, None), ) with get_request_patch: # library integrations mocked to return test data @@ -135,7 +135,7 @@ def test_emit_manual_field_override(self): inferred_trace_id = "trace-test" get_request_patch = mock.patch( "google.cloud.logging_v2.handlers.app_engine.get_request_data", - return_value=(inferred_http_request, inferred_trace_id), + return_value=(inferred_http_request, inferred_trace_id, None), ) with get_request_patch: # library integrations mocked to return test data @@ -180,7 +180,7 @@ def test_emit_manual_field_override(self): def _get_gae_labels_helper(self, trace_id): get_request_patch = mock.patch( "google.cloud.logging_v2.handlers.app_engine.get_request_data", - return_value=(None, trace_id), + return_value=(None, trace_id, None), ) client = mock.Mock(project=self.PROJECT, spec=["project"]) diff --git a/tests/unit/handlers/test_handlers.py b/tests/unit/handlers/test_handlers.py index 80a1368b8..4ba052121 100644 --- a/tests/unit/handlers/test_handlers.py +++ b/tests/unit/handlers/test_handlers.py @@ -16,6 +16,7 @@ import unittest from unittest.mock import patch import mock +import json from google.cloud.logging_v2.handlers._monitored_resources import ( _FUNCTION_ENV_VARS, @@ -23,6 +24,204 @@ ) +class TestCloudLoggingFilter(unittest.TestCase): + + PROJECT = "PROJECT" + + @staticmethod + def _get_target_class(): + from google.cloud.logging.handlers import CloudLoggingFilter + + return CloudLoggingFilter + + def _make_one(self, *args, **kw): + return self._get_target_class()(*args, **kw) + + @staticmethod + def create_app(): + import flask + + app = flask.Flask(__name__) + + @app.route("/") + def index(): + return "test flask trace" # pragma: NO COVER + + return app + + def test_filter_record(self): + """ + test adding fields to a standard record + """ + import logging + + filter_obj = self._make_one() + logname = "loggername" + message = "hello world，嗨 世界" + expected_location = { + "line": 1, + "file": "testpath", + "function": "test-function", + } + record = logging.LogRecord( + logname, + logging.INFO, + expected_location["file"], + expected_location["line"], + message, + None, + None, + func=expected_location["function"], + ) + + success = filter_obj.filter(record) + self.assertTrue(success) + + self.assertEqual(record.msg, message) + self.assertEqual(record._msg_str, message) + self.assertEqual(record._source_location, expected_location) + self.assertEqual(record._source_location_str, json.dumps(expected_location)) + self.assertIsNone(record._resource) + self.assertIsNone(record._trace) + self.assertEqual(record._trace_str, "") + self.assertIsNone(record._span_id) + self.assertEqual(record._span_id_str, "") + self.assertIsNone(record._http_request) + self.assertEqual(record._http_request_str, "{}") + self.assertIsNone(record._labels) + self.assertEqual(record._labels_str, "{}") + + def test_minimal_record(self): + """ + test filter adds empty strings on missing attributes + """ + import logging + + filter_obj = self._make_one() + record = logging.LogRecord(None, logging.INFO, None, None, None, None, None,) + record.created = None + + success = filter_obj.filter(record) + self.assertTrue(success) + + self.assertIsNone(record.msg) + self.assertEqual(record._msg_str, "") + self.assertIsNone(record._source_location) + self.assertEqual(record._source_location_str, "{}") + self.assertIsNone(record._resource) + self.assertIsNone(record._trace) + self.assertEqual(record._trace_str, "") + self.assertIsNone(record._span_id) + self.assertEqual(record._span_id_str, "") + self.assertIsNone(record._http_request) + self.assertEqual(record._http_request_str, "{}") + self.assertIsNone(record._labels) + self.assertEqual(record._labels_str, "{}") + + def test_record_with_request(self): + """ + test filter adds http request data when available + """ + import logging + + filter_obj = self._make_one() + record = logging.LogRecord(None, logging.INFO, None, None, None, None, None,) + record.created = None + + expected_path = "http://testserver/123" + expected_agent = "Mozilla/5.0" + expected_trace = "123" + expected_span = "456" + combined_trace = f"{expected_trace}/{expected_span}" + expected_request = { + "requestMethod": "GET", + "requestUrl": expected_path, + "userAgent": expected_agent, + "protocol": "HTTP/1.1", + } + + app = self.create_app() + with app.test_request_context( + expected_path, + headers={ + "User-Agent": expected_agent, + "X_CLOUD_TRACE_CONTEXT": combined_trace, + }, + ): + success = filter_obj.filter(record) + self.assertTrue(success) + + self.assertEqual(record._trace, expected_trace) + self.assertEqual(record._trace_str, expected_trace) + self.assertEqual(record._span_id, expected_span) + self.assertEqual(record._span_id_str, expected_span) + self.assertEqual(record._http_request, expected_request) + self.assertEqual(record._http_request_str, json.dumps(expected_request)) + + def test_user_overrides(self): + """ + ensure user can override fields + """ + import logging + + filter_obj = self._make_one() + record = logging.LogRecord( + "name", logging.INFO, "default", 99, "message", None, None, func="default" + ) + record.created = 5.03 + + app = self.create_app() + with app.test_client() as c: + c.put( + path="http://testserver/123", + data="body", + headers={"User-Agent": "default", "X_CLOUD_TRACE_CONTEXT": "default"}, + ) + # override values + overwritten_resource = "test" + record.resource = overwritten_resource + overwritten_trace = "456" + record.trace = overwritten_trace + overwritten_span = "789" + record.span_id = overwritten_span + overwritten_method = "GET" + overwritten_url = "www.google.com" + overwritten_agent = "custom" + overwritten_protocol = "test" + overwritten_request_object = { + "requestMethod": overwritten_method, + "requestUrl": overwritten_url, + "userAgent": overwritten_agent, + "protocol": overwritten_protocol, + } + overwritten_line = 22 + overwritten_function = "test-func" + overwritten_file = "test-file" + overwritten_source_location = { + "file": overwritten_file, + "line": overwritten_line, + "function": overwritten_function, + } + record.http_request = overwritten_request_object + record.source_location = overwritten_source_location + success = filter_obj.filter(record) + self.assertTrue(success) + + self.assertEqual(record._trace, overwritten_trace) + self.assertEqual(record._trace_str, overwritten_trace) + self.assertEqual(record._span_id, overwritten_span) + self.assertEqual(record._span_id_str, overwritten_span) + self.assertEqual(record._http_request, overwritten_request_object) + self.assertEqual( + record._http_request_str, json.dumps(overwritten_request_object) + ) + self.assertEqual(record._source_location, overwritten_source_location) + self.assertEqual( + record._source_location_str, json.dumps(overwritten_source_location) + ) + self.assertEqual(record._resource, overwritten_resource) + + class TestCloudLoggingHandler(unittest.TestCase): PROJECT = "PROJECT" @@ -93,11 +292,10 @@ def test_emit(self): logname = "loggername" message = "hello world" record = logging.LogRecord(logname, logging, None, None, message, None, None) - handler.emit(record) - + handler.handle(record) self.assertEqual( handler.transport.send_called_with, - (record, message, _GLOBAL_RESOURCE, None, None, None, None), + (record, message, _GLOBAL_RESOURCE, None, None, None, None, None), ) def test_emit_manual_field_override(self): @@ -105,8 +303,15 @@ def test_emit_manual_field_override(self): from google.cloud.logging_v2.resource import Resource client = _Client(self.PROJECT) + default_labels = { + "default_key": "default-value", + "overwritten_key": "bad_value", + } handler = self._make_one( - client, transport=_Transport, resource=_GLOBAL_RESOURCE + client, + transport=_Transport, + resource=_GLOBAL_RESOURCE, + labels=default_labels, ) logname = "loggername" message = "hello world" @@ -118,11 +323,18 @@ def test_emit_manual_field_override(self): setattr(record, "span_id", expected_span) expected_http = {"reuqest_url": "manual"} setattr(record, "http_request", expected_http) + expected_source = {"file": "test-file"} + setattr(record, "source_location", expected_source) expected_resource = Resource(type="test", labels={}) setattr(record, "resource", expected_resource) - expected_labels = {"test-label": "manual"} - setattr(record, "labels", expected_labels) - handler.emit(record) + added_labels = {"added_key": "added_value", "overwritten_key": "new_value"} + expected_labels = { + "default_key": "default-value", + "overwritten_key": "new_value", + "added_key": "added_value", + } + setattr(record, "labels", added_labels) + handler.handle(record) self.assertEqual( handler.transport.send_called_with, @@ -134,6 +346,7 @@ def test_emit_manual_field_override(self): expected_trace, expected_span, expected_http, + expected_source, ), ) @@ -249,6 +462,7 @@ def send( trace=None, span_id=None, http_request=None, + source_location=None, ): self.send_called_with = ( record, @@ -258,4 +472,5 @@ def send( trace, span_id, http_request, + source_location, ) diff --git a/tests/unit/handlers/test_structured_log.py b/tests/unit/handlers/test_structured_log.py new file mode 100644 index 000000000..0536583a5 --- /dev/null +++ b/tests/unit/handlers/test_structured_log.py @@ -0,0 +1,213 @@ +# Copyright 2021 Google LLC All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import unittest + + +class TestStructuredLogHandler(unittest.TestCase): + PROJECT = "PROJECT" + + def _get_target_class(self): + from google.cloud.logging.handlers import StructuredLogHandler + + return StructuredLogHandler + + def _make_one(self, *args, **kw): + return self._get_target_class()(*args, **kw) + + @staticmethod + def create_app(): + import flask + + app = flask.Flask(__name__) + + @app.route("/") + def index(): + return "test flask trace" # pragma: NO COVER + + return app + + def test_ctor_defaults(self): + handler = self._make_one() + self.assertIsNone(handler.project_id) + + def test_ctor_w_project(self): + handler = self._make_one(project_id="foo") + self.assertEqual(handler.project_id, "foo") + + def test_format(self): + import logging + import json + + labels = {"default_key": "default-value"} + handler = self._make_one(labels=labels) + logname = "loggername" + message = "hello world，嗨 世界" + pathname = "testpath" + lineno = 1 + func = "test-function" + record = logging.LogRecord( + logname, logging.INFO, pathname, lineno, message, None, None, func=func + ) + expected_payload = { + "message": message, + "severity": record.levelname, + "logging.googleapis.com/trace": "", + "logging.googleapis.com/spanId": "", + "logging.googleapis.com/sourceLocation": { + "file": pathname, + "line": lineno, + "function": func, + }, + "httpRequest": {}, + "logging.googleapis.com/labels": labels, + } + handler.filter(record) + result = json.loads(handler.format(record)) + for (key, value) in expected_payload.items(): + self.assertEqual(value, result[key]) + self.assertEqual( + len(expected_payload.keys()), + len(result.keys()), + f"result dictionary has unexpected keys: {result.keys()}", + ) + + def test_format_minimal(self): + import logging + import json + + handler = self._make_one() + record = logging.LogRecord(None, logging.INFO, None, None, None, None, None,) + record.created = None + expected_payload = { + "message": "", + "logging.googleapis.com/trace": "", + "logging.googleapis.com/sourceLocation": {}, + "httpRequest": {}, + "logging.googleapis.com/labels": {}, + } + handler.filter(record) + result = json.loads(handler.format(record)) + for (key, value) in expected_payload.items(): + self.assertEqual( + value, result[key], f"expected_payload[{key}] != result[{key}]" + ) + + def test_format_with_quotes(self): + """ + When logging a message containing quotes, escape chars should be added + """ + import logging + import json + + handler = self._make_one() + message = '"test"' + expected_result = '\\"test\\"' + record = logging.LogRecord(None, logging.INFO, None, None, message, None, None,) + record.created = None + handler.filter(record) + result = json.loads(handler.format(record)) + result["message"] = expected_result + self.assertEqual(result["message"], expected_result) + + def test_format_with_request(self): + import logging + import json + + handler = self._make_one() + logname = "loggername" + message = "hello world，嗨 世界" + record = logging.LogRecord(logname, logging.INFO, "", 0, message, None, None) + expected_path = "http://testserver/123" + expected_agent = "Mozilla/5.0" + expected_trace = "123" + expected_span = "456" + trace_header = f"{expected_trace}/{expected_span};o=0" + expected_payload = { + "logging.googleapis.com/trace": expected_trace, + "logging.googleapis.com/spanId": expected_span, + "httpRequest": { + "requestMethod": "GET", + "requestUrl": expected_path, + "userAgent": expected_agent, + "protocol": "HTTP/1.1", + }, + } + + app = self.create_app() + with app.test_request_context( + expected_path, + headers={ + "User-Agent": expected_agent, + "X_CLOUD_TRACE_CONTEXT": trace_header, + }, + ): + handler.filter(record) + result = json.loads(handler.format(record)) + for (key, value) in expected_payload.items(): + self.assertEqual(value, result[key]) + + def test_format_overrides(self): + """ + Allow users to override log fields using `logging.info("", extra={})` + + If supported fields were overriden by the user, those choices should + take precedence. + """ + import logging + import json + + default_labels = { + "default_key": "default-value", + "overwritten_key": "bad_value", + } + handler = self._make_one(labels=default_labels) + logname = "loggername" + message = "hello world，嗨 世界" + record = logging.LogRecord(logname, logging.INFO, "", 0, message, None, None) + overwrite_path = "http://overwrite" + inferred_path = "http://testserver/123" + overwrite_trace = "abc" + overwrite_span = "def" + inferred_trace_span = "123/456;" + overwrite_file = "test-file" + record.http_request = {"requestUrl": overwrite_path} + record.source_location = {"file": overwrite_file} + record.trace = overwrite_trace + record.span_id = overwrite_span + added_labels = {"added_key": "added_value", "overwritten_key": "new_value"} + record.labels = added_labels + expected_payload = { + "logging.googleapis.com/trace": overwrite_trace, + "logging.googleapis.com/spanId": overwrite_span, + "logging.googleapis.com/sourceLocation": {"file": overwrite_file}, + "httpRequest": {"requestUrl": overwrite_path}, + "logging.googleapis.com/labels": { + "default_key": "default-value", + "overwritten_key": "new_value", + "added_key": "added_value", + }, + } + + app = self.create_app() + with app.test_client() as c: + c.put( + path=inferred_path, + data="body", + headers={"X_CLOUD_TRACE_CONTEXT": inferred_trace_span}, + ) + handler.filter(record) + result = json.loads(handler.format(record)) + for (key, value) in expected_payload.items(): + self.assertEqual(value, result[key]) diff --git a/tests/unit/test_client.py b/tests/unit/test_client.py index f33f1cbdc..9dbfa87fd 100644 --- a/tests/unit/test_client.py +++ b/tests/unit/test_client.py @@ -799,7 +799,13 @@ def test_setup_logging(self): handler.transport.worker.stop() expected_kwargs = { - "excluded_loggers": ("google.cloud", "google.auth", "google_auth_httplib2"), + "excluded_loggers": ( + "google.cloud", + "google.auth", + "google_auth_httplib2", + "google.api_core.bidi", + "werkzeug", + ), "log_level": 20, } self.assertEqual(kwargs, expected_kwargs) @@ -836,7 +842,13 @@ def test_setup_logging_w_extra_kwargs(self): handler.transport.worker.stop() expected_kwargs = { - "excluded_loggers": ("google.cloud", "google.auth", "google_auth_httplib2"), + "excluded_loggers": ( + "google.cloud", + "google.auth", + "google_auth_httplib2", + "google.api_core.bidi", + "werkzeug", + ), "log_level": 20, } self.assertEqual(kwargs, expected_kwargs)