From 49a387dbe24c0be33e034fd494106a7004bba369 Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Thu, 20 May 2021 15:48:56 -0700 Subject: [PATCH 01/26] refactored structured log handling of json input --- google/cloud/logging_v2/handlers/handlers.py | 11 +++++++++-- google/cloud/logging_v2/handlers/structured_log.py | 2 +- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/google/cloud/logging_v2/handlers/handlers.py b/google/cloud/logging_v2/handlers/handlers.py index 46922d54f..db50f60f4 100644 --- a/google/cloud/logging_v2/handlers/handlers.py +++ b/google/cloud/logging_v2/handlers/handlers.py @@ -14,6 +14,7 @@ """Python :mod:`logging` handlers for Cloud Logging.""" +import collections import json import logging @@ -99,8 +100,14 @@ def filter(self, record): record._http_request_str = json.dumps(record._http_request or {}) record._source_location_str = json.dumps(record._source_location or {}) record._labels_str = json.dumps(record._labels or {}) - # break quotes for parsing through structured logging - record._msg_str = str(record.msg).replace('"', '\\"') if record.msg else "" + # set string payload values + encoded_string = json.dumps(record.msg) + if isinstance(record.msg, collections.abc.Mapping) and len(encoded_string) > 2: + # if dict input, extract key/value pairs + record._payload_str = encoded_string[1:-1] + else: + # otherwise, the message is the string value of the input + record._payload_str = '"message": {}'.format(encoded_string) return True diff --git a/google/cloud/logging_v2/handlers/structured_log.py b/google/cloud/logging_v2/handlers/structured_log.py index 0edb5c39e..9936db63d 100644 --- a/google/cloud/logging_v2/handlers/structured_log.py +++ b/google/cloud/logging_v2/handlers/structured_log.py @@ -20,7 +20,7 @@ from google.cloud.logging_v2.handlers.handlers import CloudLoggingFilter GCP_FORMAT = ( - '{"message": "%(_msg_str)s", ' + '{%(_payload_str)s, ' '"severity": "%(levelname)s", ' '"logging.googleapis.com/labels": %(_labels_str)s, ' '"logging.googleapis.com/trace": "%(_trace_str)s", ' From c6168047a61e2a216b98cc736f7625476073aefb Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Thu, 20 May 2021 16:20:12 -0700 Subject: [PATCH 02/26] network transport supports dict messages --- .../logging_v2/handlers/transports/background_thread.py | 4 +++- google/cloud/logging_v2/handlers/transports/base.py | 2 +- google/cloud/logging_v2/handlers/transports/sync.py | 6 ++++-- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/google/cloud/logging_v2/handlers/transports/background_thread.py b/google/cloud/logging_v2/handlers/transports/background_thread.py index 3d654dbd8..418e4c573 100644 --- a/google/cloud/logging_v2/handlers/transports/background_thread.py +++ b/google/cloud/logging_v2/handlers/transports/background_thread.py @@ -20,6 +20,7 @@ from __future__ import print_function import atexit +import collections import datetime import logging import sys @@ -231,8 +232,9 @@ def enqueue(self, record, message, **kwargs): formatted by the associated log formatters. kwargs: Additional optional arguments for the logger """ + info = message if isinstance(message, collections.abc.Mapping) else {"message": message} queue_entry = { - "info": {"message": message, "python_logger": record.name}, + "info": info, "severity": _helpers._normalize_severity(record.levelno), "timestamp": datetime.datetime.utcfromtimestamp(record.created), } diff --git a/google/cloud/logging_v2/handlers/transports/base.py b/google/cloud/logging_v2/handlers/transports/base.py index d60a5a070..bd52b4e75 100644 --- a/google/cloud/logging_v2/handlers/transports/base.py +++ b/google/cloud/logging_v2/handlers/transports/base.py @@ -27,7 +27,7 @@ def send(self, record, message, **kwargs): Args: record (logging.LogRecord): Python log record that the handler was called with. - message (str): The message from the ``LogRecord`` after being + message (str or dict): The message from the ``LogRecord`` after being formatted by the associated log formatters. kwargs: Additional optional arguments for the logger """ diff --git a/google/cloud/logging_v2/handlers/transports/sync.py b/google/cloud/logging_v2/handlers/transports/sync.py index 35ee73daa..088e2a5ea 100644 --- a/google/cloud/logging_v2/handlers/transports/sync.py +++ b/google/cloud/logging_v2/handlers/transports/sync.py @@ -16,6 +16,7 @@ Logs directly to the the Cloud Logging API with a synchronous call. """ +import collections from google.cloud.logging_v2 import _helpers from google.cloud.logging_v2.handlers.transports.base import Transport @@ -36,11 +37,12 @@ def send(self, record, message, **kwargs): Args: record (logging.LogRecord): Python log record that the handler was called with. - message (str): The message from the ``LogRecord`` after being + message (str or dict): The message from the ``LogRecord`` after being formatted by the associated log formatters. kwargs: Additional optional arguments for the logger """ - info = {"message": message, "python_logger": record.name} + # wrap message in a dict if needed + info = message if isinstance(message, collections.abc.Mapping) else {"message": message} self.logger.log_struct( info, severity=_helpers._normalize_severity(record.levelno), **kwargs, ) From 46fe6a09c1c3def7cc9b9103a1429c723721369c Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Thu, 20 May 2021 16:35:02 -0700 Subject: [PATCH 03/26] allow non-ascii chars for StructuredLogHandler --- google/cloud/logging_v2/handlers/handlers.py | 8 ++++---- tests/unit/handlers/test_handlers.py | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/google/cloud/logging_v2/handlers/handlers.py b/google/cloud/logging_v2/handlers/handlers.py index db50f60f4..af3bad12b 100644 --- a/google/cloud/logging_v2/handlers/handlers.py +++ b/google/cloud/logging_v2/handlers/handlers.py @@ -97,11 +97,11 @@ def filter(self, record): # create string representations for structured logging record._trace_str = record._trace or "" record._span_id_str = record._span_id or "" - record._http_request_str = json.dumps(record._http_request or {}) - record._source_location_str = json.dumps(record._source_location or {}) - record._labels_str = json.dumps(record._labels or {}) + record._http_request_str = json.dumps(record._http_request or {}, ensure_ascii=False) + record._source_location_str = json.dumps(record._source_location or {}, ensure_ascii=False) + record._labels_str = json.dumps(record._labels or {}, ensure_ascii=False) # set string payload values - encoded_string = json.dumps(record.msg) + encoded_string = json.dumps(record.msg or "", ensure_ascii=False) if isinstance(record.msg, collections.abc.Mapping) and len(encoded_string) > 2: # if dict input, extract key/value pairs record._payload_str = encoded_string[1:-1] diff --git a/tests/unit/handlers/test_handlers.py b/tests/unit/handlers/test_handlers.py index 4ba052121..87af10d1a 100644 --- a/tests/unit/handlers/test_handlers.py +++ b/tests/unit/handlers/test_handlers.py @@ -78,7 +78,7 @@ def test_filter_record(self): self.assertTrue(success) self.assertEqual(record.msg, message) - self.assertEqual(record._msg_str, message) + self.assertEqual(record._payload_str, '"message": "{}"'.format(message)) self.assertEqual(record._source_location, expected_location) self.assertEqual(record._source_location_str, json.dumps(expected_location)) self.assertIsNone(record._resource) @@ -105,7 +105,7 @@ def test_minimal_record(self): self.assertTrue(success) self.assertIsNone(record.msg) - self.assertEqual(record._msg_str, "") + self.assertEqual(record._payload_str, '"message": ""') self.assertIsNone(record._source_location) self.assertEqual(record._source_location_str, "{}") self.assertIsNone(record._resource) From 471cb836b53c0406e17cb86ce7ceae7c5f990ed7 Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Thu, 20 May 2021 16:40:01 -0700 Subject: [PATCH 04/26] unit tests pass --- tests/unit/handlers/transports/test_background_thread.py | 4 ++-- tests/unit/handlers/transports/test_sync.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/unit/handlers/transports/test_background_thread.py b/tests/unit/handlers/transports/test_background_thread.py index 5410c5f10..9db7ebb1d 100644 --- a/tests/unit/handlers/transports/test_background_thread.py +++ b/tests/unit/handlers/transports/test_background_thread.py @@ -279,7 +279,7 @@ def test_enqueue_defaults(self): self._enqueue_record(worker, message) entry = worker._queue.get_nowait() - expected_info = {"message": message, "python_logger": "testing"} + expected_info = {"message": message} self.assertEqual(entry["info"], expected_info) self.assertEqual(entry["severity"], LogSeverity.INFO) self.assertIsInstance(entry["timestamp"], datetime.datetime) @@ -313,7 +313,7 @@ def test_enqueue_explicit(self): entry = worker._queue.get_nowait() - expected_info = {"message": message, "python_logger": "testing"} + expected_info = {"message": message} self.assertEqual(entry["info"], expected_info) self.assertEqual(entry["severity"], LogSeverity.ERROR) self.assertIs(entry["resource"], resource) diff --git a/tests/unit/handlers/transports/test_sync.py b/tests/unit/handlers/transports/test_sync.py index 9f0642757..e89aa7820 100644 --- a/tests/unit/handlers/transports/test_sync.py +++ b/tests/unit/handlers/transports/test_sync.py @@ -50,7 +50,7 @@ def test_send(self): ) transport.send(record, message, resource=_GLOBAL_RESOURCE) - EXPECTED_STRUCT = {"message": message, "python_logger": python_logger_name} + EXPECTED_STRUCT = {"message": message} EXPECTED_SENT = ( EXPECTED_STRUCT, LogSeverity.INFO, From 797282f81b84c535e2352d7e488f79897186aa46 Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Thu, 20 May 2021 16:49:00 -0700 Subject: [PATCH 05/26] system tests pass --- tests/system/test_system.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/system/test_system.py b/tests/system/test_system.py index cc6d03804..f458d49bb 100644 --- a/tests/system/test_system.py +++ b/tests/system/test_system.py @@ -291,7 +291,7 @@ def test_log_handler_async(self): cloud_logger.warn(LOG_MESSAGE) handler.flush() entries = _list_entries(logger) - expected_payload = {"message": LOG_MESSAGE, "python_logger": handler.name} + expected_payload = {"message": LOG_MESSAGE} self.assertEqual(len(entries), 1) self.assertEqual(entries[0].payload, expected_payload) @@ -313,7 +313,7 @@ def test_log_handler_sync(self): cloud_logger.warn(LOG_MESSAGE) entries = _list_entries(logger) - expected_payload = {"message": LOG_MESSAGE, "python_logger": LOGGER_NAME} + expected_payload = {"message": LOG_MESSAGE} self.assertEqual(len(entries), 1) self.assertEqual(entries[0].payload, expected_payload) @@ -366,7 +366,7 @@ def test_log_root_handler(self): logging.warn(LOG_MESSAGE) entries = _list_entries(logger) - expected_payload = {"message": LOG_MESSAGE, "python_logger": "root"} + expected_payload = {"message": LOG_MESSAGE} self.assertEqual(len(entries), 1) self.assertEqual(entries[0].payload, expected_payload) From cbc57ef7ab4d42ab7be184d0e5cd458017010a91 Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Thu, 20 May 2021 16:50:18 -0700 Subject: [PATCH 06/26] fixed lint issues --- google/cloud/logging_v2/handlers/handlers.py | 8 ++++++-- google/cloud/logging_v2/handlers/structured_log.py | 2 +- .../logging_v2/handlers/transports/background_thread.py | 6 +++++- google/cloud/logging_v2/handlers/transports/sync.py | 6 +++++- 4 files changed, 17 insertions(+), 5 deletions(-) diff --git a/google/cloud/logging_v2/handlers/handlers.py b/google/cloud/logging_v2/handlers/handlers.py index af3bad12b..42b1ac4b5 100644 --- a/google/cloud/logging_v2/handlers/handlers.py +++ b/google/cloud/logging_v2/handlers/handlers.py @@ -97,8 +97,12 @@ def filter(self, record): # create string representations for structured logging record._trace_str = record._trace or "" record._span_id_str = record._span_id or "" - record._http_request_str = json.dumps(record._http_request or {}, ensure_ascii=False) - record._source_location_str = json.dumps(record._source_location or {}, ensure_ascii=False) + record._http_request_str = json.dumps( + record._http_request or {}, ensure_ascii=False + ) + record._source_location_str = json.dumps( + record._source_location or {}, ensure_ascii=False + ) record._labels_str = json.dumps(record._labels or {}, ensure_ascii=False) # set string payload values encoded_string = json.dumps(record.msg or "", ensure_ascii=False) diff --git a/google/cloud/logging_v2/handlers/structured_log.py b/google/cloud/logging_v2/handlers/structured_log.py index 9936db63d..43ae00538 100644 --- a/google/cloud/logging_v2/handlers/structured_log.py +++ b/google/cloud/logging_v2/handlers/structured_log.py @@ -20,7 +20,7 @@ from google.cloud.logging_v2.handlers.handlers import CloudLoggingFilter GCP_FORMAT = ( - '{%(_payload_str)s, ' + "{%(_payload_str)s, " '"severity": "%(levelname)s", ' '"logging.googleapis.com/labels": %(_labels_str)s, ' '"logging.googleapis.com/trace": "%(_trace_str)s", ' diff --git a/google/cloud/logging_v2/handlers/transports/background_thread.py b/google/cloud/logging_v2/handlers/transports/background_thread.py index 418e4c573..4ac34dd0f 100644 --- a/google/cloud/logging_v2/handlers/transports/background_thread.py +++ b/google/cloud/logging_v2/handlers/transports/background_thread.py @@ -232,7 +232,11 @@ def enqueue(self, record, message, **kwargs): formatted by the associated log formatters. kwargs: Additional optional arguments for the logger """ - info = message if isinstance(message, collections.abc.Mapping) else {"message": message} + info = ( + message + if isinstance(message, collections.abc.Mapping) + else {"message": message} + ) queue_entry = { "info": info, "severity": _helpers._normalize_severity(record.levelno), diff --git a/google/cloud/logging_v2/handlers/transports/sync.py b/google/cloud/logging_v2/handlers/transports/sync.py index 088e2a5ea..22fb5f3b1 100644 --- a/google/cloud/logging_v2/handlers/transports/sync.py +++ b/google/cloud/logging_v2/handlers/transports/sync.py @@ -42,7 +42,11 @@ def send(self, record, message, **kwargs): kwargs: Additional optional arguments for the logger """ # wrap message in a dict if needed - info = message if isinstance(message, collections.abc.Mapping) else {"message": message} + info = ( + message + if isinstance(message, collections.abc.Mapping) + else {"message": message} + ) self.logger.log_struct( info, severity=_helpers._normalize_severity(record.levelno), **kwargs, ) From 204cfff78f2a939ffa4fba7dbb99d571ffa69819 Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Thu, 20 May 2021 18:24:46 -0700 Subject: [PATCH 07/26] added structured logging environment tests --- tests/environment | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/environment b/tests/environment index 834e5f4eb..5978d5f17 160000 --- a/tests/environment +++ b/tests/environment @@ -1 +1 @@ -Subproject commit 834e5f4ebd76e2dbabb9237a54f9184964dcda1a +Subproject commit 5978d5f17b518293951a912aadccde0c37a1d4cb From 16b7b9e969e4365c67053477a2c71cec4f8f9f04 Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Fri, 21 May 2021 11:54:33 -0700 Subject: [PATCH 08/26] added generic log function to logger --- .../handlers/transports/background_thread.py | 13 ++---- .../logging_v2/handlers/transports/sync.py | 17 ++++--- google/cloud/logging_v2/logger.py | 45 +++++++++++++++++++ 3 files changed, 57 insertions(+), 18 deletions(-) diff --git a/google/cloud/logging_v2/handlers/transports/background_thread.py b/google/cloud/logging_v2/handlers/transports/background_thread.py index 4ac34dd0f..cb2998927 100644 --- a/google/cloud/logging_v2/handlers/transports/background_thread.py +++ b/google/cloud/logging_v2/handlers/transports/background_thread.py @@ -139,7 +139,7 @@ def _thread_main(self): if item is _WORKER_TERMINATOR: done = True # Continue processing items. else: - batch.log_struct(**item) + batch.log(**item) self._safely_commit_batch(batch) @@ -228,17 +228,12 @@ def enqueue(self, record, message, **kwargs): Args: record (logging.LogRecord): Python log record that the handler was called with. - message (str): The message from the ``LogRecord`` after being + message (str or dict): The message from the ``LogRecord`` after being formatted by the associated log formatters. kwargs: Additional optional arguments for the logger """ - info = ( - message - if isinstance(message, collections.abc.Mapping) - else {"message": message} - ) queue_entry = { - "info": info, + "message": message, "severity": _helpers._normalize_severity(record.levelno), "timestamp": datetime.datetime.utcfromtimestamp(record.created), } @@ -292,7 +287,7 @@ def send(self, record, message, **kwargs): Args: record (logging.LogRecord): Python log record that the handler was called with. - message (str): The message from the ``LogRecord`` after being + message (str or dict): The message from the ``LogRecord`` after being formatted by the associated log formatters. kwargs: Additional optional arguments for the logger """ diff --git a/google/cloud/logging_v2/handlers/transports/sync.py b/google/cloud/logging_v2/handlers/transports/sync.py index 22fb5f3b1..c4c0018f4 100644 --- a/google/cloud/logging_v2/handlers/transports/sync.py +++ b/google/cloud/logging_v2/handlers/transports/sync.py @@ -41,12 +41,11 @@ def send(self, record, message, **kwargs): formatted by the associated log formatters. kwargs: Additional optional arguments for the logger """ - # wrap message in a dict if needed - info = ( - message - if isinstance(message, collections.abc.Mapping) - else {"message": message} - ) - self.logger.log_struct( - info, severity=_helpers._normalize_severity(record.levelno), **kwargs, - ) + if isinstance(message, collections.abc.Mapping): + self.logger.log_struct( + message, severity=_helpers._normalize_severity(record.levelno), **kwargs, + ) + else: + self.logger.log_text( + str(message), severity=_helpers._normalize_severity(record.levelno), **kwargs, + ) diff --git a/google/cloud/logging_v2/logger.py b/google/cloud/logging_v2/logger.py index fafb70629..cdc88df28 100644 --- a/google/cloud/logging_v2/logger.py +++ b/google/cloud/logging_v2/logger.py @@ -14,6 +14,8 @@ """Define API Loggers.""" +import collections + from google.cloud.logging_v2._helpers import _add_defaults_to_filter from google.cloud.logging_v2.entries import LogEntry from google.cloud.logging_v2.entries import ProtobufEntry @@ -21,6 +23,7 @@ from google.cloud.logging_v2.entries import TextEntry from google.cloud.logging_v2.resource import Resource +import google.protobuf.message _GLOBAL_RESOURCE = Resource(type="global", labels={}) @@ -197,6 +200,30 @@ def log_proto(self, message, *, client=None, **kw): """ self._do_log(client, ProtobufEntry, message, **kw) + def log(self, message, **kw): + """Log an arbitrary message via a POST request. + Type will be inferred based on the input message. + + See + https://cloud.google.com/logging/docs/reference/v2/rest/v2/entries/list + + Args: + message (Optional[str or dict or google.protobuf.Message]): The message. to log + client (Optional[~logging_v2.client.Client]): + The client to use. If not passed, falls back to the + ``client`` stored on the current sink. + kw (Optional[dict]): additional keyword arguments for the entry. + See :class:`~logging_v2.entries.LogEntry`. + """ + entry_type = LogEntry + if isinstance(message, google.protobuf.message.Message): + entry_type = ProtobufEntry + elif isinstance(message, collections.abc.Mapping): + entry_type = StructEntry + elif isinstance(message, str): + entry_type = TextEntry + self._do_log(client, entry_type, message, **kw) + def delete(self, logger_name=None, *, client=None): """Delete all entries in a logger via a DELETE request @@ -361,6 +388,24 @@ def log_proto(self, message, **kw): """ self.entries.append(ProtobufEntry(payload=message, **kw)) + def log(self, message, **kw): + """Add an arbitrary message to be logged during :meth:`commit`. + Type will be inferred based on the input message. + + Args: + message (Optional[str or dict or google.protobuf.Message]): The message. to log + kw (Optional[dict]): Additional keyword arguments for the entry. + See :class:`~logging_v2.entries.LogEntry`. + """ + entry_type = LogEntry + if isinstance(message, google.protobuf.message.Message): + entry_type = ProtobufEntry + elif isinstance(message, collections.abc.Mapping): + entry_type = StructEntry + elif isinstance(message, str): + entry_type = TextEntry + self.entries.append(entry_type(payload=message, **kw)) + def commit(self, *, client=None): """Send saved log entries as a single API call. From 4b563849d591754cf0e46fc4047ab829c395c2cf Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Fri, 21 May 2021 12:06:47 -0700 Subject: [PATCH 09/26] fixed unit tests --- .../logging_v2/handlers/transports/sync.py | 12 ++++-------- .../transports/test_background_thread.py | 18 ++++++++---------- tests/unit/handlers/transports/test_sync.py | 9 ++++----- 3 files changed, 16 insertions(+), 23 deletions(-) diff --git a/google/cloud/logging_v2/handlers/transports/sync.py b/google/cloud/logging_v2/handlers/transports/sync.py index c4c0018f4..2ec8a7153 100644 --- a/google/cloud/logging_v2/handlers/transports/sync.py +++ b/google/cloud/logging_v2/handlers/transports/sync.py @@ -41,11 +41,7 @@ def send(self, record, message, **kwargs): formatted by the associated log formatters. kwargs: Additional optional arguments for the logger """ - if isinstance(message, collections.abc.Mapping): - self.logger.log_struct( - message, severity=_helpers._normalize_severity(record.levelno), **kwargs, - ) - else: - self.logger.log_text( - str(message), severity=_helpers._normalize_severity(record.levelno), **kwargs, - ) + + self.logger.log( + message, severity=_helpers._normalize_severity(record.levelno), **kwargs, + ) diff --git a/tests/unit/handlers/transports/test_background_thread.py b/tests/unit/handlers/transports/test_background_thread.py index 9db7ebb1d..155ca3fb0 100644 --- a/tests/unit/handlers/transports/test_background_thread.py +++ b/tests/unit/handlers/transports/test_background_thread.py @@ -279,8 +279,7 @@ def test_enqueue_defaults(self): self._enqueue_record(worker, message) entry = worker._queue.get_nowait() - expected_info = {"message": message} - self.assertEqual(entry["info"], expected_info) + self.assertEqual(entry["message"], message) self.assertEqual(entry["severity"], LogSeverity.INFO) self.assertIsInstance(entry["timestamp"], datetime.datetime) self.assertNotIn("resource", entry.keys()) @@ -313,8 +312,7 @@ def test_enqueue_explicit(self): entry = worker._queue.get_nowait() - expected_info = {"message": message} - self.assertEqual(entry["info"], expected_info) + self.assertEqual(entry["message"], message) self.assertEqual(entry["severity"], LogSeverity.ERROR) self.assertIs(entry["resource"], resource) self.assertIs(entry["labels"], labels) @@ -389,9 +387,9 @@ def test__thread_main_max_latency(self, time): worker._queue = mock.create_autospec(queue.Queue, instance=True) worker._queue.get.side_effect = [ - {"info": {"message": "1"}}, # Single record. + {"message": 1}, # Single record. queue.Empty(), # Emulate a queue.get() timeout. - {"info": {"message": "1"}}, # Second record. + {"message": "2"}, # Second record. background_thread._WORKER_TERMINATOR, # Stop the thread. queue.Empty(), # Emulate a queue.get() timeout. ] @@ -480,9 +478,9 @@ def __init__(self): self.commit_called = False self.commit_count = None - def log_struct( + def log( self, - info, + message, severity=logging.INFO, resource=None, labels=None, @@ -496,8 +494,8 @@ def log_struct( assert resource is None resource = _GLOBAL_RESOURCE - self.log_struct_called_with = (info, severity, resource, labels, trace, span_id) - self.entries.append(info) + self.log_called_with = (message, severity, resource, labels, trace, span_id) + self.entries.append(message) def commit(self): self.commit_called = True diff --git a/tests/unit/handlers/transports/test_sync.py b/tests/unit/handlers/transports/test_sync.py index e89aa7820..62d37669b 100644 --- a/tests/unit/handlers/transports/test_sync.py +++ b/tests/unit/handlers/transports/test_sync.py @@ -50,9 +50,8 @@ def test_send(self): ) transport.send(record, message, resource=_GLOBAL_RESOURCE) - EXPECTED_STRUCT = {"message": message} EXPECTED_SENT = ( - EXPECTED_STRUCT, + message, LogSeverity.INFO, _GLOBAL_RESOURCE, None, @@ -60,7 +59,7 @@ def test_send(self): None, None, ) - self.assertEqual(transport.logger.log_struct_called_with, EXPECTED_SENT) + self.assertEqual(transport.logger.log_called_with, EXPECTED_SENT) class _Logger(object): @@ -69,7 +68,7 @@ class _Logger(object): def __init__(self, name): self.name = name - def log_struct( + def log( self, message, severity=None, @@ -79,7 +78,7 @@ def log_struct( span_id=None, http_request=None, ): - self.log_struct_called_with = ( + self.log_called_with = ( message, severity, resource, From 090177de4c69a032620af3e5d2ffaaa12a28cad2 Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Fri, 21 May 2021 12:31:28 -0700 Subject: [PATCH 10/26] added more unit tests --- google/cloud/logging_v2/logger.py | 4 +- tests/unit/handlers/transports/test_sync.py | 25 +++ tests/unit/test_logger.py | 193 ++++++++++++++++++++ 3 files changed, 220 insertions(+), 2 deletions(-) diff --git a/google/cloud/logging_v2/logger.py b/google/cloud/logging_v2/logger.py index cdc88df28..ffe7ea706 100644 --- a/google/cloud/logging_v2/logger.py +++ b/google/cloud/logging_v2/logger.py @@ -200,7 +200,7 @@ def log_proto(self, message, *, client=None, **kw): """ self._do_log(client, ProtobufEntry, message, **kw) - def log(self, message, **kw): + def log(self, message=None, *, client=None, **kw): """Log an arbitrary message via a POST request. Type will be inferred based on the input message. @@ -388,7 +388,7 @@ def log_proto(self, message, **kw): """ self.entries.append(ProtobufEntry(payload=message, **kw)) - def log(self, message, **kw): + def log(self, message=None, **kw): """Add an arbitrary message to be logged during :meth:`commit`. Type will be inferred based on the input message. diff --git a/tests/unit/handlers/transports/test_sync.py b/tests/unit/handlers/transports/test_sync.py index 62d37669b..7af302691 100644 --- a/tests/unit/handlers/transports/test_sync.py +++ b/tests/unit/handlers/transports/test_sync.py @@ -61,6 +61,31 @@ def test_send(self): ) self.assertEqual(transport.logger.log_called_with, EXPECTED_SENT) + def test_send_struct(self): + from google.cloud.logging_v2.logger import _GLOBAL_RESOURCE + from google.cloud.logging_v2._helpers import LogSeverity + + client = _Client(self.PROJECT) + + stackdriver_logger_name = "python" + python_logger_name = "mylogger" + transport = self._make_one(client, stackdriver_logger_name) + message = {"message":"hello world", "extra":"test"} + record = logging.LogRecord( + python_logger_name, logging.INFO, None, None, message, None, None + ) + + transport.send(record, message, resource=_GLOBAL_RESOURCE) + EXPECTED_SENT = ( + message, + LogSeverity.INFO, + _GLOBAL_RESOURCE, + None, + None, + None, + None, + ) + self.assertEqual(transport.logger.log_called_with, EXPECTED_SENT) class _Logger(object): from google.cloud.logging_v2.logger import _GLOBAL_RESOURCE diff --git a/tests/unit/test_logger.py b/tests/unit/test_logger.py index 5ad486178..4103ae0d3 100644 --- a/tests/unit/test_logger.py +++ b/tests/unit/test_logger.py @@ -464,6 +464,82 @@ def test_log_proto_w_explicit(self): self.assertEqual(api._write_entries_called_with, (ENTRIES, None, None, None)) + + def test_log_inference_empty(self): + DEFAULT_LABELS = {"foo": "spam"} + ENTRIES = [ + { + "logName": "projects/%s/logs/%s" % (self.PROJECT, self.LOGGER_NAME), + "resource": {"type": "global", "labels": {}}, + "labels": DEFAULT_LABELS, + } + ] + client = _Client(self.PROJECT) + api = client.logging_api = _DummyLoggingAPI() + logger = self._make_one(self.LOGGER_NAME, client=client, labels=DEFAULT_LABELS) + + logger.log() + + self.assertEqual(api._write_entries_called_with, (ENTRIES, None, None, None)) + + def test_log_inference_text(self): + RESOURCE = {"type": "global", "labels": {}} + TEXT = "TEXT" + ENTRIES = [ + { + "logName": "projects/%s/logs/%s" % (self.PROJECT, self.LOGGER_NAME), + "textPayload": TEXT, + "resource": RESOURCE, + } + ] + client = _Client(self.PROJECT) + api = client.logging_api = _DummyLoggingAPI() + logger = self._make_one(self.LOGGER_NAME, client=client) + + logger.log(TEXT) + + self.assertEqual(api._write_entries_called_with, (ENTRIES, None, None, None)) + + + def test_log_inference_struct(self): + STRUCT = {"message": "MESSAGE", "weather": "cloudy"} + RESOURCE = {"type": "global", "labels": {}} + ENTRIES = [ + { + "logName": "projects/%s/logs/%s" % (self.PROJECT, self.LOGGER_NAME), + "jsonPayload": STRUCT, + "resource": RESOURCE, + } + ] + client = _Client(self.PROJECT) + api = client.logging_api = _DummyLoggingAPI() + logger = self._make_one(self.LOGGER_NAME, client=client) + + logger.log(STRUCT) + + self.assertEqual(api._write_entries_called_with, (ENTRIES, None, None, None)) + + def test_log_inference_proto(self): + import json + from google.protobuf.json_format import MessageToJson + from google.protobuf.struct_pb2 import Struct, Value + + message = Struct(fields={"foo": Value(bool_value=True)}) + ENTRIES = [ + { + "logName": "projects/%s/logs/%s" % (self.PROJECT, self.LOGGER_NAME), + "protoPayload": json.loads(MessageToJson(message)), + "resource": {"type": "global", "labels": {}}, + } + ] + client = _Client(self.PROJECT) + api = client.logging_api = _DummyLoggingAPI() + logger = self._make_one(self.LOGGER_NAME, client=client) + + logger.log(message) + + self.assertEqual(api._write_entries_called_with, (ENTRIES, None, None, None)) + def test_delete_w_bound_client(self): client = _Client(project=self.PROJECT) api = client.logging_api = _DummyLoggingAPI() @@ -902,6 +978,123 @@ def test_log_proto_explicit(self): ) self.assertEqual(batch.entries, [ENTRY]) + def test_log_inference_empty(self): + """ + When calling batch.log with empty input, it should + call batch.log_empty + """ + from google.cloud.logging import LogEntry + + ENTRY = LogEntry() + client = _Client(project=self.PROJECT, connection=_make_credentials()) + logger = _Logger() + batch = self._make_one(logger, client=client) + batch.log() + self.assertEqual(batch.entries, [ENTRY]) + + def test_log_inference_text(self): + """ + When calling batch.log with text input, it should + call batch.log_text + """ + from google.cloud.logging_v2.entries import _GLOBAL_RESOURCE + from google.cloud.logging import TextEntry + + TEXT = "This is the entry text" + ENTRY = TextEntry(payload=TEXT, resource=_GLOBAL_RESOURCE) + client = _Client(project=self.PROJECT, connection=_make_credentials()) + logger = _Logger() + batch = self._make_one(logger, client=client) + batch.log(TEXT) + self.assertEqual(batch.entries, [ENTRY]) + + def test_log_inference_struct(self): + """ + When calling batch.struct with text input, it should + call batch.log_struct + """ + from google.cloud.logging_v2.entries import _GLOBAL_RESOURCE + from google.cloud.logging import StructEntry + + STRUCT = {"message": "Message text", "weather": "partly cloudy"} + ENTRY = StructEntry(payload=STRUCT, resource=_GLOBAL_RESOURCE) + client = _Client(project=self.PROJECT, connection=_make_credentials()) + logger = _Logger() + batch = self._make_one(logger, client=client) + batch.log(STRUCT) + self.assertEqual(batch.entries, [ENTRY]) + + def test_log_inference_proto(self): + """ + When calling batch.log with proto input, it should + call batch.log_proto + """ + from google.cloud.logging_v2.entries import _GLOBAL_RESOURCE + from google.cloud.logging import ProtobufEntry + from google.protobuf.struct_pb2 import Struct + from google.protobuf.struct_pb2 import Value + + message = Struct(fields={"foo": Value(bool_value=True)}) + ENTRY = ProtobufEntry(payload=message, resource=_GLOBAL_RESOURCE) + client = _Client(project=self.PROJECT, connection=_make_credentials()) + logger = _Logger() + batch = self._make_one(logger, client=client) + batch.log(message) + self.assertEqual(batch.entries, [ENTRY]) + + def test_log_inference_struct_explicit(self): + """ + When calling batch.log with struct input, it should + call batch.log_struct, along with input arguments + """ + import datetime + from google.cloud.logging import Resource + from google.cloud.logging import StructEntry + + STRUCT = {"message": "Message text", "weather": "partly cloudy"} + LABELS = {"foo": "bar", "baz": "qux"} + IID = "IID" + SEVERITY = "CRITICAL" + METHOD = "POST" + URI = "https://api.example.com/endpoint" + STATUS = "500" + TRACE = "12345678-1234-5678-1234-567812345678" + SPANID = "000000000000004a" + REQUEST = {"requestMethod": METHOD, "requestUrl": URI, "status": STATUS} + TIMESTAMP = datetime.datetime(2016, 12, 31, 0, 1, 2, 999999) + RESOURCE = Resource( + type="gae_app", labels={"module_id": "default", "version_id": "test"} + ) + ENTRY = StructEntry( + payload=STRUCT, + labels=LABELS, + insert_id=IID, + severity=SEVERITY, + http_request=REQUEST, + timestamp=TIMESTAMP, + resource=RESOURCE, + trace=TRACE, + span_id=SPANID, + trace_sampled=True, + ) + + client = _Client(project=self.PROJECT, connection=_make_credentials()) + logger = _Logger() + batch = self._make_one(logger, client=client) + batch.log( + STRUCT, + labels=LABELS, + insert_id=IID, + severity=SEVERITY, + http_request=REQUEST, + timestamp=TIMESTAMP, + resource=RESOURCE, + trace=TRACE, + span_id=SPANID, + trace_sampled=True, + ) + self.assertEqual(batch.entries, [ENTRY]) + def test_commit_w_unknown_entry_type(self): from google.cloud.logging_v2.entries import _GLOBAL_RESOURCE from google.cloud.logging import LogEntry From 7a5c55c2717c9e15fb4468717dff2394923a4839 Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Fri, 21 May 2021 13:02:26 -0700 Subject: [PATCH 11/26] added system tests --- tests/system/test_system.py | 37 ++++++++++++++++++++++++++++++++++--- 1 file changed, 34 insertions(+), 3 deletions(-) diff --git a/tests/system/test_system.py b/tests/system/test_system.py index f458d49bb..dc0fa195d 100644 --- a/tests/system/test_system.py +++ b/tests/system/test_system.py @@ -35,6 +35,8 @@ from google.cloud.logging_v2.handlers.transports import SyncTransport from google.cloud.logging_v2 import client from google.cloud.logging_v2.resource import Resource +from google.protobuf.json_format import MessageToJson +from google.protobuf.struct_pb2 import Struct, Value from test_utils.retry import RetryErrors from test_utils.retry import RetryResult @@ -277,6 +279,35 @@ def test_log_struct_w_metadata(self): self.assertEqual(request["requestUrl"], URI) self.assertEqual(request["status"], STATUS) + def test_log_w_text(self): + TEXT_PAYLOAD = "System test: test_log_w_text" + logger = Config.CLIENT.logger(self._logger_name("log_w_text")) + self.to_delete.append(logger) + logger.log(TEXT_PAYLOAD) + entries = _list_entries(logger) + self.assertEqual(len(entries), 1) + self.assertEqual(entries[0].payload, TEXT_PAYLOAD) + + def test_log_w_struct(self): + logger = Config.CLIENT.logger(self._logger_name("log_w_struct")) + self.to_delete.append(logger) + + logger.log(self.JSON_PAYLOAD) + entries = _list_entries(logger) + + self.assertEqual(len(entries), 1) + self.assertEqual(entries[0].payload, self.JSON_PAYLOAD) + + def test_log_empty(self): + logger = Config.CLIENT.logger(self._logger_name("log_empty")) + self.to_delete.append(logger) + + logger.log() + entries = _list_entries(logger) + + self.assertEqual(len(entries), 1) + self.assertIsNone(entries[0].payload) + def test_log_handler_async(self): LOG_MESSAGE = "It was the worst of times" @@ -291,7 +322,7 @@ def test_log_handler_async(self): cloud_logger.warn(LOG_MESSAGE) handler.flush() entries = _list_entries(logger) - expected_payload = {"message": LOG_MESSAGE} + expected_payload = LOG_MESSAGE self.assertEqual(len(entries), 1) self.assertEqual(entries[0].payload, expected_payload) @@ -313,7 +344,7 @@ def test_log_handler_sync(self): cloud_logger.warn(LOG_MESSAGE) entries = _list_entries(logger) - expected_payload = {"message": LOG_MESSAGE} + expected_payload = LOG_MESSAGE self.assertEqual(len(entries), 1) self.assertEqual(entries[0].payload, expected_payload) @@ -366,7 +397,7 @@ def test_log_root_handler(self): logging.warn(LOG_MESSAGE) entries = _list_entries(logger) - expected_payload = {"message": LOG_MESSAGE} + expected_payload = LOG_MESSAGE self.assertEqual(len(entries), 1) self.assertEqual(entries[0].payload, expected_payload) From a9db60fa334c46a52f00b10d3cd03ef2fc9045df Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Fri, 21 May 2021 13:03:07 -0700 Subject: [PATCH 12/26] fixed lint issues --- tests/unit/handlers/transports/test_sync.py | 3 ++- tests/unit/test_logger.py | 2 -- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/unit/handlers/transports/test_sync.py b/tests/unit/handlers/transports/test_sync.py index 7af302691..fd5f00c69 100644 --- a/tests/unit/handlers/transports/test_sync.py +++ b/tests/unit/handlers/transports/test_sync.py @@ -70,7 +70,7 @@ def test_send_struct(self): stackdriver_logger_name = "python" python_logger_name = "mylogger" transport = self._make_one(client, stackdriver_logger_name) - message = {"message":"hello world", "extra":"test"} + message = {"message": "hello world", "extra": "test"} record = logging.LogRecord( python_logger_name, logging.INFO, None, None, message, None, None ) @@ -87,6 +87,7 @@ def test_send_struct(self): ) self.assertEqual(transport.logger.log_called_with, EXPECTED_SENT) + class _Logger(object): from google.cloud.logging_v2.logger import _GLOBAL_RESOURCE diff --git a/tests/unit/test_logger.py b/tests/unit/test_logger.py index 4103ae0d3..d0e751e93 100644 --- a/tests/unit/test_logger.py +++ b/tests/unit/test_logger.py @@ -464,7 +464,6 @@ def test_log_proto_w_explicit(self): self.assertEqual(api._write_entries_called_with, (ENTRIES, None, None, None)) - def test_log_inference_empty(self): DEFAULT_LABELS = {"foo": "spam"} ENTRIES = [ @@ -500,7 +499,6 @@ def test_log_inference_text(self): self.assertEqual(api._write_entries_called_with, (ENTRIES, None, None, None)) - def test_log_inference_struct(self): STRUCT = {"message": "MESSAGE", "weather": "cloudy"} RESOURCE = {"type": "global", "labels": {}} From 1c2ccb87e2c52088f7a28deaff510058926fdbf4 Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Fri, 21 May 2021 13:37:47 -0700 Subject: [PATCH 13/26] fixed breaking test --- tests/environment | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/environment b/tests/environment index 5978d5f17..2125b0b23 160000 --- a/tests/environment +++ b/tests/environment @@ -1 +1 @@ -Subproject commit 5978d5f17b518293951a912aadccde0c37a1d4cb +Subproject commit 2125b0b23c646e58f9ed8d379842e24043d3762b From 797c95eb73708c94c62b90591f2abe94b0903ae7 Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Fri, 21 May 2021 13:59:08 -0700 Subject: [PATCH 14/26] fixed json log for pylogging --- google/cloud/logging_v2/handlers/handlers.py | 9 +++++++-- tests/environment | 2 +- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/google/cloud/logging_v2/handlers/handlers.py b/google/cloud/logging_v2/handlers/handlers.py index 42b1ac4b5..df3c90291 100644 --- a/google/cloud/logging_v2/handlers/handlers.py +++ b/google/cloud/logging_v2/handlers/handlers.py @@ -194,9 +194,14 @@ def emit(self, record): Args: record (logging.LogRecord): The record to be logged. """ - message = super(CloudLoggingHandler, self).format(record) - labels = record._labels resource = record._resource or self.resource + labels = record._labels + if isinstance(record.msg, collections.abc.Mapping): + # if input is a dictionary, pass as-is for structured logging + message = record.msg + else: + # otherwise, format message string based on superclass + message = super(CloudLoggingHandler, self).format(record) if resource.type == _GAE_RESOURCE_TYPE and record._trace is not None: # add GAE-specific label labels = {_GAE_TRACE_ID_LABEL: record._trace, **(labels or {})} diff --git a/tests/environment b/tests/environment index 2125b0b23..966e37215 160000 --- a/tests/environment +++ b/tests/environment @@ -1 +1 @@ -Subproject commit 2125b0b23c646e58f9ed8d379842e24043d3762b +Subproject commit 966e372159321acba04c51ef6b17d5eef22db995 From a0f13096efc97aa0f63f4435f712a31574c6cbd4 Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Fri, 21 May 2021 14:18:50 -0700 Subject: [PATCH 15/26] fixed lint issue --- .../cloud/logging_v2/handlers/transports/background_thread.py | 1 - google/cloud/logging_v2/handlers/transports/sync.py | 2 -- tests/system/test_system.py | 2 -- 3 files changed, 5 deletions(-) diff --git a/google/cloud/logging_v2/handlers/transports/background_thread.py b/google/cloud/logging_v2/handlers/transports/background_thread.py index cb2998927..acdfcc2b4 100644 --- a/google/cloud/logging_v2/handlers/transports/background_thread.py +++ b/google/cloud/logging_v2/handlers/transports/background_thread.py @@ -20,7 +20,6 @@ from __future__ import print_function import atexit -import collections import datetime import logging import sys diff --git a/google/cloud/logging_v2/handlers/transports/sync.py b/google/cloud/logging_v2/handlers/transports/sync.py index 2ec8a7153..6c0748be2 100644 --- a/google/cloud/logging_v2/handlers/transports/sync.py +++ b/google/cloud/logging_v2/handlers/transports/sync.py @@ -16,8 +16,6 @@ Logs directly to the the Cloud Logging API with a synchronous call. """ -import collections - from google.cloud.logging_v2 import _helpers from google.cloud.logging_v2.handlers.transports.base import Transport diff --git a/tests/system/test_system.py b/tests/system/test_system.py index dc0fa195d..f76ac2bed 100644 --- a/tests/system/test_system.py +++ b/tests/system/test_system.py @@ -35,8 +35,6 @@ from google.cloud.logging_v2.handlers.transports import SyncTransport from google.cloud.logging_v2 import client from google.cloud.logging_v2.resource import Resource -from google.protobuf.json_format import MessageToJson -from google.protobuf.struct_pb2 import Struct, Value from test_utils.retry import RetryErrors from test_utils.retry import RetryResult From 01cd6393b5546224ae6556d63bed20f18969f880 Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Fri, 21 May 2021 14:32:12 -0700 Subject: [PATCH 16/26] add python logger as label --- google/cloud/logging_v2/handlers/handlers.py | 2 +- .../logging_v2/handlers/transports/background_thread.py | 5 +++++ google/cloud/logging_v2/handlers/transports/sync.py | 7 +++++-- 3 files changed, 11 insertions(+), 3 deletions(-) diff --git a/google/cloud/logging_v2/handlers/handlers.py b/google/cloud/logging_v2/handlers/handlers.py index df3c90291..15e671531 100644 --- a/google/cloud/logging_v2/handlers/handlers.py +++ b/google/cloud/logging_v2/handlers/handlers.py @@ -93,7 +93,7 @@ def filter(self, record): record._span_id = getattr(record, "span_id", inferred_span) or None record._http_request = getattr(record, "http_request", inferred_http) record._source_location = CloudLoggingFilter._infer_source_location(record) - record._labels = {**self.default_labels, **user_labels} or None + record._labels = {"python_logger":record.name, **self.default_labels, **user_labels} # create string representations for structured logging record._trace_str = record._trace or "" record._span_id_str = record._span_id or "" diff --git a/google/cloud/logging_v2/handlers/transports/background_thread.py b/google/cloud/logging_v2/handlers/transports/background_thread.py index acdfcc2b4..e94dca2bc 100644 --- a/google/cloud/logging_v2/handlers/transports/background_thread.py +++ b/google/cloud/logging_v2/handlers/transports/background_thread.py @@ -231,6 +231,11 @@ def enqueue(self, record, message, **kwargs): formatted by the associated log formatters. kwargs: Additional optional arguments for the logger """ + # set python logger name as label if missing + labels = kwargs.pop("labels", {}) + labels['python_logger'] = labels.get('python_logger', record.name) + kwargs['labels'] = labels + # enqueue new entry queue_entry = { "message": message, "severity": _helpers._normalize_severity(record.levelno), diff --git a/google/cloud/logging_v2/handlers/transports/sync.py b/google/cloud/logging_v2/handlers/transports/sync.py index 6c0748be2..c70c4b1a0 100644 --- a/google/cloud/logging_v2/handlers/transports/sync.py +++ b/google/cloud/logging_v2/handlers/transports/sync.py @@ -39,7 +39,10 @@ def send(self, record, message, **kwargs): formatted by the associated log formatters. kwargs: Additional optional arguments for the logger """ - + # set python logger name as label if missing + labels = kwargs.pop("labels", {}) + labels['python_logger'] = labels.get('python_logger', record.name) + # send log synchronously self.logger.log( - message, severity=_helpers._normalize_severity(record.levelno), **kwargs, + message, severity=_helpers._normalize_severity(record.levelno), labels=labels, **kwargs, ) From 6ff838a5406202154bde425beb79c43431e73646 Mon Sep 17 00:00:00 2001 From: Owl Bot Date: Fri, 21 May 2021 21:33:15 +0000 Subject: [PATCH 17/26] =?UTF-8?q?=F0=9F=A6=89=20Updates=20from=20OwlBot?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- google/cloud/logging_v2/handlers/handlers.py | 6 +++++- .../logging_v2/handlers/transports/background_thread.py | 4 ++-- google/cloud/logging_v2/handlers/transports/sync.py | 7 +++++-- 3 files changed, 12 insertions(+), 5 deletions(-) diff --git a/google/cloud/logging_v2/handlers/handlers.py b/google/cloud/logging_v2/handlers/handlers.py index 15e671531..2298d4455 100644 --- a/google/cloud/logging_v2/handlers/handlers.py +++ b/google/cloud/logging_v2/handlers/handlers.py @@ -93,7 +93,11 @@ def filter(self, record): record._span_id = getattr(record, "span_id", inferred_span) or None record._http_request = getattr(record, "http_request", inferred_http) record._source_location = CloudLoggingFilter._infer_source_location(record) - record._labels = {"python_logger":record.name, **self.default_labels, **user_labels} + record._labels = { + "python_logger": record.name, + **self.default_labels, + **user_labels, + } # create string representations for structured logging record._trace_str = record._trace or "" record._span_id_str = record._span_id or "" diff --git a/google/cloud/logging_v2/handlers/transports/background_thread.py b/google/cloud/logging_v2/handlers/transports/background_thread.py index e94dca2bc..e02186d2b 100644 --- a/google/cloud/logging_v2/handlers/transports/background_thread.py +++ b/google/cloud/logging_v2/handlers/transports/background_thread.py @@ -233,8 +233,8 @@ def enqueue(self, record, message, **kwargs): """ # set python logger name as label if missing labels = kwargs.pop("labels", {}) - labels['python_logger'] = labels.get('python_logger', record.name) - kwargs['labels'] = labels + labels["python_logger"] = labels.get("python_logger", record.name) + kwargs["labels"] = labels # enqueue new entry queue_entry = { "message": message, diff --git a/google/cloud/logging_v2/handlers/transports/sync.py b/google/cloud/logging_v2/handlers/transports/sync.py index c70c4b1a0..46fd57254 100644 --- a/google/cloud/logging_v2/handlers/transports/sync.py +++ b/google/cloud/logging_v2/handlers/transports/sync.py @@ -41,8 +41,11 @@ def send(self, record, message, **kwargs): """ # set python logger name as label if missing labels = kwargs.pop("labels", {}) - labels['python_logger'] = labels.get('python_logger', record.name) + labels["python_logger"] = labels.get("python_logger", record.name) # send log synchronously self.logger.log( - message, severity=_helpers._normalize_severity(record.levelno), labels=labels, **kwargs, + message, + severity=_helpers._normalize_severity(record.levelno), + labels=labels, + **kwargs, ) From 3e210998659b2f994009bd9220114a483dc4a789 Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Wed, 26 May 2021 15:54:57 -0700 Subject: [PATCH 18/26] updated tests --- google/cloud/logging_v2/handlers/handlers.py | 7 ++----- .../logging_v2/handlers/transports/background_thread.py | 5 +++-- google/cloud/logging_v2/handlers/transports/sync.py | 3 ++- tests/unit/handlers/test_handlers.py | 8 +++++--- tests/unit/handlers/test_structured_log.py | 4 +++- tests/unit/handlers/transports/test_background_thread.py | 4 ++-- tests/unit/handlers/transports/test_sync.py | 4 ++-- 7 files changed, 19 insertions(+), 16 deletions(-) diff --git a/google/cloud/logging_v2/handlers/handlers.py b/google/cloud/logging_v2/handlers/handlers.py index 2298d4455..5894bc613 100644 --- a/google/cloud/logging_v2/handlers/handlers.py +++ b/google/cloud/logging_v2/handlers/handlers.py @@ -93,11 +93,8 @@ def filter(self, record): record._span_id = getattr(record, "span_id", inferred_span) or None record._http_request = getattr(record, "http_request", inferred_http) record._source_location = CloudLoggingFilter._infer_source_location(record) - record._labels = { - "python_logger": record.name, - **self.default_labels, - **user_labels, - } + logger_label = {"python_logger":record.name} if record.name else {} + record._labels = {**logger_label, **self.default_labels, **user_labels} or None # create string representations for structured logging record._trace_str = record._trace or "" record._span_id_str = record._span_id or "" diff --git a/google/cloud/logging_v2/handlers/transports/background_thread.py b/google/cloud/logging_v2/handlers/transports/background_thread.py index e02186d2b..9878ac9c1 100644 --- a/google/cloud/logging_v2/handlers/transports/background_thread.py +++ b/google/cloud/logging_v2/handlers/transports/background_thread.py @@ -233,8 +233,9 @@ def enqueue(self, record, message, **kwargs): """ # set python logger name as label if missing labels = kwargs.pop("labels", {}) - labels["python_logger"] = labels.get("python_logger", record.name) - kwargs["labels"] = labels + if record.name: + labels['python_logger'] = labels.get('python_logger', record.name) + kwargs['labels'] = labels # enqueue new entry queue_entry = { "message": message, diff --git a/google/cloud/logging_v2/handlers/transports/sync.py b/google/cloud/logging_v2/handlers/transports/sync.py index 46fd57254..64b4b9ff3 100644 --- a/google/cloud/logging_v2/handlers/transports/sync.py +++ b/google/cloud/logging_v2/handlers/transports/sync.py @@ -41,7 +41,8 @@ def send(self, record, message, **kwargs): """ # set python logger name as label if missing labels = kwargs.pop("labels", {}) - labels["python_logger"] = labels.get("python_logger", record.name) + if record.name: + labels['python_logger'] = labels.get('python_logger', record.name) # send log synchronously self.logger.log( message, diff --git a/tests/unit/handlers/test_handlers.py b/tests/unit/handlers/test_handlers.py index 87af10d1a..7c0a36c63 100644 --- a/tests/unit/handlers/test_handlers.py +++ b/tests/unit/handlers/test_handlers.py @@ -63,6 +63,7 @@ def test_filter_record(self): "file": "testpath", "function": "test-function", } + expected_label = {"python_logger":logname} record = logging.LogRecord( logname, logging.INFO, @@ -88,8 +89,8 @@ def test_filter_record(self): self.assertEqual(record._span_id_str, "") self.assertIsNone(record._http_request) self.assertEqual(record._http_request_str, "{}") - self.assertIsNone(record._labels) - self.assertEqual(record._labels_str, "{}") + self.assertEqual(record._labels, expected_label) + self.assertEqual(record._labels_str, json.dumps(expected_label)) def test_minimal_record(self): """ @@ -295,7 +296,7 @@ def test_emit(self): handler.handle(record) self.assertEqual( handler.transport.send_called_with, - (record, message, _GLOBAL_RESOURCE, None, None, None, None, None), + (record, message, _GLOBAL_RESOURCE, {"python_logger":logname}, None, None, None, None), ) def test_emit_manual_field_override(self): @@ -332,6 +333,7 @@ def test_emit_manual_field_override(self): "default_key": "default-value", "overwritten_key": "new_value", "added_key": "added_value", + "python_logger": logname, } setattr(record, "labels", added_labels) handler.handle(record) diff --git a/tests/unit/handlers/test_structured_log.py b/tests/unit/handlers/test_structured_log.py index 0536583a5..aabaf9e49 100644 --- a/tests/unit/handlers/test_structured_log.py +++ b/tests/unit/handlers/test_structured_log.py @@ -60,6 +60,7 @@ def test_format(self): record = logging.LogRecord( logname, logging.INFO, pathname, lineno, message, None, None, func=func ) + expected_labels = {**labels, "python_logger": logname} expected_payload = { "message": message, "severity": record.levelname, @@ -71,7 +72,7 @@ def test_format(self): "function": func, }, "httpRequest": {}, - "logging.googleapis.com/labels": labels, + "logging.googleapis.com/labels": expected_labels, } handler.filter(record) result = json.loads(handler.format(record)) @@ -197,6 +198,7 @@ def test_format_overrides(self): "default_key": "default-value", "overwritten_key": "new_value", "added_key": "added_value", + "python_logger": logname, }, } diff --git a/tests/unit/handlers/transports/test_background_thread.py b/tests/unit/handlers/transports/test_background_thread.py index 155ca3fb0..55a869165 100644 --- a/tests/unit/handlers/transports/test_background_thread.py +++ b/tests/unit/handlers/transports/test_background_thread.py @@ -283,10 +283,10 @@ def test_enqueue_defaults(self): self.assertEqual(entry["severity"], LogSeverity.INFO) self.assertIsInstance(entry["timestamp"], datetime.datetime) self.assertNotIn("resource", entry.keys()) - self.assertNotIn("labels", entry.keys()) self.assertNotIn("trace", entry.keys()) self.assertNotIn("span_id", entry.keys()) self.assertNotIn("http_request", entry.keys()) + self.assertEqual(entry["labels"], {"python_logger":"testing"}) def test_enqueue_explicit(self): import datetime @@ -315,7 +315,7 @@ def test_enqueue_explicit(self): self.assertEqual(entry["message"], message) self.assertEqual(entry["severity"], LogSeverity.ERROR) self.assertIs(entry["resource"], resource) - self.assertIs(entry["labels"], labels) + self.assertEqual(entry["labels"], {**labels, "python_logger":"testing"}) self.assertIs(entry["trace"], trace) self.assertIs(entry["span_id"], span_id) self.assertIsInstance(entry["timestamp"], datetime.datetime) diff --git a/tests/unit/handlers/transports/test_sync.py b/tests/unit/handlers/transports/test_sync.py index fd5f00c69..733050b23 100644 --- a/tests/unit/handlers/transports/test_sync.py +++ b/tests/unit/handlers/transports/test_sync.py @@ -54,7 +54,7 @@ def test_send(self): message, LogSeverity.INFO, _GLOBAL_RESOURCE, - None, + {"python_logger":python_logger_name}, None, None, None, @@ -80,7 +80,7 @@ def test_send_struct(self): message, LogSeverity.INFO, _GLOBAL_RESOURCE, - None, + {"python_logger":python_logger_name}, None, None, None, From 4d35cc461685b419a772608dc7725a197e50b2bc Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Wed, 26 May 2021 16:45:31 -0700 Subject: [PATCH 19/26] fixed system test --- tests/system/test_system.py | 62 ++++++++++++++++++------------------- 1 file changed, 31 insertions(+), 31 deletions(-) diff --git a/tests/system/test_system.py b/tests/system/test_system.py index f76ac2bed..f0e273e0e 100644 --- a/tests/system/test_system.py +++ b/tests/system/test_system.py @@ -348,38 +348,38 @@ def test_log_handler_sync(self): def test_handlers_w_extras(self): LOG_MESSAGE = "Testing with injected extras." + LOGGER_NAME = "handler_extras" + handler_name = self._logger_name(LOGGER_NAME) - for cls in [CloudLoggingHandler, AppEngineHandler]: - LOGGER_NAME = f"{cls.__name__}-handler_extras" - handler_name = self._logger_name(LOGGER_NAME) - - handler = cls(Config.CLIENT, name=handler_name, transport=SyncTransport) - - # only create the logger to delete, hidden otherwise - logger = Config.CLIENT.logger(handler.name) - self.to_delete.append(logger) - - cloud_logger = logging.getLogger(LOGGER_NAME) - cloud_logger.addHandler(handler) - expected_request = {"requestUrl": "localhost"} - expected_source = {"file": "test.py"} - extra = { - "trace": "123", - "span_id": "456", - "http_request": expected_request, - "source_location": expected_source, - "resource": Resource(type="cloudiot_device", labels={}), - "labels": {"test-label": "manual"}, - } - cloud_logger.warn(LOG_MESSAGE, extra=extra) - - entries = _list_entries(logger) - self.assertEqual(len(entries), 1) - self.assertEqual(entries[0].trace, extra["trace"]) - self.assertEqual(entries[0].span_id, extra["span_id"]) - self.assertEqual(entries[0].http_request, expected_request) - self.assertEqual(entries[0].labels, extra["labels"]) - self.assertEqual(entries[0].resource.type, extra["resource"].type) + handler = CloudLoggingHandler( + Config.CLIENT, name=handler_name, transport=SyncTransport + ) + + # only create the logger to delete, hidden otherwise + logger = Config.CLIENT.logger(handler.name) + self.to_delete.append(logger) + + cloud_logger = logging.getLogger(LOGGER_NAME) + cloud_logger.addHandler(handler) + expected_request = {"requestUrl": "localhost"} + expected_source = {"file": "test.py"} + extra = { + "trace": "123", + "span_id": "456", + "http_request": expected_request, + "source_location": expected_source, + "resource": Resource(type="cloudiot_device", labels={}), + "labels": {"test-label": "manual"}, + } + cloud_logger.warn(LOG_MESSAGE, extra=extra) + + entries = _list_entries(logger) + self.assertEqual(len(entries), 1) + self.assertEqual(entries[0].trace, extra["trace"]) + self.assertEqual(entries[0].span_id, extra["span_id"]) + self.assertEqual(entries[0].http_request, expected_request) + self.assertEqual(entries[0].labels, {**extra["labels"], "python_logger":LOGGER_NAME}) + self.assertEqual(entries[0].resource.type, extra["resource"].type) def test_log_root_handler(self): LOG_MESSAGE = "It was the best of times." From ad1662f2345cfd7208a35fe395acf55cdcec98a2 Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Fri, 4 Jun 2021 16:28:51 -0700 Subject: [PATCH 20/26] ran blacken --- google/cloud/logging_v2/handlers/handlers.py | 2 +- .../handlers/transports/background_thread.py | 4 ++-- google/cloud/logging_v2/handlers/transports/sync.py | 2 +- tests/system/test_system.py | 4 +++- tests/unit/handlers/test_handlers.py | 13 +++++++++++-- .../handlers/transports/test_background_thread.py | 4 ++-- tests/unit/handlers/transports/test_sync.py | 4 ++-- 7 files changed, 22 insertions(+), 11 deletions(-) diff --git a/google/cloud/logging_v2/handlers/handlers.py b/google/cloud/logging_v2/handlers/handlers.py index 5894bc613..2a3868166 100644 --- a/google/cloud/logging_v2/handlers/handlers.py +++ b/google/cloud/logging_v2/handlers/handlers.py @@ -93,7 +93,7 @@ def filter(self, record): record._span_id = getattr(record, "span_id", inferred_span) or None record._http_request = getattr(record, "http_request", inferred_http) record._source_location = CloudLoggingFilter._infer_source_location(record) - logger_label = {"python_logger":record.name} if record.name else {} + logger_label = {"python_logger": record.name} if record.name else {} record._labels = {**logger_label, **self.default_labels, **user_labels} or None # create string representations for structured logging record._trace_str = record._trace or "" diff --git a/google/cloud/logging_v2/handlers/transports/background_thread.py b/google/cloud/logging_v2/handlers/transports/background_thread.py index 9878ac9c1..6f80dd6f8 100644 --- a/google/cloud/logging_v2/handlers/transports/background_thread.py +++ b/google/cloud/logging_v2/handlers/transports/background_thread.py @@ -234,8 +234,8 @@ def enqueue(self, record, message, **kwargs): # set python logger name as label if missing labels = kwargs.pop("labels", {}) if record.name: - labels['python_logger'] = labels.get('python_logger', record.name) - kwargs['labels'] = labels + labels["python_logger"] = labels.get("python_logger", record.name) + kwargs["labels"] = labels # enqueue new entry queue_entry = { "message": message, diff --git a/google/cloud/logging_v2/handlers/transports/sync.py b/google/cloud/logging_v2/handlers/transports/sync.py index 64b4b9ff3..796f0d2ff 100644 --- a/google/cloud/logging_v2/handlers/transports/sync.py +++ b/google/cloud/logging_v2/handlers/transports/sync.py @@ -42,7 +42,7 @@ def send(self, record, message, **kwargs): # set python logger name as label if missing labels = kwargs.pop("labels", {}) if record.name: - labels['python_logger'] = labels.get('python_logger', record.name) + labels["python_logger"] = labels.get("python_logger", record.name) # send log synchronously self.logger.log( message, diff --git a/tests/system/test_system.py b/tests/system/test_system.py index f0e273e0e..f1059b2a2 100644 --- a/tests/system/test_system.py +++ b/tests/system/test_system.py @@ -378,7 +378,9 @@ def test_handlers_w_extras(self): self.assertEqual(entries[0].trace, extra["trace"]) self.assertEqual(entries[0].span_id, extra["span_id"]) self.assertEqual(entries[0].http_request, expected_request) - self.assertEqual(entries[0].labels, {**extra["labels"], "python_logger":LOGGER_NAME}) + self.assertEqual( + entries[0].labels, {**extra["labels"], "python_logger": LOGGER_NAME} + ) self.assertEqual(entries[0].resource.type, extra["resource"].type) def test_log_root_handler(self): diff --git a/tests/unit/handlers/test_handlers.py b/tests/unit/handlers/test_handlers.py index 7c0a36c63..4fba738a6 100644 --- a/tests/unit/handlers/test_handlers.py +++ b/tests/unit/handlers/test_handlers.py @@ -63,7 +63,7 @@ def test_filter_record(self): "file": "testpath", "function": "test-function", } - expected_label = {"python_logger":logname} + expected_label = {"python_logger": logname} record = logging.LogRecord( logname, logging.INFO, @@ -296,7 +296,16 @@ def test_emit(self): handler.handle(record) self.assertEqual( handler.transport.send_called_with, - (record, message, _GLOBAL_RESOURCE, {"python_logger":logname}, None, None, None, None), + ( + record, + message, + _GLOBAL_RESOURCE, + {"python_logger": logname}, + None, + None, + None, + None, + ), ) def test_emit_manual_field_override(self): diff --git a/tests/unit/handlers/transports/test_background_thread.py b/tests/unit/handlers/transports/test_background_thread.py index 55a869165..642f0f2f6 100644 --- a/tests/unit/handlers/transports/test_background_thread.py +++ b/tests/unit/handlers/transports/test_background_thread.py @@ -286,7 +286,7 @@ def test_enqueue_defaults(self): self.assertNotIn("trace", entry.keys()) self.assertNotIn("span_id", entry.keys()) self.assertNotIn("http_request", entry.keys()) - self.assertEqual(entry["labels"], {"python_logger":"testing"}) + self.assertEqual(entry["labels"], {"python_logger": "testing"}) def test_enqueue_explicit(self): import datetime @@ -315,7 +315,7 @@ def test_enqueue_explicit(self): self.assertEqual(entry["message"], message) self.assertEqual(entry["severity"], LogSeverity.ERROR) self.assertIs(entry["resource"], resource) - self.assertEqual(entry["labels"], {**labels, "python_logger":"testing"}) + self.assertEqual(entry["labels"], {**labels, "python_logger": "testing"}) self.assertIs(entry["trace"], trace) self.assertIs(entry["span_id"], span_id) self.assertIsInstance(entry["timestamp"], datetime.datetime) diff --git a/tests/unit/handlers/transports/test_sync.py b/tests/unit/handlers/transports/test_sync.py index 733050b23..7495abf78 100644 --- a/tests/unit/handlers/transports/test_sync.py +++ b/tests/unit/handlers/transports/test_sync.py @@ -54,7 +54,7 @@ def test_send(self): message, LogSeverity.INFO, _GLOBAL_RESOURCE, - {"python_logger":python_logger_name}, + {"python_logger": python_logger_name}, None, None, None, @@ -80,7 +80,7 @@ def test_send_struct(self): message, LogSeverity.INFO, _GLOBAL_RESOURCE, - {"python_logger":python_logger_name}, + {"python_logger": python_logger_name}, None, None, None, From fd3cbc5750302598f085a3a17e7a021b7be15525 Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Fri, 11 Jun 2021 13:07:45 -0700 Subject: [PATCH 21/26] fixed unit tests --- google/cloud/logging_v2/handlers/structured_log.py | 10 +++++----- tests/unit/handlers/test_handlers.py | 8 ++++---- tests/unit/handlers/test_structured_log.py | 1 - 3 files changed, 9 insertions(+), 10 deletions(-) diff --git a/google/cloud/logging_v2/handlers/structured_log.py b/google/cloud/logging_v2/handlers/structured_log.py index be8235f80..0b0462f9d 100644 --- a/google/cloud/logging_v2/handlers/structured_log.py +++ b/google/cloud/logging_v2/handlers/structured_log.py @@ -14,20 +14,20 @@ """Logging handler for printing formatted structured logs to standard output. """ +import collections import json import logging.handlers from google.cloud.logging_v2.handlers.handlers import CloudLoggingFilter GCP_FORMAT = ( - '{' + '{%(_payload_str)s' '"severity": "%(levelname)s", ' '"logging.googleapis.com/labels": %(_labels_str)s, ' '"logging.googleapis.com/trace": "%(_trace_str)s", ' '"logging.googleapis.com/spanId": "%(_span_id_str)s", ' '"logging.googleapis.com/sourceLocation": %(_source_location_str)s, ' - '"httpRequest": %(_http_request_str)s, ' - '%(_payload_str)s' + '"httpRequest": %(_http_request_str)s ' '}' ) @@ -66,13 +66,13 @@ def format(self, record): # if input is a dictionary, encode it as a json string encoded_msg = json.dumps(record.msg, ensure_ascii=False) # strip out open and close parentheses - payload = encoded_msg.lstrip('{').rstrip('}') + payload = encoded_msg.lstrip('{').rstrip('}') + "," elif record.msg: # otherwise, format based on superclass super_message = super(StructuredLogHandler, self).format(record) # properly break any formatting in string to make it json safe encoded_message = json.dumps(super_message, ensure_ascii=False) - payload = '"message": {}'.format(encoded_message) + payload = '"message": {},'.format(encoded_message) record._payload_str = payload or "" # convert to GCP structred logging format diff --git a/tests/unit/handlers/test_handlers.py b/tests/unit/handlers/test_handlers.py index 379674c32..eb48711cc 100644 --- a/tests/unit/handlers/test_handlers.py +++ b/tests/unit/handlers/test_handlers.py @@ -79,7 +79,6 @@ def test_filter_record(self): self.assertTrue(success) self.assertEqual(record.msg, message) - self.assertEqual(record._payload_str, '"message": "{}"'.format(message)) self.assertEqual(record._source_location, expected_location) self.assertEqual(record._source_location_str, json.dumps(expected_location)) self.assertIsNone(record._resource) @@ -106,7 +105,6 @@ def test_minimal_record(self): self.assertTrue(success) self.assertIsNone(record.msg) - self.assertEqual(record._payload_str, '"message": ""') self.assertIsNone(record._source_location) self.assertEqual(record._source_location_str, "{}") self.assertIsNone(record._resource) @@ -379,14 +377,16 @@ def test_emit_with_custom_formatter(self): handler.setFormatter(logFormatter) message = "test" expected_result = "logname :: INFO :: test" + logname = "logname" + expected_label = {"python_logger":logname} record = logging.LogRecord( - "logname", logging.INFO, None, None, message, None, None + logname, logging.INFO, None, None, message, None, None ) handler.handle(record) self.assertEqual( handler.transport.send_called_with, - (record, expected_result, _GLOBAL_RESOURCE, None, None, None, None, None,), + (record, expected_result, _GLOBAL_RESOURCE, expected_label, None, None, None, None,), ) def test_format_with_arguments(self): diff --git a/tests/unit/handlers/test_structured_log.py b/tests/unit/handlers/test_structured_log.py index d3d9269f8..dd09edbbf 100644 --- a/tests/unit/handlers/test_structured_log.py +++ b/tests/unit/handlers/test_structured_log.py @@ -92,7 +92,6 @@ def test_format_minimal(self): record = logging.LogRecord(None, logging.INFO, None, None, None, None, None,) record.created = None expected_payload = { - "message": "", "logging.googleapis.com/trace": "", "logging.googleapis.com/sourceLocation": {}, "httpRequest": {}, From 0a039c188f92eecb2a4fd8018e455f27f2cdf28a Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Fri, 11 Jun 2021 13:18:43 -0700 Subject: [PATCH 22/26] added comment --- google/cloud/logging_v2/handlers/handlers.py | 1 + 1 file changed, 1 insertion(+) diff --git a/google/cloud/logging_v2/handlers/handlers.py b/google/cloud/logging_v2/handlers/handlers.py index 38ccd198f..5d16e74b5 100644 --- a/google/cloud/logging_v2/handlers/handlers.py +++ b/google/cloud/logging_v2/handlers/handlers.py @@ -93,6 +93,7 @@ def filter(self, record): record._span_id = getattr(record, "span_id", inferred_span) or None record._http_request = getattr(record, "http_request", inferred_http) record._source_location = CloudLoggingFilter._infer_source_location(record) + # add logger name as a label if possible logger_label = {"python_logger": record.name} if record.name else {} record._labels = {**logger_label, **self.default_labels, **user_labels} or None # create string representations for structured logging From 6a5a79450549bbdf094bc7f2076ff4d5fbd3ccf3 Mon Sep 17 00:00:00 2001 From: Owl Bot Date: Fri, 11 Jun 2021 20:19:56 +0000 Subject: [PATCH 23/26] =?UTF-8?q?=F0=9F=A6=89=20Updates=20from=20OwlBot?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- google/cloud/logging_v2/handlers/structured_log.py | 6 +++--- tests/unit/handlers/test_handlers.py | 13 +++++++++++-- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/google/cloud/logging_v2/handlers/structured_log.py b/google/cloud/logging_v2/handlers/structured_log.py index 0b0462f9d..368d756d8 100644 --- a/google/cloud/logging_v2/handlers/structured_log.py +++ b/google/cloud/logging_v2/handlers/structured_log.py @@ -21,14 +21,14 @@ from google.cloud.logging_v2.handlers.handlers import CloudLoggingFilter GCP_FORMAT = ( - '{%(_payload_str)s' + "{%(_payload_str)s" '"severity": "%(levelname)s", ' '"logging.googleapis.com/labels": %(_labels_str)s, ' '"logging.googleapis.com/trace": "%(_trace_str)s", ' '"logging.googleapis.com/spanId": "%(_span_id_str)s", ' '"logging.googleapis.com/sourceLocation": %(_source_location_str)s, ' '"httpRequest": %(_http_request_str)s ' - '}' + "}" ) @@ -66,7 +66,7 @@ def format(self, record): # if input is a dictionary, encode it as a json string encoded_msg = json.dumps(record.msg, ensure_ascii=False) # strip out open and close parentheses - payload = encoded_msg.lstrip('{').rstrip('}') + "," + payload = encoded_msg.lstrip("{").rstrip("}") + "," elif record.msg: # otherwise, format based on superclass super_message = super(StructuredLogHandler, self).format(record) diff --git a/tests/unit/handlers/test_handlers.py b/tests/unit/handlers/test_handlers.py index eb48711cc..c51175261 100644 --- a/tests/unit/handlers/test_handlers.py +++ b/tests/unit/handlers/test_handlers.py @@ -378,7 +378,7 @@ def test_emit_with_custom_formatter(self): message = "test" expected_result = "logname :: INFO :: test" logname = "logname" - expected_label = {"python_logger":logname} + expected_label = {"python_logger": logname} record = logging.LogRecord( logname, logging.INFO, None, None, message, None, None ) @@ -386,7 +386,16 @@ def test_emit_with_custom_formatter(self): self.assertEqual( handler.transport.send_called_with, - (record, expected_result, _GLOBAL_RESOURCE, expected_label, None, None, None, None,), + ( + record, + expected_result, + _GLOBAL_RESOURCE, + expected_label, + None, + None, + None, + None, + ), ) def test_format_with_arguments(self): From 36e90e72f31fe1f6e90f88cd1e257df0598e777b Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Fri, 11 Jun 2021 13:23:08 -0700 Subject: [PATCH 24/26] fixed lint issues --- google/cloud/logging_v2/handlers/structured_log.py | 6 +++--- tests/system/test_system.py | 1 - tests/unit/handlers/test_handlers.py | 13 +++++++++++-- 3 files changed, 14 insertions(+), 6 deletions(-) diff --git a/google/cloud/logging_v2/handlers/structured_log.py b/google/cloud/logging_v2/handlers/structured_log.py index 0b0462f9d..368d756d8 100644 --- a/google/cloud/logging_v2/handlers/structured_log.py +++ b/google/cloud/logging_v2/handlers/structured_log.py @@ -21,14 +21,14 @@ from google.cloud.logging_v2.handlers.handlers import CloudLoggingFilter GCP_FORMAT = ( - '{%(_payload_str)s' + "{%(_payload_str)s" '"severity": "%(levelname)s", ' '"logging.googleapis.com/labels": %(_labels_str)s, ' '"logging.googleapis.com/trace": "%(_trace_str)s", ' '"logging.googleapis.com/spanId": "%(_span_id_str)s", ' '"logging.googleapis.com/sourceLocation": %(_source_location_str)s, ' '"httpRequest": %(_http_request_str)s ' - '}' + "}" ) @@ -66,7 +66,7 @@ def format(self, record): # if input is a dictionary, encode it as a json string encoded_msg = json.dumps(record.msg, ensure_ascii=False) # strip out open and close parentheses - payload = encoded_msg.lstrip('{').rstrip('}') + "," + payload = encoded_msg.lstrip("{").rstrip("}") + "," elif record.msg: # otherwise, format based on superclass super_message = super(StructuredLogHandler, self).format(record) diff --git a/tests/system/test_system.py b/tests/system/test_system.py index b0332c53c..365e94215 100644 --- a/tests/system/test_system.py +++ b/tests/system/test_system.py @@ -32,7 +32,6 @@ from google.api_core.exceptions import ServiceUnavailable import google.cloud.logging from google.cloud._helpers import UTC -from google.cloud.logging_v2.handlers import AppEngineHandler from google.cloud.logging_v2.handlers import CloudLoggingHandler from google.cloud.logging_v2.handlers.transports import SyncTransport from google.cloud.logging_v2 import client diff --git a/tests/unit/handlers/test_handlers.py b/tests/unit/handlers/test_handlers.py index eb48711cc..c51175261 100644 --- a/tests/unit/handlers/test_handlers.py +++ b/tests/unit/handlers/test_handlers.py @@ -378,7 +378,7 @@ def test_emit_with_custom_formatter(self): message = "test" expected_result = "logname :: INFO :: test" logname = "logname" - expected_label = {"python_logger":logname} + expected_label = {"python_logger": logname} record = logging.LogRecord( logname, logging.INFO, None, None, message, None, None ) @@ -386,7 +386,16 @@ def test_emit_with_custom_formatter(self): self.assertEqual( handler.transport.send_called_with, - (record, expected_result, _GLOBAL_RESOURCE, expected_label, None, None, None, None,), + ( + record, + expected_result, + _GLOBAL_RESOURCE, + expected_label, + None, + None, + None, + None, + ), ) def test_format_with_arguments(self): From d9fa11b99ebe17f8acace558a88839d52fd7a896 Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Thu, 17 Jun 2021 11:27:05 -0700 Subject: [PATCH 25/26] removed stackdriver branding from test --- tests/unit/handlers/transports/test_sync.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/unit/handlers/transports/test_sync.py b/tests/unit/handlers/transports/test_sync.py index 7495abf78..cc8ffe284 100644 --- a/tests/unit/handlers/transports/test_sync.py +++ b/tests/unit/handlers/transports/test_sync.py @@ -41,9 +41,9 @@ def test_send(self): client = _Client(self.PROJECT) - stackdriver_logger_name = "python" + client_name = "python" python_logger_name = "mylogger" - transport = self._make_one(client, stackdriver_logger_name) + transport = self._make_one(client, client_name) message = "hello world" record = logging.LogRecord( python_logger_name, logging.INFO, None, None, message, None, None @@ -67,9 +67,9 @@ def test_send_struct(self): client = _Client(self.PROJECT) - stackdriver_logger_name = "python" + client_name = "python" python_logger_name = "mylogger" - transport = self._make_one(client, stackdriver_logger_name) + transport = self._make_one(client, client_name) message = {"message": "hello world", "extra": "test"} record = logging.LogRecord( python_logger_name, logging.INFO, None, None, message, None, None From cdf22189998b30485e3d39ed14ca1257ea5a9547 Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Thu, 17 Jun 2021 11:28:59 -0700 Subject: [PATCH 26/26] improved comments --- google/cloud/logging_v2/handlers/_monitored_resources.py | 2 ++ google/cloud/logging_v2/handlers/structured_log.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/google/cloud/logging_v2/handlers/_monitored_resources.py b/google/cloud/logging_v2/handlers/_monitored_resources.py index e257f08e4..144258749 100644 --- a/google/cloud/logging_v2/handlers/_monitored_resources.py +++ b/google/cloud/logging_v2/handlers/_monitored_resources.py @@ -169,6 +169,8 @@ def _create_global_resource(project): def detect_resource(project=""): """Return the default monitored resource based on the local environment. + If GCP resource not found, defaults to `global`. + Args: project (str): The project ID to pass on to the resource (if needed) Returns: diff --git a/google/cloud/logging_v2/handlers/structured_log.py b/google/cloud/logging_v2/handlers/structured_log.py index 368d756d8..c981a1f27 100644 --- a/google/cloud/logging_v2/handlers/structured_log.py +++ b/google/cloud/logging_v2/handlers/structured_log.py @@ -59,7 +59,7 @@ def format(self, record): Args: record (logging.LogRecord): The log record. Returns: - str: A JSON string formatted for GKE fluentd. + str: A JSON string formatted for GCP structured logging. """ payload = None if isinstance(record.msg, collections.abc.Mapping):