Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly

Appearance settings

Commit d08be9a

Browse filesBrowse files
feat: include context on batch log errors (#650)
1 parent 329c861 commit d08be9a
Copy full SHA for d08be9a

File tree

Expand file treeCollapse file tree

2 files changed

+141
-4
lines changed
Filter options
Expand file treeCollapse file tree

2 files changed

+141
-4
lines changed

‎google/cloud/logging_v2/logger.py

Copy file name to clipboardExpand all lines: google/cloud/logging_v2/logger.py
+39-4Lines changed: 39 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@
1515
"""Define API Loggers."""
1616

1717
import collections
18+
import re
1819

1920
from google.cloud.logging_v2._helpers import _add_defaults_to_filter
2021
from google.cloud.logging_v2.entries import LogEntry
@@ -25,6 +26,9 @@
2526
from google.cloud.logging_v2.handlers._monitored_resources import detect_resource
2627
from google.cloud.logging_v2._instrumentation import _add_instrumentation
2728

29+
from google.api_core.exceptions import InvalidArgument
30+
from google.rpc.error_details_pb2 import DebugInfo
31+
2832
import google.protobuf.message
2933

3034
_GLOBAL_RESOURCE = Resource(type="global", labels={})
@@ -459,8 +463,39 @@ def commit(self, *, client=None, partial_success=True):
459463
kwargs["labels"] = self.logger.labels
460464

461465
entries = [entry.to_api_repr() for entry in self.entries]
462-
463-
client.logging_api.write_entries(
464-
entries, partial_success=partial_success, **kwargs
465-
)
466+
try:
467+
client.logging_api.write_entries(
468+
entries, partial_success=partial_success, **kwargs
469+
)
470+
except InvalidArgument as e:
471+
# InvalidArgument is often sent when a log is too large
472+
# attempt to attach extra contex on which log caused error
473+
self._append_context_to_error(e)
474+
raise e
466475
del self.entries[:]
476+
477+
def _append_context_to_error(self, err):
478+
"""
479+
Attempts to Modify `write_entries` exception messages to contain
480+
context on which log in the batch caused the error.
481+
482+
Best-effort basis. If another exception occurs while processing the
483+
input exception, the input will be left unmodified
484+
485+
Args:
486+
err (~google.api_core.exceptions.InvalidArgument):
487+
The original exception object
488+
"""
489+
try:
490+
# find debug info proto if in details
491+
debug_info = next(x for x in err.details if isinstance(x, DebugInfo))
492+
# parse out the index of the faulty entry
493+
error_idx = re.search("(?<=key: )[0-9]+", debug_info.detail).group(0)
494+
# find the faulty entry object
495+
found_entry = self.entries[int(error_idx)]
496+
str_entry = str(found_entry.to_api_repr())
497+
# modify error message to contain extra context
498+
err.message = f"{err.message}: {str_entry:.2000}..."
499+
except Exception:
500+
# if parsing fails, abort changes and leave err unmodified
501+
pass

‎tests/unit/test_logger.py

Copy file name to clipboardExpand all lines: tests/unit/test_logger.py
+102Lines changed: 102 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,8 +16,10 @@
1616
from datetime import datetime
1717
from datetime import timedelta
1818
from datetime import timezone
19+
import sys
1920

2021
import unittest
22+
import pytest
2123

2224
import mock
2325

@@ -1739,6 +1741,87 @@ def test_context_mgr_failure(self):
17391741
self.assertEqual(list(batch.entries), UNSENT)
17401742
self.assertIsNone(api._write_entries_called_with)
17411743

1744+
@pytest.mark.skipif(
1745+
sys.version_info < (3, 8),
1746+
reason="InvalidArgument init with details requires python3.8+",
1747+
)
1748+
def test_append_context_to_error(self):
1749+
"""
1750+
If an InvalidArgument exception contains info on the log that threw it,
1751+
we should be able to add it to the exceptiom message. If not, the
1752+
exception should be unchanged
1753+
"""
1754+
from google.api_core.exceptions import InvalidArgument
1755+
from google.rpc.error_details_pb2 import DebugInfo
1756+
from google.cloud.logging import TextEntry
1757+
1758+
logger = _Logger()
1759+
client = _Client(project=self.PROJECT)
1760+
batch = self._make_one(logger, client=client)
1761+
test_entries = [TextEntry(payload=str(i)) for i in range(11)]
1762+
batch.entries = test_entries
1763+
starting_message = "test"
1764+
# test that properly formatted exceptions add log details
1765+
for idx, entry in enumerate(test_entries):
1766+
api_entry = entry.to_api_repr()
1767+
err = InvalidArgument(
1768+
starting_message, details=["padding", DebugInfo(detail=f"key: {idx}")]
1769+
)
1770+
batch._append_context_to_error(err)
1771+
self.assertEqual(err.message, f"{starting_message}: {str(api_entry)}...")
1772+
self.assertIn(str(idx), str(entry))
1773+
# test with missing debug info
1774+
err = InvalidArgument(starting_message, details=[])
1775+
batch._append_context_to_error(err)
1776+
self.assertEqual(
1777+
err.message, starting_message, "message should have been unchanged"
1778+
)
1779+
# test with missing key
1780+
err = InvalidArgument(
1781+
starting_message, details=["padding", DebugInfo(detail="no k e y here")]
1782+
)
1783+
batch._append_context_to_error(err)
1784+
self.assertEqual(
1785+
err.message, starting_message, "message should have been unchanged"
1786+
)
1787+
# test with key out of range
1788+
err = InvalidArgument(
1789+
starting_message, details=["padding", DebugInfo(detail="key: 100")]
1790+
)
1791+
batch._append_context_to_error(err)
1792+
self.assertEqual(
1793+
err.message, starting_message, "message should have been unchanged"
1794+
)
1795+
1796+
@pytest.mark.skipif(
1797+
sys.version_info < (3, 8),
1798+
reason="InvalidArgument init with details requires python3.8+",
1799+
)
1800+
def test_batch_error_gets_context(self):
1801+
"""
1802+
Simulate an InvalidArgument sent as part of a batch commit, to ensure
1803+
_append_context_to_error is thrown
1804+
"""
1805+
from google.api_core.exceptions import InvalidArgument
1806+
from google.rpc.error_details_pb2 import DebugInfo
1807+
from google.cloud.logging import TextEntry
1808+
1809+
logger = _Logger()
1810+
client = _Client(project=self.PROJECT)
1811+
starting_message = "hello"
1812+
exception = InvalidArgument(
1813+
starting_message, details=[DebugInfo(detail="key: 1")]
1814+
)
1815+
client.logging_api = _DummyLoggingExceptionAPI(exception)
1816+
batch = self._make_one(logger, client=client)
1817+
test_entries = [TextEntry(payload=str(i)) for i in range(11)]
1818+
batch.entries = test_entries
1819+
with self.assertRaises(InvalidArgument) as e:
1820+
batch.commit()
1821+
expected_log = test_entries[1]
1822+
api_entry = expected_log.to_api_repr()
1823+
self.assertEqual(e.message, f"{starting_message}: {str(api_entry)}...")
1824+
17421825

17431826
class _Logger(object):
17441827

@@ -1773,6 +1856,25 @@ def logger_delete(self, logger_name):
17731856
self._logger_delete_called_with = logger_name
17741857

17751858

1859+
class _DummyLoggingExceptionAPI(object):
1860+
def __init__(self, exception):
1861+
self.exception = exception
1862+
1863+
def write_entries(
1864+
self,
1865+
entries,
1866+
*,
1867+
logger_name=None,
1868+
resource=None,
1869+
labels=None,
1870+
partial_success=False,
1871+
):
1872+
raise self.exception
1873+
1874+
def logger_delete(self, logger_name):
1875+
raise self.exception
1876+
1877+
17761878
class _Client(object):
17771879
def __init__(self, project, connection=None):
17781880
self.project = project

0 commit comments

Comments
0 (0)
Morty Proxy This is a proxified and sanitized view of the page, visit original site.