Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly
Appearance settings

Commit 99d6e00

Browse filesBrowse files
daspecsterdaspecster
authored
Merge pull request googleapis#2907 from daspecster/increase-bigtable-receive-length
Add max_message_length for larger rows.
2 parents 1a0ce44 + 0ed6d3a commit 99d6e00
Copy full SHA for 99d6e00

File tree

Expand file treeCollapse file tree

5 files changed

+84
-10
lines changed
Filter options
Expand file treeCollapse file tree

5 files changed

+84
-10
lines changed

‎bigtable/google/cloud/bigtable/client.py

Copy file name to clipboardExpand all lines: bigtable/google/cloud/bigtable/client.py
+10-1Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -65,6 +65,14 @@
6565
READ_ONLY_SCOPE = 'https://www.googleapis.com/auth/bigtable.data.readonly'
6666
"""Scope for reading table data."""
6767

68+
# NOTE: 'grpc.max_message_length' will no longer be recognized in
69+
# grpcio 1.1 and later.
70+
_MAX_MSG_LENGTH_100MB = 100 * 1024 * 1024
71+
_GRPC_MAX_LENGTH_OPTIONS = (
72+
('grpc.max_message_length', _MAX_MSG_LENGTH_100MB),
73+
('grpc.max_receive_message_length', _MAX_MSG_LENGTH_100MB),
74+
)
75+
6876

6977
def _make_data_stub(client):
7078
"""Creates gRPC stub to make requests to the Data API.
@@ -77,7 +85,8 @@ def _make_data_stub(client):
7785
"""
7886
if client.emulator_host is None:
7987
return make_secure_stub(client.credentials, client.user_agent,
80-
bigtable_pb2.BigtableStub, DATA_API_HOST)
88+
bigtable_pb2.BigtableStub, DATA_API_HOST,
89+
extra_options=_GRPC_MAX_LENGTH_OPTIONS)
8190
else:
8291
return make_insecure_stub(bigtable_pb2.BigtableStub,
8392
client.emulator_host)

‎bigtable/unit_tests/test_client.py

Copy file name to clipboardExpand all lines: bigtable/unit_tests/test_client.py
+7-1Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -46,13 +46,18 @@ def test_without_emulator(self):
4646
fake_stub = object()
4747
make_secure_stub_args = []
4848

49-
def mock_make_secure_stub(*args):
49+
def mock_make_secure_stub(*args, **kwargs):
5050
make_secure_stub_args.append(args)
51+
make_secure_stub_args.append(kwargs)
5152
return fake_stub
5253

5354
with _Monkey(MUT, make_secure_stub=mock_make_secure_stub):
5455
result = self._call_fut(client)
5556

57+
extra_options = {'extra_options': (
58+
('grpc.max_message_length', 104857600),
59+
('grpc.max_receive_message_length', 104857600)
60+
)}
5661
self.assertIs(result, fake_stub)
5762
self.assertEqual(make_secure_stub_args, [
5863
(
@@ -61,6 +66,7 @@ def mock_make_secure_stub(*args):
6166
MUT.bigtable_pb2.BigtableStub,
6267
MUT.DATA_API_HOST,
6368
),
69+
extra_options,
6470
])
6571

6672
def test_with_emulator(self):

‎core/google/cloud/_helpers.py

Copy file name to clipboardExpand all lines: core/google/cloud/_helpers.py
+19-6Lines changed: 19 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -465,7 +465,7 @@ def _name_from_project_path(path, project, template):
465465
return match.group('name')
466466

467467

468-
def make_secure_channel(credentials, user_agent, host):
468+
def make_secure_channel(credentials, user_agent, host, extra_options=None):
469469
"""Makes a secure channel for an RPC service.
470470
471471
Uses / depends on gRPC.
@@ -480,22 +480,30 @@ def make_secure_channel(credentials, user_agent, host):
480480
:type host: str
481481
:param host: The host for the service.
482482
483+
:type extra_options: tuple
484+
:param extra_options: (Optional) Extra gRPC options used when creating the
485+
channel.
486+
483487
:rtype: :class:`grpc._channel.Channel`
484488
:returns: gRPC secure channel with credentials attached.
485489
"""
486490
target = '%s:%d' % (host, http_client.HTTPS_PORT)
487491
http_request = google_auth_httplib2.Request(http=httplib2.Http())
488-
options = (
489-
('grpc.primary_user_agent', user_agent),
490-
)
492+
493+
user_agent_option = ('grpc.primary_user_agent', user_agent)
494+
if extra_options is not None:
495+
options = (user_agent_option,) + extra_options
496+
else:
497+
options = (user_agent_option,)
491498
return google.auth.transport.grpc.secure_authorized_channel(
492499
credentials,
493500
http_request,
494501
target,
495502
options=options)
496503

497504

498-
def make_secure_stub(credentials, user_agent, stub_class, host):
505+
def make_secure_stub(credentials, user_agent, stub_class, host,
506+
extra_options=None):
499507
"""Makes a secure stub for an RPC service.
500508
501509
Uses / depends on gRPC.
@@ -513,10 +521,15 @@ def make_secure_stub(credentials, user_agent, stub_class, host):
513521
:type host: str
514522
:param host: The host for the service.
515523
524+
:type extra_options: tuple
525+
:param extra_options: (Optional) Extra gRPC options passed when creating
526+
the channel.
527+
516528
:rtype: object, instance of ``stub_class``
517529
:returns: The stub object used to make gRPC requests to a given API.
518530
"""
519-
channel = make_secure_channel(credentials, user_agent, host)
531+
channel = make_secure_channel(credentials, user_agent, host,
532+
extra_options=extra_options)
520533
return stub_class(channel)
521534

522535

‎core/unit_tests/test__helpers.py

Copy file name to clipboardExpand all lines: core/unit_tests/test__helpers.py
+31-2Lines changed: 31 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -644,6 +644,33 @@ def test_it(self):
644644
secure_authorized_channel.assert_called_once_with(
645645
credentials, mock.ANY, expected_target, options=expected_options)
646646

647+
def test_extra_options(self):
648+
from six.moves import http_client
649+
650+
credentials = object()
651+
host = 'HOST'
652+
user_agent = 'USER_AGENT'
653+
extra_options = (('some', 'option'),)
654+
655+
secure_authorized_channel_patch = mock.patch(
656+
'google.auth.transport.grpc.secure_authorized_channel',
657+
autospec=True)
658+
659+
with secure_authorized_channel_patch as secure_authorized_channel:
660+
result = self._call_fut(credentials, user_agent, host,
661+
extra_options)
662+
663+
self.assertIs(result, secure_authorized_channel.return_value)
664+
665+
expected_target = '%s:%d' % (host, http_client.HTTPS_PORT)
666+
expected_options = (
667+
('grpc.primary_user_agent', user_agent),
668+
extra_options[0],
669+
)
670+
671+
secure_authorized_channel.assert_called_once_with(
672+
credentials, mock.ANY, expected_target, options=expected_options)
673+
647674

648675
class Test_make_secure_stub(unittest.TestCase):
649676

@@ -664,21 +691,23 @@ def stub_class(channel):
664691
channels.append(channel)
665692
return result
666693

667-
def mock_channel(*args):
694+
def mock_channel(*args, **kwargs):
668695
channel_args.append(args)
696+
channel_args.append(kwargs)
669697
return channel_obj
670698

671699
credentials = object()
672700
user_agent = 'you-sir-age-int'
673701
host = 'localhost'
702+
extra_options = {'extra_options': None}
674703
with _Monkey(MUT, make_secure_channel=mock_channel):
675704
stub = self._call_fut(credentials, user_agent,
676705
stub_class, host)
677706

678707
self.assertIs(stub, result)
679708
self.assertEqual(channels, [channel_obj])
680709
self.assertEqual(channel_args,
681-
[(credentials, user_agent, host)])
710+
[(credentials, user_agent, host), extra_options])
682711

683712

684713
class Test_make_insecure_stub(unittest.TestCase):

‎system_tests/bigtable.py

Copy file name to clipboardExpand all lines: system_tests/bigtable.py
+17Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -356,6 +356,23 @@ def _write_to_row(self, row1=None, row2=None, row3=None, row4=None):
356356
cell4 = Cell(CELL_VAL4, timestamp4)
357357
return cell1, cell2, cell3, cell4
358358

359+
def test_read_large_cell_limit(self):
360+
row = self._table.row(ROW_KEY)
361+
self.rows_to_delete.append(row)
362+
363+
number_of_bytes = 10 * 1024 * 1024
364+
data = b'1' * number_of_bytes # 10MB of 1's.
365+
row.set_cell(COLUMN_FAMILY_ID1, COL_NAME1, data)
366+
row.commit()
367+
368+
# Read back the contents of the row.
369+
partial_row_data = self._table.read_row(ROW_KEY)
370+
self.assertEqual(partial_row_data.row_key, ROW_KEY)
371+
cell = partial_row_data.cells[COLUMN_FAMILY_ID1]
372+
column = cell[COL_NAME1]
373+
self.assertEqual(len(column), 1)
374+
self.assertEqual(column[0].value, data)
375+
359376
def test_read_row(self):
360377
row = self._table.row(ROW_KEY)
361378
self.rows_to_delete.append(row)

0 commit comments

Comments
0 (0)
Morty Proxy This is a proxified and sanitized view of the page, visit original site.