Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly
Appearance settings

Commit 08bc708

Browse filesBrowse files
Pulkit0110Pulkit0110
authored
feat: update generation for MRD (#1730)
Adding a support to pass `generation` in the constructor of MRD. It is done to make it compatible with the other public interfaces. Earlier, `generation_number` was exposed as the parameter which would still be valid after these changes. But using `generation` is more preferred.
1 parent 74c9ecc commit 08bc708
Copy full SHA for 08bc708

File tree

Expand file treeCollapse file tree

7 files changed

+95
-78
lines changed
Open diff view settings
Filter options
Expand file treeCollapse file tree

7 files changed

+95
-78
lines changed
Open diff view settings
Collapse file

‎google/cloud/storage/asyncio/async_multi_range_downloader.py‎

Copy file name to clipboardExpand all lines: google/cloud/storage/asyncio/async_multi_range_downloader.py
+27-11Lines changed: 27 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -128,10 +128,11 @@ async def create_mrd(
128128
client: AsyncGrpcClient,
129129
bucket_name: str,
130130
object_name: str,
131-
generation_number: Optional[int] = None,
131+
generation: Optional[int] = None,
132132
read_handle: Optional[_storage_v2.BidiReadHandle] = None,
133133
retry_policy: Optional[AsyncRetry] = None,
134134
metadata: Optional[List[Tuple[str, str]]] = None,
135+
**kwargs,
135136
) -> AsyncMultiRangeDownloader:
136137
"""Initializes a MultiRangeDownloader and opens the underlying bidi-gRPC
137138
object for reading.
@@ -145,8 +146,8 @@ async def create_mrd(
145146
:type object_name: str
146147
:param object_name: The name of the object to be read.
147148
148-
:type generation_number: int
149-
:param generation_number: (Optional) If present, selects a specific
149+
:type generation: int
150+
:param generation: (Optional) If present, selects a specific
150151
revision of this object.
151152
152153
:type read_handle: _storage_v2.BidiReadHandle
@@ -162,7 +163,14 @@ async def create_mrd(
162163
:rtype: :class:`~google.cloud.storage.asyncio.async_multi_range_downloader.AsyncMultiRangeDownloader`
163164
:returns: An initialized AsyncMultiRangeDownloader instance for reading.
164165
"""
165-
mrd = cls(client, bucket_name, object_name, generation_number, read_handle)
166+
mrd = cls(
167+
client,
168+
bucket_name,
169+
object_name,
170+
generation=generation,
171+
read_handle=read_handle,
172+
**kwargs,
173+
)
166174
await mrd.open(retry_policy=retry_policy, metadata=metadata)
167175
return mrd
168176

@@ -171,8 +179,9 @@ def __init__(
171179
client: AsyncGrpcClient,
172180
bucket_name: str,
173181
object_name: str,
174-
generation_number: Optional[int] = None,
182+
generation: Optional[int] = None,
175183
read_handle: Optional[_storage_v2.BidiReadHandle] = None,
184+
**kwargs,
176185
) -> None:
177186
"""Constructor for AsyncMultiRangeDownloader, clients are not adviced to
178187
use it directly. Instead it's adviced to use the classmethod `create_mrd`.
@@ -186,20 +195,27 @@ def __init__(
186195
:type object_name: str
187196
:param object_name: The name of the object to be read.
188197
189-
:type generation_number: int
190-
:param generation_number: (Optional) If present, selects a specific revision of
198+
:type generation: int
199+
:param generation: (Optional) If present, selects a specific revision of
191200
this object.
192201
193202
:type read_handle: _storage_v2.BidiReadHandle
194203
:param read_handle: (Optional) An existing read handle.
195204
"""
205+
if "generation_number" in kwargs:
206+
if generation is not None:
207+
raise TypeError(
208+
"Cannot set both 'generation' and 'generation_number'. "
209+
"Use 'generation' for new code."
210+
)
211+
generation = kwargs.pop("generation_number")
196212

197213
raise_if_no_fast_crc32c()
198214

199215
self.client = client
200216
self.bucket_name = bucket_name
201217
self.object_name = object_name
202-
self.generation_number = generation_number
218+
self.generation = generation
203219
self.read_handle: Optional[_storage_v2.BidiReadHandle] = read_handle
204220
self.read_obj_str: Optional[_AsyncReadObjectStream] = None
205221
self._is_stream_open: bool = False
@@ -276,7 +292,7 @@ async def _do_open():
276292
client=self.client.grpc_client,
277293
bucket_name=self.bucket_name,
278294
object_name=self.object_name,
279-
generation_number=self.generation_number,
295+
generation_number=self.generation,
280296
read_handle=self.read_handle,
281297
)
282298

@@ -291,7 +307,7 @@ async def _do_open():
291307
)
292308

293309
if self.read_obj_str.generation_number:
294-
self.generation_number = self.read_obj_str.generation_number
310+
self.generation = self.read_obj_str.generation_number
295311
if self.read_obj_str.read_handle:
296312
self.read_handle = self.read_obj_str.read_handle
297313
if self.read_obj_str.persisted_size is not None:
@@ -435,7 +451,7 @@ async def generator():
435451
client=self.client.grpc_client,
436452
bucket_name=self.bucket_name,
437453
object_name=self.object_name,
438-
generation_number=self.generation_number,
454+
generation_number=self.generation,
439455
read_handle=current_handle,
440456
)
441457

Collapse file

‎tests/system/test_zonal.py‎

Copy file name to clipboardExpand all lines: tests/system/test_zonal.py
+2-1Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -264,7 +264,8 @@ async def _run():
264264

265265
event_loop.run_until_complete(_run())
266266

267-
@pytest.mark.skip(reason='Flaky test b/478129078')
267+
268+
@pytest.mark.skip(reason="Flaky test b/478129078")
268269
def test_mrd_open_with_read_handle(event_loop, grpc_client_direct):
269270
object_name = f"test_read_handl-{str(uuid.uuid4())[:4]}"
270271

Collapse file

‎tests/unit/asyncio/retry/test_reads_resumption_strategy.py‎

Copy file name to clipboardExpand all lines: tests/unit/asyncio/retry/test_reads_resumption_strategy.py
+1-3Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -27,9 +27,7 @@
2727
from google.cloud._storage_v2.types.storage import BidiReadObjectRedirectedError
2828

2929
_READ_ID = 1
30-
LOGGER_NAME = (
31-
"google.cloud.storage.asyncio.retry.reads_resumption_strategy"
32-
)
30+
LOGGER_NAME = "google.cloud.storage.asyncio.retry.reads_resumption_strategy"
3331

3432

3533
class TestDownloadState(unittest.TestCase):
Collapse file

‎tests/unit/asyncio/test_async_multi_range_downloader.py‎

Copy file name to clipboardExpand all lines: tests/unit/asyncio/test_async_multi_range_downloader.py
+49-9Lines changed: 49 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -49,7 +49,7 @@ async def _make_mock_mrd(
4949
mock_cls_async_read_object_stream,
5050
bucket_name=_TEST_BUCKET_NAME,
5151
object_name=_TEST_OBJECT_NAME,
52-
generation_number=_TEST_GENERATION_NUMBER,
52+
generation=_TEST_GENERATION_NUMBER,
5353
read_handle=_TEST_READ_HANDLE,
5454
):
5555
mock_client = mock.MagicMock()
@@ -62,7 +62,7 @@ async def _make_mock_mrd(
6262
mock_stream.read_handle = _TEST_READ_HANDLE
6363

6464
mrd = await AsyncMultiRangeDownloader.create_mrd(
65-
mock_client, bucket_name, object_name, generation_number, read_handle
65+
mock_client, bucket_name, object_name, generation, read_handle
6666
)
6767

6868
return mrd, mock_client
@@ -89,7 +89,7 @@ async def test_create_mrd(self, mock_cls_async_read_object_stream):
8989
assert mrd.client == mock_client
9090
assert mrd.bucket_name == _TEST_BUCKET_NAME
9191
assert mrd.object_name == _TEST_OBJECT_NAME
92-
assert mrd.generation_number == _TEST_GENERATION_NUMBER
92+
assert mrd.generation == _TEST_GENERATION_NUMBER
9393
assert mrd.read_handle == _TEST_READ_HANDLE
9494
assert mrd.persisted_size == _TEST_OBJECT_SIZE
9595
assert mrd.is_stream_open
@@ -303,9 +303,7 @@ async def test_downloading_without_opening_should_throw_error(self):
303303
assert not mrd.is_stream_open
304304

305305
@mock.patch("google.cloud.storage.asyncio._utils.google_crc32c")
306-
def test_init_raises_if_crc32c_c_extension_is_missing(
307-
self, mock_google_crc32c
308-
):
306+
def test_init_raises_if_crc32c_c_extension_is_missing(self, mock_google_crc32c):
309307
mock_google_crc32c.implementation = "python"
310308
mock_client = mock.MagicMock()
311309

@@ -317,9 +315,7 @@ def test_init_raises_if_crc32c_c_extension_is_missing(
317315
)
318316

319317
@pytest.mark.asyncio
320-
@mock.patch(
321-
"google.cloud.storage.asyncio.retry.reads_resumption_strategy.Checksum"
322-
)
318+
@mock.patch("google.cloud.storage.asyncio.retry.reads_resumption_strategy.Checksum")
323319
async def test_download_ranges_raises_on_checksum_mismatch(
324320
self, mock_checksum_class
325321
):
@@ -405,3 +401,47 @@ async def close_side_effect():
405401

406402
mock_close.assert_called_once()
407403
assert not mrd.is_stream_open
404+
405+
@mock.patch(
406+
"google.cloud.storage.asyncio.async_multi_range_downloader._AsyncReadObjectStream"
407+
)
408+
@pytest.mark.asyncio
409+
async def test_create_mrd_with_generation_number(
410+
self, mock_cls_async_read_object_stream
411+
):
412+
# Arrange
413+
mock_client = mock.MagicMock()
414+
mock_client.grpc_client = mock.AsyncMock()
415+
416+
mock_stream = mock_cls_async_read_object_stream.return_value
417+
mock_stream.open = AsyncMock()
418+
mock_stream.generation_number = _TEST_GENERATION_NUMBER
419+
mock_stream.persisted_size = _TEST_OBJECT_SIZE
420+
mock_stream.read_handle = _TEST_READ_HANDLE
421+
422+
# Act
423+
mrd = await AsyncMultiRangeDownloader.create_mrd(
424+
mock_client,
425+
_TEST_BUCKET_NAME,
426+
_TEST_OBJECT_NAME,
427+
generation_number=_TEST_GENERATION_NUMBER,
428+
read_handle=_TEST_READ_HANDLE,
429+
)
430+
431+
# Assert
432+
assert mrd.generation == _TEST_GENERATION_NUMBER
433+
434+
@pytest.mark.asyncio
435+
async def test_create_mrd_with_both_generation_and_generation_number(self):
436+
# Arrange
437+
mock_client = mock.MagicMock()
438+
439+
# Act & Assert
440+
with pytest.raises(TypeError):
441+
await AsyncMultiRangeDownloader.create_mrd(
442+
mock_client,
443+
_TEST_BUCKET_NAME,
444+
_TEST_OBJECT_NAME,
445+
generation=_TEST_GENERATION_NUMBER,
446+
generation_number=_TEST_GENERATION_NUMBER,
447+
)
Collapse file

‎tests/unit/asyncio/test_async_read_object_stream.py‎

Copy file name to clipboardExpand all lines: tests/unit/asyncio/test_async_read_object_stream.py
+12-36Lines changed: 12 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -79,9 +79,7 @@ async def instantiate_read_obj_stream_with_read_handle(
7979
return read_obj_stream
8080

8181

82-
@mock.patch(
83-
"google.cloud.storage.asyncio.async_read_object_stream.AsyncBidiRpc"
84-
)
82+
@mock.patch("google.cloud.storage.asyncio.async_read_object_stream.AsyncBidiRpc")
8583
@mock.patch(
8684
"google.cloud.storage.asyncio.async_grpc_client.AsyncGrpcClient.grpc_client"
8785
)
@@ -110,9 +108,7 @@ def test_init_with_bucket_object_generation(mock_client, mock_async_bidi_rpc):
110108
assert read_obj_stream.rpc == rpc_sentinel
111109

112110

113-
@mock.patch(
114-
"google.cloud.storage.asyncio.async_read_object_stream.AsyncBidiRpc"
115-
)
111+
@mock.patch("google.cloud.storage.asyncio.async_read_object_stream.AsyncBidiRpc")
116112
@mock.patch(
117113
"google.cloud.storage.asyncio.async_grpc_client.AsyncGrpcClient.grpc_client"
118114
)
@@ -136,9 +132,7 @@ async def test_open(mock_client, mock_cls_async_bidi_rpc):
136132
assert read_obj_stream.is_stream_open
137133

138134

139-
@mock.patch(
140-
"google.cloud.storage.asyncio.async_read_object_stream.AsyncBidiRpc"
141-
)
135+
@mock.patch("google.cloud.storage.asyncio.async_read_object_stream.AsyncBidiRpc")
142136
@mock.patch(
143137
"google.cloud.storage.asyncio.async_grpc_client.AsyncGrpcClient.grpc_client"
144138
)
@@ -162,9 +156,7 @@ async def test_open_with_read_handle(mock_client, mock_cls_async_bidi_rpc):
162156
assert read_obj_stream.is_stream_open
163157

164158

165-
@mock.patch(
166-
"google.cloud.storage.asyncio.async_read_object_stream.AsyncBidiRpc"
167-
)
159+
@mock.patch("google.cloud.storage.asyncio.async_read_object_stream.AsyncBidiRpc")
168160
@mock.patch(
169161
"google.cloud.storage.asyncio.async_grpc_client.AsyncGrpcClient.grpc_client"
170162
)
@@ -185,9 +177,7 @@ async def test_open_when_already_open_should_raise_error(
185177
assert str(exc.value) == "Stream is already open"
186178

187179

188-
@mock.patch(
189-
"google.cloud.storage.asyncio.async_read_object_stream.AsyncBidiRpc"
190-
)
180+
@mock.patch("google.cloud.storage.asyncio.async_read_object_stream.AsyncBidiRpc")
191181
@mock.patch(
192182
"google.cloud.storage.asyncio.async_grpc_client.AsyncGrpcClient.grpc_client"
193183
)
@@ -208,9 +198,7 @@ async def test_close(mock_client, mock_cls_async_bidi_rpc):
208198
assert not read_obj_stream.is_stream_open
209199

210200

211-
@mock.patch(
212-
"google.cloud.storage.asyncio.async_read_object_stream.AsyncBidiRpc"
213-
)
201+
@mock.patch("google.cloud.storage.asyncio.async_read_object_stream.AsyncBidiRpc")
214202
@mock.patch(
215203
"google.cloud.storage.asyncio.async_grpc_client.AsyncGrpcClient.grpc_client"
216204
)
@@ -232,9 +220,7 @@ async def test_requests_done(mock_client, mock_cls_async_bidi_rpc):
232220
read_obj_stream.socket_like_rpc.recv.assert_called_once()
233221

234222

235-
@mock.patch(
236-
"google.cloud.storage.asyncio.async_read_object_stream.AsyncBidiRpc"
237-
)
223+
@mock.patch("google.cloud.storage.asyncio.async_read_object_stream.AsyncBidiRpc")
238224
@mock.patch(
239225
"google.cloud.storage.asyncio.async_grpc_client.AsyncGrpcClient.grpc_client"
240226
)
@@ -255,9 +241,7 @@ async def test_close_without_open_should_raise_error(
255241
assert str(exc.value) == "Stream is not open"
256242

257243

258-
@mock.patch(
259-
"google.cloud.storage.asyncio.async_read_object_stream.AsyncBidiRpc"
260-
)
244+
@mock.patch("google.cloud.storage.asyncio.async_read_object_stream.AsyncBidiRpc")
261245
@mock.patch(
262246
"google.cloud.storage.asyncio.async_grpc_client.AsyncGrpcClient.grpc_client"
263247
)
@@ -278,9 +262,7 @@ async def test_send(mock_client, mock_cls_async_bidi_rpc):
278262
)
279263

280264

281-
@mock.patch(
282-
"google.cloud.storage.asyncio.async_read_object_stream.AsyncBidiRpc"
283-
)
265+
@mock.patch("google.cloud.storage.asyncio.async_read_object_stream.AsyncBidiRpc")
284266
@mock.patch(
285267
"google.cloud.storage.asyncio.async_grpc_client.AsyncGrpcClient.grpc_client"
286268
)
@@ -301,9 +283,7 @@ async def test_send_without_open_should_raise_error(
301283
assert str(exc.value) == "Stream is not open"
302284

303285

304-
@mock.patch(
305-
"google.cloud.storage.asyncio.async_read_object_stream.AsyncBidiRpc"
306-
)
286+
@mock.patch("google.cloud.storage.asyncio.async_read_object_stream.AsyncBidiRpc")
307287
@mock.patch(
308288
"google.cloud.storage.asyncio.async_grpc_client.AsyncGrpcClient.grpc_client"
309289
)
@@ -326,9 +306,7 @@ async def test_recv(mock_client, mock_cls_async_bidi_rpc):
326306
assert response == bidi_read_object_response
327307

328308

329-
@mock.patch(
330-
"google.cloud.storage.asyncio.async_read_object_stream.AsyncBidiRpc"
331-
)
309+
@mock.patch("google.cloud.storage.asyncio.async_read_object_stream.AsyncBidiRpc")
332310
@mock.patch(
333311
"google.cloud.storage.asyncio.async_grpc_client.AsyncGrpcClient.grpc_client"
334312
)
@@ -349,9 +327,7 @@ async def test_recv_without_open_should_raise_error(
349327
assert str(exc.value) == "Stream is not open"
350328

351329

352-
@mock.patch(
353-
"google.cloud.storage.asyncio.async_read_object_stream.AsyncBidiRpc"
354-
)
330+
@mock.patch("google.cloud.storage.asyncio.async_read_object_stream.AsyncBidiRpc")
355331
@mock.patch(
356332
"google.cloud.storage.asyncio.async_grpc_client.AsyncGrpcClient.grpc_client"
357333
)

0 commit comments

Comments
0 (0)
Morty Proxy This is a proxified and sanitized view of the page, visit original site.