Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly

Appearance settings

Commit 1333c95

Browse filesBrowse files
authored
feat: implement "append_from_file" (#1686)
feat: implement `append_from_file`
1 parent fe6d7c9 commit 1333c95
Copy full SHA for 1333c95

File tree

Expand file treeCollapse file tree

2 files changed

+45
-12
lines changed
Open diff view settings
Filter options
Expand file treeCollapse file tree

2 files changed

+45
-12
lines changed
Open diff view settings
Collapse file

‎google/cloud/storage/_experimental/asyncio/async_appendable_object_writer.py‎

Copy file name to clipboardExpand all lines: google/cloud/storage/_experimental/asyncio/async_appendable_object_writer.py
+14-3Lines changed: 14 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@
2121
if you want to use these Rapid Storage APIs.
2222
2323
"""
24+
from io import BufferedReader
2425
from typing import Optional, Union
2526

2627
from google_crc32c import Checksum
@@ -339,6 +340,16 @@ async def append_from_stream(self, stream_obj):
339340
"""
340341
raise NotImplementedError("append_from_stream is not implemented yet.")
341342

342-
async def append_from_file(self, file_path: str):
343-
"""Create a file object from `file_path` and call append_from_stream(file_obj)"""
344-
raise NotImplementedError("append_from_file is not implemented yet.")
343+
async def append_from_file(
344+
self, file_obj: BufferedReader, block_size: int = _DEFAULT_FLUSH_INTERVAL_BYTES
345+
):
346+
"""
347+
Appends data to an Appendable Object using file_handle which is opened
348+
for reading in binary mode.
349+
350+
:type file_obj: file
351+
:param file_obj: A file handle opened in binary mode for reading.
352+
353+
"""
354+
while block := file_obj.read(block_size):
355+
await self.append(block)
Collapse file

‎tests/unit/asyncio/test_async_appendable_object_writer.py‎

Copy file name to clipboardExpand all lines: tests/unit/asyncio/test_async_appendable_object_writer.py
+31-9Lines changed: 31 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@
1212
# See the License for the specific language governing permissions and
1313
# limitations under the License.
1414

15+
from io import BytesIO
1516
import pytest
1617
from unittest import mock
1718

@@ -23,6 +24,7 @@
2324
)
2425
from google.cloud.storage._experimental.asyncio.async_appendable_object_writer import (
2526
_MAX_CHUNK_SIZE_BYTES,
27+
_DEFAULT_FLUSH_INTERVAL_BYTES,
2628
)
2729
from google.cloud import _storage_v2
2830

@@ -287,9 +289,6 @@ async def test_unimplemented_methods_raise_error(mock_client):
287289
with pytest.raises(NotImplementedError):
288290
await writer.append_from_stream(mock.Mock())
289291

290-
with pytest.raises(NotImplementedError):
291-
await writer.append_from_file("file.txt")
292-
293292

294293
@pytest.mark.asyncio
295294
@mock.patch(
@@ -536,9 +535,6 @@ async def test_append_flushes_when_buffer_is_full(
536535
mock_write_object_stream, mock_client
537536
):
538537
"""Test that append flushes the stream when the buffer size is reached."""
539-
from google.cloud.storage._experimental.asyncio.async_appendable_object_writer import (
540-
_DEFAULT_FLUSH_INTERVAL_BYTES,
541-
)
542538

543539
writer = AsyncAppendableObjectWriter(mock_client, BUCKET, OBJECT)
544540
writer._is_stream_open = True
@@ -559,9 +555,6 @@ async def test_append_flushes_when_buffer_is_full(
559555
)
560556
async def test_append_handles_large_data(mock_write_object_stream, mock_client):
561557
"""Test that append handles data larger than the buffer size."""
562-
from google.cloud.storage._experimental.asyncio.async_appendable_object_writer import (
563-
_DEFAULT_FLUSH_INTERVAL_BYTES,
564-
)
565558

566559
writer = AsyncAppendableObjectWriter(mock_client, BUCKET, OBJECT)
567560
writer._is_stream_open = True
@@ -602,3 +595,32 @@ async def test_append_data_two_times(mock_write_object_stream, mock_client):
602595
total_data_length = len(data1) + len(data2)
603596
assert writer.offset == total_data_length
604597
assert writer.simple_flush.await_count == 0
598+
599+
600+
@pytest.mark.asyncio
601+
@pytest.mark.parametrize(
602+
"file_size, block_size",
603+
[
604+
(10, 4 * 1024),
605+
(0, _DEFAULT_FLUSH_INTERVAL_BYTES),
606+
(20 * 1024 * 1024, _DEFAULT_FLUSH_INTERVAL_BYTES),
607+
(16 * 1024 * 1024, _DEFAULT_FLUSH_INTERVAL_BYTES),
608+
],
609+
)
610+
async def test_append_from_file(file_size, block_size, mock_client):
611+
# arrange
612+
fp = BytesIO(b"a" * file_size)
613+
writer = AsyncAppendableObjectWriter(mock_client, BUCKET, OBJECT)
614+
writer._is_stream_open = True
615+
writer.append = mock.AsyncMock()
616+
617+
# act
618+
await writer.append_from_file(fp, block_size=block_size)
619+
620+
# assert
621+
exepected_calls = (
622+
file_size // block_size
623+
if file_size % block_size == 0
624+
else file_size // block_size + 1
625+
)
626+
assert writer.append.await_count == exepected_calls

0 commit comments

Comments
0 (0)
Morty Proxy This is a proxified and sanitized view of the page, visit original site.