apify · jirimoravcik · Jan 24, 2023 · Jan 18, 2023 · Jan 18, 2023 · Jan 18, 2023
diff --git a/setup.py b/setup.py
@@ -50,7 +50,7 @@
    package_data={'apify': ['py.typed']},
    python_requires='>=3.8',
    install_requires=[
-        'apify-client ~= 0.7.0b30',
+        'apify-client ~= 0.7.0b39',
        'httpx ~= 0.23.0',
        'psutil ~= 5.9.4',
        'pydantic ~= 1.10.2',

diff --git a/src/apify/_utils.py b/src/apify/_utils.py
@@ -10,11 +10,16 @@
 import mimetypes
 import os
 import re
+import secrets
 import sys
 import time
+from collections import OrderedDict
+from collections.abc import MutableMapping
 from datetime import datetime, timezone
 from enum import Enum
-from typing import Any, Callable, Dict, Generic, NoReturn, Optional, Type, TypeVar, Union, cast, overload
+from typing import Any, Callable, Dict, Generic, ItemsView, Iterator, NoReturn, Optional
+from typing import OrderedDict as OrderedDictType
+from typing import Type, TypeVar, Union, ValuesView, cast, overload

 import aioshutil
 import psutil
@@ -319,5 +324,62 @@ def wrapper(*args: Any, **kwargs: Any) -> Any:
    return cast(MetadataType, wrapper)


+def _crypto_random_object_id(length: int = 17) -> str:
+    """Python reimplementation of cryptoRandomObjectId from `@apify/utilities`."""
+    chars = 'abcdefghijklmnopqrstuvwxyzABCEDFGHIJKLMNOPQRSTUVWXYZ0123456789'
+    return ''.join(secrets.choice(chars) for _ in range(length))
+
+
+T = TypeVar('T')
+
+
+class LRUCache(MutableMapping, Generic[T]):
+    """Attempt to reimplement LRUCache from `@apify/datastructures` using `OrderedDict`."""
+
+    _cache: OrderedDictType[str, T]
+
+    _max_length: int
+
+    def __init__(self, max_length: int) -> None:
+        """Create a LRUCache with a specific max_length."""
+        self._cache = OrderedDict()
+        self._max_length = max_length
+
+    def __getitem__(self, key: str) -> T:
+        """Get an item from the cache. Move it to the end if present."""
+        val = self._cache[key]
+        # No 'key in cache' condition since the previous line would raise KeyError
+        self._cache.move_to_end(key)
+        return val
+
+    # Sadly TS impl returns bool indicating whether the key was already present or not
+    def __setitem__(self, key: str, value: T) -> None:
+        """Add an item to the cache. Remove least used item if max_length exceeded."""
+        self._cache[key] = value
+        if len(self._cache) > self._max_length:
+            self._cache.popitem(last=False)
+
+    def __delitem__(self, key: str) -> None:
+        """Remove an item from the cache."""
+        # TODO: maybe do? self._cache.__delitem__(key)
+        del self._cache[key]
+
+    def __iter__(self) -> Iterator:
+        """Iterate over the keys of the cache in order of insertion."""
+        yield from self._cache.__iter__()
+
+    def __len__(self) -> int:
+        """Get the number of items in the cache."""
+        return len(self._cache)
+
+    def values(self) -> ValuesView[T]:  # Needed so we don't mutate the cache by __getitem__
+        """Iterate over the values in the cache in order of insertion."""
+        return self._cache.values()
+
+    def items(self) -> ItemsView[str, T]:  # Needed so we don't mutate the cache by __getitem__
+        """Iterate over the pairs of (key, value) in the cache in order of insertion."""
+        return self._cache.items()
+
+
 def _is_running_in_ipython() -> bool:
    return getattr(builtins, '__IPYTHON__', False)
diff --git a/src/apify/memory_storage/resource_clients/dataset_collection.py b/src/apify/memory_storage/resource_clients/dataset_collection.py
@@ -21,7 +21,7 @@ def __init__(self, *, base_storage_directory: str, client: 'MemoryStorage') -> N
        self._datasets_directory = base_storage_directory
        self._client = client

-    def list(self) -> ListPage:
+    def list(self) -> ListPage[Dict]:
        """TODO: docs."""
        def map_store(store: DatasetClient) -> Dict:
            return store.to_dataset_info()

diff --git a/src/apify/memory_storage/resource_clients/key_value_store_collection.py b/src/apify/memory_storage/resource_clients/key_value_store_collection.py
@@ -21,7 +21,7 @@ def __init__(self, *, base_storage_directory: str, client: 'MemoryStorage') -> N
        self._key_value_stores_directory = base_storage_directory
        self._client = client

-    def list(self) -> ListPage:
+    def list(self) -> ListPage[Dict]:
        """TODO: docs."""
        def map_store(store: KeyValueStoreClient) -> Dict:
            return store.to_key_value_store_info()

diff --git a/src/apify/memory_storage/resource_clients/request_queue_collection.py b/src/apify/memory_storage/resource_clients/request_queue_collection.py
@@ -21,7 +21,7 @@ def __init__(self, *, base_storage_directory: str, client: 'MemoryStorage') -> N
        self._request_queues_directory = base_storage_directory
        self._client = client

-    def list(self) -> ListPage:
+    def list(self) -> ListPage[Dict]:
        """TODO: docs."""
        def map_store(store: RequestQueueClient) -> Dict:
            return store.to_request_queue_info()

diff --git a/src/apify/storages/_utils.py b/src/apify/storages/_utils.py
diff --git a/src/apify/storages/dataset.py b/src/apify/storages/dataset.py
@@ -21,7 +21,6 @@ class Dataset:
    _id: str
    _name: Optional[str]
    _client: Union[DatasetClientAsync, DatasetClient]
-    _config: Configuration

    def __init__(self, id: str, name: Optional[str], client: Union[ApifyClientAsync, MemoryStorage]) -> None:
        """TODO: docs (constructor should be "internal")."""
@@ -32,7 +31,6 @@ def __init__(self, id: str, name: Optional[str], client: Union[ApifyClientAsync,
        self._id = id
        self._name = name
        self._client = client.dataset(self._id)
-        self._config = Configuration.get_global_configuration()  # We always use the global config

    @classmethod
    async def _create_instance(cls, dataset_id_or_name: str, client: Union[ApifyClientAsync, MemoryStorage]) -> 'Dataset':

diff --git a/src/apify/storages/key_value_store.py b/src/apify/storages/key_value_store.py
@@ -18,7 +18,6 @@ class KeyValueStore:
    _id: str
    _name: Optional[str]
    _client: Union[KeyValueStoreClientAsync, KeyValueStoreClient]
-    _config: Configuration

    def __init__(self, id: str, name: Optional[str], client: Union[ApifyClientAsync, MemoryStorage]) -> None:
        """TODO: docs (constructor should be "internal")."""
@@ -27,7 +26,6 @@ def __init__(self, id: str, name: Optional[str], client: Union[ApifyClientAsync,
        self._id = id
        self._name = name
        self._client = client.key_value_store(self._id)
-        self._config = Configuration.get_global_configuration()  # We always use the global config

    @classmethod
    async def _create_instance(cls, store_id_or_name: str, client: Union[ApifyClientAsync, MemoryStorage]) -> 'KeyValueStore':