Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly
Appearance settings

Commit b224218

Browse filesBrowse files
Pijukatelbarjin
Pijukatel
and
barjin
authored
fix: Presigned resource urls shouldn't follow base url (#500)
### Description - Allow the possibility to define a custom public URL; if not used, the default API public URL will be used. - Python version of the following fix JS client fix apify/apify-client-js#745 ### Issues - Closes: #496 ### Testing - Added new tests for relevant kvs and datasets clients' methods. --------- Co-authored-by: Jindřich Bär <jindrichbar@gmail.com>
1 parent c63ee1f commit b224218
Copy full SHA for b224218

8 files changed

+171-32Lines changed: 171 additions & 32 deletions

File tree

Expand file treeCollapse file tree
Open diff view settings
Filter options
Expand file treeCollapse file tree
Open diff view settings
Collapse file

‎src/apify_client/client.py‎

Copy file name to clipboardExpand all lines: src/apify_client/client.py
+19-3Lines changed: 19 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -64,6 +64,7 @@ def __init__(
6464
token: str | None = None,
6565
*,
6666
api_url: str | None = None,
67+
api_public_url: str | None = None,
6768
max_retries: int | None = 8,
6869
min_delay_between_retries_millis: int | None = 500,
6970
timeout_secs: int | None = DEFAULT_TIMEOUT,
@@ -72,7 +73,10 @@ def __init__(
7273
7374
Args:
7475
token: The Apify API token.
75-
api_url: The URL of the Apify API server to which to connect to. Defaults to https://api.apify.com.
76+
api_url: The URL of the Apify API server to which to connect. Defaults to https://api.apify.com. It can
77+
be an internal URL that is not globally accessible, in such case `api_public_url` should be set as well.
78+
api_public_url: The globally accessible URL of the Apify API server. It should be set only if the `api_url`
79+
is an internal URL that is not globally accessible.
7680
max_retries: How many times to retry a failed request at most.
7781
min_delay_between_retries_millis: How long will the client wait between retrying requests
7882
(increases exponentially from this value).
@@ -81,6 +85,8 @@ def __init__(
8185
self.token = token
8286
api_url = (api_url or DEFAULT_API_URL).rstrip('/')
8387
self.base_url = f'{api_url}/{API_VERSION}'
88+
api_public_url = (api_public_url or DEFAULT_API_URL).rstrip('/')
89+
self.public_base_url = f'{api_public_url}/{API_VERSION}'
8490
self.max_retries = max_retries or 8
8591
self.min_delay_between_retries_millis = min_delay_between_retries_millis or 500
8692
self.timeout_secs = timeout_secs or DEFAULT_TIMEOUT
@@ -103,6 +109,7 @@ def __init__(
103109
token: str | None = None,
104110
*,
105111
api_url: str | None = None,
112+
api_public_url: str | None = None,
106113
max_retries: int | None = 8,
107114
min_delay_between_retries_millis: int | None = 500,
108115
timeout_secs: int | None = DEFAULT_TIMEOUT,
@@ -111,7 +118,10 @@ def __init__(
111118
112119
Args:
113120
token: The Apify API token.
114-
api_url: The URL of the Apify API server to which to connect to. Defaults to https://api.apify.com.
121+
api_url: The URL of the Apify API server to which to connect. Defaults to https://api.apify.com. It can
122+
be an internal URL that is not globally accessible, in such case `api_public_url` should be set as well.
123+
api_public_url: The globally accessible URL of the Apify API server. It should be set only if the `api_url`
124+
is an internal URL that is not globally accessible.
115125
max_retries: How many times to retry a failed request at most.
116126
min_delay_between_retries_millis: How long will the client wait between retrying requests
117127
(increases exponentially from this value).
@@ -120,6 +130,7 @@ def __init__(
120130
super().__init__(
121131
token,
122132
api_url=api_url,
133+
api_public_url=api_public_url,
123134
max_retries=max_retries,
124135
min_delay_between_retries_millis=min_delay_between_retries_millis,
125136
timeout_secs=timeout_secs,
@@ -286,6 +297,7 @@ def __init__(
286297
token: str | None = None,
287298
*,
288299
api_url: str | None = None,
300+
api_public_url: str | None = None,
289301
max_retries: int | None = 8,
290302
min_delay_between_retries_millis: int | None = 500,
291303
timeout_secs: int | None = DEFAULT_TIMEOUT,
@@ -294,7 +306,10 @@ def __init__(
294306
295307
Args:
296308
token: The Apify API token.
297-
api_url: The URL of the Apify API server to which to connect to. Defaults to https://api.apify.com.
309+
api_url: The URL of the Apify API server to which to connect. Defaults to https://api.apify.com. It can
310+
be an internal URL that is not globally accessible, in such case `api_public_url` should be set as well.
311+
api_public_url: The globally accessible URL of the Apify API server. It should be set only if the `api_url`
312+
is an internal URL that is not globally accessible.
298313
max_retries: How many times to retry a failed request at most.
299314
min_delay_between_retries_millis: How long will the client wait between retrying requests
300315
(increases exponentially from this value).
@@ -303,6 +318,7 @@ def __init__(
303318
super().__init__(
304319
token,
305320
api_url=api_url,
321+
api_public_url=api_public_url,
306322
max_retries=max_retries,
307323
min_delay_between_retries_millis=min_delay_between_retries_millis,
308324
timeout_secs=timeout_secs,
Collapse file

‎src/apify_client/clients/base/base_client.py‎

Copy file name to clipboardExpand all lines: src/apify_client/clients/base/base_client.py
+8-4Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -18,10 +18,14 @@ class _BaseBaseClient(metaclass=WithLogDetailsClient):
1818
http_client: HTTPClient | HTTPClientAsync
1919
root_client: ApifyClient | ApifyClientAsync
2020

21-
def _url(self, path: str | None = None) -> str:
22-
if path is not None:
23-
return f'{self.url}/{path}'
24-
return self.url
21+
def _url(self, path: str | None = None, *, public: bool = False) -> str:
22+
url = f'{self.url}/{path}' if path is not None else self.url
23+
24+
if public:
25+
if not url.startswith(self.root_client.base_url):
26+
raise ValueError('API based URL has to start with `self.root_client.base_url`')
27+
return url.replace(self.root_client.base_url, self.root_client.public_base_url, 1)
28+
return url
2529

2630
def _params(self, **kwargs: Any) -> dict:
2731
return {
Collapse file

‎src/apify_client/clients/resource_clients/dataset.py‎

Copy file name to clipboardExpand all lines: src/apify_client/clients/resource_clients/dataset.py
+2-2Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -619,7 +619,7 @@ def create_items_public_url(
619619
)
620620
request_params['signature'] = signature
621621

622-
items_public_url = urlparse(self._url('items'))
622+
items_public_url = urlparse(self._url('items', public=True))
623623
filtered_params = {k: v for k, v in request_params.items() if v is not None}
624624
if filtered_params:
625625
items_public_url = items_public_url._replace(query=urlencode(filtered_params))
@@ -1126,7 +1126,7 @@ async def create_items_public_url(
11261126
)
11271127
request_params['signature'] = signature
11281128

1129-
items_public_url = urlparse(self._url('items'))
1129+
items_public_url = urlparse(self._url('items', public=True))
11301130
filtered_params = {k: v for k, v in request_params.items() if v is not None}
11311131
if filtered_params:
11321132
items_public_url = items_public_url._replace(query=urlencode(filtered_params))
Collapse file

‎src/apify_client/clients/resource_clients/key_value_store.py‎

Copy file name to clipboardExpand all lines: src/apify_client/clients/resource_clients/key_value_store.py
+2-2Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -307,7 +307,7 @@ def create_keys_public_url(
307307
)
308308
request_params['signature'] = signature
309309

310-
keys_public_url = urlparse(self._url('keys'))
310+
keys_public_url = urlparse(self._url('keys', public=True))
311311

312312
filtered_params = {k: v for k, v in request_params.items() if v is not None}
313313
if filtered_params:
@@ -597,7 +597,7 @@ async def create_keys_public_url(
597597
)
598598
request_params['signature'] = signature
599599

600-
keys_public_url = urlparse(self._url('keys'))
600+
keys_public_url = urlparse(self._url('keys', public=True))
601601
filtered_params = {k: v for k, v in request_params.items() if v is not None}
602602
if filtered_params:
603603
keys_public_url = keys_public_url._replace(query=urlencode(filtered_params))
Collapse file

‎tests/integration/conftest.py‎

Copy file name to clipboardExpand all lines: tests/integration/conftest.py
+9-11Lines changed: 9 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -9,13 +9,16 @@
99

1010

1111
@pytest.fixture
12-
def apify_client() -> ApifyClient:
13-
api_token = os.getenv(TOKEN_ENV_VAR)
14-
api_url = os.getenv(API_URL_ENV_VAR)
15-
16-
if not api_token:
12+
def api_token() -> str:
13+
token = os.getenv(TOKEN_ENV_VAR)
14+
if not token:
1715
raise RuntimeError(f'{TOKEN_ENV_VAR} environment variable is missing, cannot run tests!')
16+
return token
1817

18+
19+
@pytest.fixture
20+
def apify_client(api_token: str) -> ApifyClient:
21+
api_url = os.getenv(API_URL_ENV_VAR)
1922
return ApifyClient(api_token, api_url=api_url)
2023

2124

@@ -25,11 +28,6 @@ def apify_client() -> ApifyClient:
2528
# but `pytest-asyncio` closes the event loop after each test,
2629
# and uses a new one for the next test.
2730
@pytest.fixture
28-
def apify_client_async() -> ApifyClientAsync:
29-
api_token = os.getenv(TOKEN_ENV_VAR)
31+
def apify_client_async(api_token: str) -> ApifyClientAsync:
3032
api_url = os.getenv(API_URL_ENV_VAR)
31-
32-
if not api_token:
33-
raise RuntimeError(f'{TOKEN_ENV_VAR} environment variable is missing, cannot run tests!')
34-
3533
return ApifyClientAsync(api_token, api_url=api_url)
Collapse file

‎tests/integration/integration_test_utils.py‎

Copy file name to clipboard
+16Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,26 @@
11
import secrets
22
import string
33

4+
import pytest
5+
46

57
def random_string(length: int = 10) -> str:
68
return ''.join(secrets.choice(string.ascii_letters) for _ in range(length))
79

810

911
def random_resource_name(resource: str) -> str:
1012
return f'python-client-test-{resource}-{random_string(5)}'
13+
14+
15+
parametrized_api_urls = pytest.mark.parametrize(
16+
('api_url', 'api_public_url'),
17+
[
18+
('https://api.apify.com', 'https://api.apify.com'),
19+
('https://api.apify.com', None),
20+
('https://api.apify.com', 'https://custom-public-url.com'),
21+
('https://api.apify.com', 'https://custom-public-url.com/with/custom/path'),
22+
('https://api.apify.com', 'https://custom-public-url.com/with/custom/path/'),
23+
('http://10.0.88.214:8010', 'https://api.apify.com'),
24+
('http://10.0.88.214:8010', None),
25+
],
26+
)
Collapse file

‎tests/integration/test_dataset.py‎

Copy file name to clipboardExpand all lines: tests/integration/test_dataset.py
+58-5Lines changed: 58 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,40 @@
11
from __future__ import annotations
22

3-
from typing import TYPE_CHECKING
3+
from unittest import mock
4+
from unittest.mock import Mock
45

56
import impit
67

7-
from integration.integration_test_utils import random_resource_name
8-
9-
if TYPE_CHECKING:
10-
from apify_client import ApifyClient, ApifyClientAsync
8+
from integration.integration_test_utils import parametrized_api_urls, random_resource_name
9+
10+
from apify_client import ApifyClient, ApifyClientAsync
11+
from apify_client.client import DEFAULT_API_URL
12+
13+
MOCKED_API_DATASET_RESPONSE = """{
14+
"data": {
15+
"id": "someID",
16+
"name": "name",
17+
"userId": "userId",
18+
"createdAt": "2025-09-11T08:48:51.806Z",
19+
"modifiedAt": "2025-09-11T08:48:51.806Z",
20+
"accessedAt": "2025-09-11T08:48:51.806Z",
21+
"actId": null,
22+
"actRunId": null,
23+
"schema": null,
24+
"stats": {
25+
"readCount": 0,
26+
"writeCount": 0,
27+
"deleteCount": 0,
28+
"listCount": 0,
29+
"storageBytes": 0
30+
},
31+
"fields": [],
32+
"consoleUrl": "https://console.apify.com/storage/datasets/someID",
33+
"itemsPublicUrl": "https://api.apify.com/v2/datasets/someID/items",
34+
"generalAccess": "FOLLOW_USER_SETTING",
35+
"urlSigningSecretKey": "urlSigningSecretKey"
36+
}
37+
}"""
1138

1239

1340
class TestDatasetSync:
@@ -47,6 +74,19 @@ def test_dataset_should_create_public_items_non_expiring_url(self, apify_client:
4774
dataset.delete()
4875
assert apify_client.dataset(created_dataset['id']).get() is None
4976

77+
@parametrized_api_urls
78+
def test_public_url(self, api_token: str, api_url: str, api_public_url: str) -> None:
79+
apify_client = ApifyClient(token=api_token, api_url=api_url, api_public_url=api_public_url)
80+
dataset = apify_client.dataset('someID')
81+
82+
# Mock the API call to return predefined response
83+
with mock.patch.object(apify_client.http_client, 'call', return_value=Mock(text=MOCKED_API_DATASET_RESPONSE)):
84+
public_url = dataset.create_items_public_url()
85+
assert public_url == (
86+
f'{(api_public_url or DEFAULT_API_URL).strip("/")}/v2/datasets/'
87+
f'someID/items?signature={public_url.split("signature=")[1]}'
88+
)
89+
5090

5191
class TestDatasetAsync:
5292
async def test_dataset_should_create_public_items_expiring_url_with_params(
@@ -88,3 +128,16 @@ async def test_dataset_should_create_public_items_non_expiring_url(
88128

89129
await dataset.delete()
90130
assert await apify_client_async.dataset(created_dataset['id']).get() is None
131+
132+
@parametrized_api_urls
133+
async def test_public_url(self, api_token: str, api_url: str, api_public_url: str) -> None:
134+
apify_client = ApifyClientAsync(token=api_token, api_url=api_url, api_public_url=api_public_url)
135+
dataset = apify_client.dataset('someID')
136+
137+
# Mock the API call to return predefined response
138+
with mock.patch.object(apify_client.http_client, 'call', return_value=Mock(text=MOCKED_API_DATASET_RESPONSE)):
139+
public_url = await dataset.create_items_public_url()
140+
assert public_url == (
141+
f'{(api_public_url or DEFAULT_API_URL).strip("/")}/v2/datasets/'
142+
f'someID/items?signature={public_url.split("signature=")[1]}'
143+
)
Collapse file

‎tests/integration/test_key_value_store.py‎

Copy file name to clipboardExpand all lines: tests/integration/test_key_value_store.py
+57-5Lines changed: 57 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,39 @@
11
from __future__ import annotations
22

3-
from typing import TYPE_CHECKING
3+
from unittest import mock
4+
from unittest.mock import Mock
45

56
import impit
67

7-
from integration.integration_test_utils import random_resource_name
8-
9-
if TYPE_CHECKING:
10-
from apify_client import ApifyClient, ApifyClientAsync
8+
from integration.integration_test_utils import parametrized_api_urls, random_resource_name
9+
10+
from apify_client import ApifyClient, ApifyClientAsync
11+
from apify_client.client import DEFAULT_API_URL
12+
13+
MOCKED_API_KVS_RESPONSE = """{
14+
"data": {
15+
"id": "someID",
16+
"name": "name",
17+
"userId": "userId",
18+
"createdAt": "2025-09-11T08:48:51.806Z",
19+
"modifiedAt": "2025-09-11T08:48:51.806Z",
20+
"accessedAt": "2025-09-11T08:48:51.806Z",
21+
"actId": null,
22+
"actRunId": null,
23+
"schema": null,
24+
"stats": {
25+
"readCount": 0,
26+
"writeCount": 0,
27+
"deleteCount": 0,
28+
"listCount": 0,
29+
"storageBytes": 0
30+
},
31+
"consoleUrl": "https://console.apify.com/storage/key-value-stores/someID",
32+
"keysPublicUrl": "https://api.apify.com/v2/key-value-stores/someID/keys",
33+
"generalAccess": "FOLLOW_USER_SETTING",
34+
"urlSigningSecretKey": "urlSigningSecretKey"
35+
}
36+
}"""
1137

1238

1339
class TestKeyValueStoreSync:
@@ -47,6 +73,19 @@ def test_key_value_store_should_create_public_keys_non_expiring_url(self, apify_
4773
store.delete()
4874
assert apify_client.key_value_store(created_store['id']).get() is None
4975

76+
@parametrized_api_urls
77+
def test_public_url(self, api_token: str, api_url: str, api_public_url: str) -> None:
78+
apify_client = ApifyClient(token=api_token, api_url=api_url, api_public_url=api_public_url)
79+
kvs = apify_client.key_value_store('someID')
80+
81+
# Mock the API call to return predefined response
82+
with mock.patch.object(apify_client.http_client, 'call', return_value=Mock(text=MOCKED_API_KVS_RESPONSE)):
83+
public_url = kvs.create_keys_public_url()
84+
assert public_url == (
85+
f'{(api_public_url or DEFAULT_API_URL).strip("/")}/v2/key-value-stores/'
86+
f'someID/keys?signature={public_url.split("signature=")[1]}'
87+
)
88+
5089

5190
class TestKeyValueStoreAsync:
5291
async def test_key_value_store_should_create_expiring_keys_public_url_with_params(
@@ -90,3 +129,16 @@ async def test_key_value_store_should_create_public_keys_non_expiring_url(
90129

91130
await store.delete()
92131
assert await apify_client_async.key_value_store(created_store['id']).get() is None
132+
133+
@parametrized_api_urls
134+
async def test_public_url(self, api_token: str, api_url: str, api_public_url: str) -> None:
135+
apify_client = ApifyClientAsync(token=api_token, api_url=api_url, api_public_url=api_public_url)
136+
kvs = apify_client.key_value_store('someID')
137+
138+
# Mock the API call to return predefined response
139+
with mock.patch.object(apify_client.http_client, 'call', return_value=Mock(text=MOCKED_API_KVS_RESPONSE)):
140+
public_url = await kvs.create_keys_public_url()
141+
assert public_url == (
142+
f'{(api_public_url or DEFAULT_API_URL).strip("/")}/v2/key-value-stores/'
143+
f'someID/keys?signature={public_url.split("signature=")[1]}'
144+
)

0 commit comments

Comments
0 (0)
Morty Proxy This is a proxified and sanitized view of the page, visit original site.