From bbe6c27326b70e60b84fd33c7184202d5192839a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franti=C5=A1ek=20Nesveda?= Date: Fri, 6 Dec 2024 13:05:04 +0100 Subject: [PATCH] feat: add new config variables to `Actor.config` --- poetry.lock | 8 ++--- pyproject.toml | 2 +- src/apify/_configuration.py | 34 +++++++++++++++++++++- tests/unit/actor/test_actor_env_helpers.py | 23 ++++++++++++++- 4 files changed, 60 insertions(+), 7 deletions(-) diff --git a/poetry.lock b/poetry.lock index ceed67d1..6a051c92 100644 --- a/poetry.lock +++ b/poetry.lock @@ -62,13 +62,13 @@ more_itertools = ">=10.0.0" [[package]] name = "apify-shared" -version = "1.1.2" +version = "1.2.1" description = "Tools and constants shared across Apify projects." optional = false python-versions = ">=3.8" files = [ - {file = "apify_shared-1.1.2-py3-none-any.whl", hash = "sha256:a422b4cb3ea536257f0c0dd3778b3c22ec6647708d2fc06c97fe0c9fe183c3c9"}, - {file = "apify_shared-1.1.2.tar.gz", hash = "sha256:3f98f526635c4c3f92c1496916513a1c4013bb47158d7e0f914df8987147366c"}, + {file = "apify_shared-1.2.1-py3-none-any.whl", hash = "sha256:cee136729c41c215796d8ca9f7b2e5736995d44f9471663d61827d16f592df9b"}, + {file = "apify_shared-1.2.1.tar.gz", hash = "sha256:986557e2b01c584aa57258fb4af83d32ecb6979c0d804ccbfda7c0e79a2d00b1"}, ] [package.extras] @@ -3529,4 +3529,4 @@ scrapy = ["scrapy"] [metadata] lock-version = "2.0" python-versions = "^3.9" -content-hash = "dd3449989524195813aa229339ebdc3732798bca1e360e8e7b13a1ba1954d2b2" +content-hash = "fd101911ca38d535954db3fe16589f6c4d81bc30bbf9fa6484c82314bd25998f" diff --git a/pyproject.toml b/pyproject.toml index 722f0c73..c1c1d65c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -44,7 +44,7 @@ keywords = [ [tool.poetry.dependencies] python = "^3.9" apify-client = ">=1.8.1" -apify-shared = ">=1.1.2" +apify-shared = ">=1.2.1" crawlee = "~0.4.0" cryptography = ">=42.0.0" # TODO: relax the upper bound once the issue is resolved: diff --git a/src/apify/_configuration.py b/src/apify/_configuration.py index d839c30a..018a6e98 100644 --- a/src/apify/_configuration.py +++ b/src/apify/_configuration.py @@ -1,7 +1,7 @@ from __future__ import annotations from datetime import datetime, timedelta -from typing import Annotated +from typing import Annotated, Any from pydantic import AliasChoices, BeforeValidator, Field from typing_extensions import deprecated @@ -13,6 +13,14 @@ from apify._utils import docs_group +def _transform_to_list(value: Any) -> list[str] | None: + if value is None: + return None + if not value: + return [] + return value if isinstance(value, list) else str(value).split(',') + + @docs_group('Classes') class Configuration(CrawleeConfiguration): """A class for specifying the configuration of an Actor. @@ -33,6 +41,13 @@ class Configuration(CrawleeConfiguration): ), ] = None + actor_full_name: Annotated[ + str | None, + Field( + description='Full name of the Actor', + ), + ] = None + actor_run_id: Annotated[ str | None, Field( @@ -67,6 +82,14 @@ class Configuration(CrawleeConfiguration): ), ] = None + actor_build_tags: Annotated[ + list[str] | None, + Field( + description='Build tags of the Actor build used in the run', + ), + BeforeValidator(_transform_to_list), + ] = None + actor_task_id: Annotated[ str | None, Field( @@ -185,6 +208,15 @@ class Configuration(CrawleeConfiguration): BeforeValidator(lambda val: val or None), ] = None + max_total_charge_usd: Annotated[ + float | None, + Field( + alias='actor_max_total_charge_usd', + description='For pay-per-event Actors, the user-set limit on total charges. Do not exceed this limit', + ), + BeforeValidator(lambda val: val or None), + ] = None + meta_origin: Annotated[ str | None, Field( diff --git a/tests/unit/actor/test_actor_env_helpers.py b/tests/unit/actor/test_actor_env_helpers.py index 6cc0560b..770fb856 100644 --- a/tests/unit/actor/test_actor_env_helpers.py +++ b/tests/unit/actor/test_actor_env_helpers.py @@ -9,6 +9,7 @@ from apify_shared.consts import ( BOOL_ENV_VARS, + COMMA_SEPARATED_LIST_ENV_VARS, DATETIME_ENV_VARS, FLOAT_ENV_VARS, INTEGER_ENV_VARS, @@ -108,9 +109,29 @@ async def test_get_env_with_randomized_env_vars(monkeypatch: pytest.MonkeyPatch) continue string_get_env_var = string_env_var.name.lower() - expected_get_env[string_get_env_var] = ''.join(random.choices(string.ascii_uppercase + string.digits, k=10)) + expected_value = ''.join(random.choices(string.ascii_uppercase + string.digits, k=10)) + # URLs have to be valid + if string_get_env_var.endswith('url'): + expected_value = f'http://example.com/{expected_value}' + expected_get_env[string_get_env_var] = expected_value monkeypatch.setenv(string_env_var, expected_get_env[string_get_env_var]) + for list_env_var in COMMA_SEPARATED_LIST_ENV_VARS: + if list_env_var in ignored_env_vars: + continue + + available_values = ['val1', 'val2'] + + list_get_env_var = list_env_var.name.lower() + expected_value_count = random.randint(0, len(available_values)) + expected_get_env[list_get_env_var] = random.sample(available_values, expected_value_count) + monkeypatch.setenv(list_env_var, ','.join(expected_get_env[list_get_env_var])) + + # Test behavior with mising env var in case of empty list + if expected_value_count == 0 and random.random() < 0.5: + monkeypatch.delenv(list_env_var) + expected_get_env[list_get_env_var] = None + # We need this override so that the actor doesn't fail when connecting to the platform events websocket monkeypatch.delenv(ActorEnvVars.EVENTS_WEBSOCKET_URL) monkeypatch.delenv(ApifyEnvVars.ACTOR_EVENTS_WS_URL)