Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly
Appearance settings
This repository was archived by the owner on Sep 5, 2023. It is now read-only.

Commit 06a127b

Browse filesBrowse files
gcf-owl-bot[bot]gcf-owl-bot[bot]
authored
feat: Max concurrent backfill tasks (#168)
* feat: Max concurrent backfill tasks You can now set the number of maximum concurrent backfill tasks for a stream using the Datastream API. PiperOrigin-RevId: 530067890 Source-Link: googleapis/googleapis@b2c290f Source-Link: https://github.com/googleapis/googleapis-gen/commit/83c5413e535cb6fa710541dc6e169a8a44b8a65d Copy-Tag: eyJwIjoiLmdpdGh1Yi8uT3dsQm90LnlhbWwiLCJoIjoiODNjNTQxM2U1MzVjYjZmYTcxMDU0MWRjNmUxNjlhOGE0NGI4YTY1ZCJ9 * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md --------- Co-authored-by: Owl Bot <gcf-owl-bot[bot]@users.noreply.github.com>
1 parent 3e2f9f4 commit 06a127b
Copy full SHA for 06a127b

File tree

Expand file treeCollapse file tree

6 files changed

+69
-13
lines changed
Filter options
Expand file treeCollapse file tree

6 files changed

+69
-13
lines changed

‎google/cloud/datastream_v1/types/datastream.py

Copy file name to clipboardExpand all lines: google/cloud/datastream_v1/types/datastream.py
+7Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1056,6 +1056,9 @@ class CreatePrivateConnectionRequest(proto.Message):
10561056
The request ID must be a valid UUID with the
10571057
exception that zero UUID is not supported
10581058
(00000000-0000-0000-0000-000000000000).
1059+
force (bool):
1060+
Optional. If set to true, will skip
1061+
validations.
10591062
"""
10601063

10611064
parent: str = proto.Field(
@@ -1075,6 +1078,10 @@ class CreatePrivateConnectionRequest(proto.Message):
10751078
proto.STRING,
10761079
number=4,
10771080
)
1081+
force: bool = proto.Field(
1082+
proto.BOOL,
1083+
number=6,
1084+
)
10781085

10791086

10801087
class ListPrivateConnectionsRequest(proto.Message):

‎google/cloud/datastream_v1/types/datastream_resources.py

Copy file name to clipboardExpand all lines: google/cloud/datastream_v1/types/datastream_resources.py
+45-10Lines changed: 45 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -226,7 +226,12 @@ class BigQueryProfile(proto.Message):
226226

227227

228228
class StaticServiceIpConnectivity(proto.Message):
229-
r"""Static IP address connectivity."""
229+
r"""Static IP address connectivity. Used when the source database
230+
is configured to allow incoming connections from the Datastream
231+
public IP addresses for the region specified in the connection
232+
profile.
233+
234+
"""
230235

231236

232237
class ForwardSshTunnelConnectivity(proto.Message):
@@ -792,15 +797,20 @@ class OracleSourceConfig(proto.Message):
792797
Oracle objects to exclude from the stream.
793798
max_concurrent_cdc_tasks (int):
794799
Maximum number of concurrent CDC tasks. The
795-
number should be non negative. If not set (or
796-
set to 0), the system's default value will be
800+
number should be non-negative. If not set (or
801+
set to 0), the system's default value is used.
802+
max_concurrent_backfill_tasks (int):
803+
Maximum number of concurrent backfill tasks.
804+
The number should be non-negative. If not set
805+
(or set to 0), the system's default value is
797806
used.
798807
drop_large_objects (google.cloud.datastream_v1.types.OracleSourceConfig.DropLargeObjects):
799808
Drop large object values.
800809
801810
This field is a member of `oneof`_ ``large_objects_handling``.
802811
stream_large_objects (google.cloud.datastream_v1.types.OracleSourceConfig.StreamLargeObjects):
803-
Stream large object values.
812+
Stream large object values. NOTE: This
813+
feature is currently experimental.
804814
805815
This field is a member of `oneof`_ ``large_objects_handling``.
806816
"""
@@ -825,6 +835,10 @@ class StreamLargeObjects(proto.Message):
825835
proto.INT32,
826836
number=3,
827837
)
838+
max_concurrent_backfill_tasks: int = proto.Field(
839+
proto.INT32,
840+
number=4,
841+
)
828842
drop_large_objects: DropLargeObjects = proto.Field(
829843
proto.MESSAGE,
830844
number=100,
@@ -967,12 +981,18 @@ class PostgresqlSourceConfig(proto.Message):
967981
PostgreSQL objects to exclude from the
968982
stream.
969983
replication_slot (str):
970-
Required. The name of the logical replication
971-
slot that's configured with the pgoutput plugin.
984+
Required. Immutable. The name of the logical
985+
replication slot that's configured with the
986+
pgoutput plugin.
972987
publication (str):
973988
Required. The name of the publication that includes the set
974989
of all tables that are defined in the stream's
975990
include_objects.
991+
max_concurrent_backfill_tasks (int):
992+
Maximum number of concurrent backfill tasks.
993+
The number should be non negative. If not set
994+
(or set to 0), the system's default value will
995+
be used.
976996
"""
977997

978998
include_objects: "PostgresqlRdbms" = proto.Field(
@@ -993,6 +1013,10 @@ class PostgresqlSourceConfig(proto.Message):
9931013
proto.STRING,
9941014
number=4,
9951015
)
1016+
max_concurrent_backfill_tasks: int = proto.Field(
1017+
proto.INT32,
1018+
number=5,
1019+
)
9961020

9971021

9981022
class MysqlColumn(proto.Message):
@@ -1122,6 +1146,11 @@ class MysqlSourceConfig(proto.Message):
11221146
number should be non negative. If not set (or
11231147
set to 0), the system's default value will be
11241148
used.
1149+
max_concurrent_backfill_tasks (int):
1150+
Maximum number of concurrent backfill tasks.
1151+
The number should be non negative. If not set
1152+
(or set to 0), the system's default value will
1153+
be used.
11251154
"""
11261155

11271156
include_objects: "MysqlRdbms" = proto.Field(
@@ -1138,6 +1167,10 @@ class MysqlSourceConfig(proto.Message):
11381167
proto.INT32,
11391168
number=3,
11401169
)
1170+
max_concurrent_backfill_tasks: int = proto.Field(
1171+
proto.INT32,
1172+
number=4,
1173+
)
11411174

11421175

11431176
class SourceConfig(proto.Message):
@@ -1268,7 +1301,8 @@ class GcsDestinationConfig(proto.Message):
12681301
file_rotation_interval (google.protobuf.duration_pb2.Duration):
12691302
The maximum duration for which new events are
12701303
added before a file is closed and a new file is
1271-
created.
1304+
created. Values within the range of 15-60
1305+
seconds are allowed.
12721306
avro_file_format (google.cloud.datastream_v1.types.AvroFileFormat):
12731307
AVRO file format configuration.
12741308
@@ -1307,7 +1341,7 @@ class GcsDestinationConfig(proto.Message):
13071341

13081342

13091343
class BigQueryDestinationConfig(proto.Message):
1310-
r"""
1344+
r"""BigQuery destination configuration
13111345
13121346
This message has `oneof`_ fields (mutually exclusive fields).
13131347
For each oneof, at most one member field can be set at the same time.
@@ -1340,7 +1374,7 @@ class SingleTargetDataset(proto.Message):
13401374
13411375
Attributes:
13421376
dataset_id (str):
1343-
1377+
The dataset ID of the target dataset.
13441378
"""
13451379

13461380
dataset_id: str = proto.Field(
@@ -1354,7 +1388,8 @@ class SourceHierarchyDatasets(proto.Message):
13541388
13551389
Attributes:
13561390
dataset_template (google.cloud.datastream_v1.types.BigQueryDestinationConfig.SourceHierarchyDatasets.DatasetTemplate):
1357-
1391+
The dataset template to use for dynamic
1392+
dataset creation.
13581393
"""
13591394

13601395
class DatasetTemplate(proto.Message):

‎samples/generated_samples/snippet_metadata_google.cloud.datastream.v1.json

Copy file name to clipboardExpand all lines: samples/generated_samples/snippet_metadata_google.cloud.datastream.v1.json
+1-1Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@
88
],
99
"language": "PYTHON",
1010
"name": "google-cloud-datastream",
11-
"version": "1.5.1"
11+
"version": "0.1.0"
1212
},
1313
"snippets": [
1414
{

‎samples/generated_samples/snippet_metadata_google.cloud.datastream.v1alpha1.json

Copy file name to clipboardExpand all lines: samples/generated_samples/snippet_metadata_google.cloud.datastream.v1alpha1.json
+1-1Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@
88
],
99
"language": "PYTHON",
1010
"name": "google-cloud-datastream",
11-
"version": "1.5.1"
11+
"version": "0.1.0"
1212
},
1313
"snippets": [
1414
{

‎scripts/fixup_datastream_v1_keywords.py

Copy file name to clipboardExpand all lines: scripts/fixup_datastream_v1_keywords.py
+1-1Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,7 @@ class datastreamCallTransformer(cst.CSTTransformer):
4040
CTRL_PARAMS: Tuple[str] = ('retry', 'timeout', 'metadata')
4141
METHOD_TO_PARAMS: Dict[str, Tuple[str]] = {
4242
'create_connection_profile': ('parent', 'connection_profile_id', 'connection_profile', 'request_id', 'validate_only', 'force', ),
43-
'create_private_connection': ('parent', 'private_connection_id', 'private_connection', 'request_id', ),
43+
'create_private_connection': ('parent', 'private_connection_id', 'private_connection', 'request_id', 'force', ),
4444
'create_route': ('parent', 'route_id', 'route', 'request_id', ),
4545
'create_stream': ('parent', 'stream_id', 'stream', 'request_id', 'validate_only', 'force', ),
4646
'delete_connection_profile': ('name', 'request_id', ),

‎tests/unit/gapic/datastream_v1/test_datastream.py

Copy file name to clipboardExpand all lines: tests/unit/gapic/datastream_v1/test_datastream.py
+14Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10421,6 +10421,7 @@ def test_create_stream_rest(request_type):
1042110421
},
1042210422
"exclude_objects": {},
1042310423
"max_concurrent_cdc_tasks": 2550,
10424+
"max_concurrent_backfill_tasks": 3076,
1042410425
"drop_large_objects": {},
1042510426
"stream_large_objects": {},
1042610427
},
@@ -10450,6 +10451,7 @@ def test_create_stream_rest(request_type):
1045010451
},
1045110452
"exclude_objects": {},
1045210453
"max_concurrent_cdc_tasks": 2550,
10454+
"max_concurrent_backfill_tasks": 3076,
1045310455
},
1045410456
"postgresql_source_config": {
1045510457
"include_objects": {
@@ -10479,6 +10481,7 @@ def test_create_stream_rest(request_type):
1047910481
"exclude_objects": {},
1048010482
"replication_slot": "replication_slot_value",
1048110483
"publication": "publication_value",
10484+
"max_concurrent_backfill_tasks": 3076,
1048210485
},
1048310486
},
1048410487
"destination_config": {
@@ -10766,6 +10769,7 @@ def test_create_stream_rest_bad_request(
1076610769
},
1076710770
"exclude_objects": {},
1076810771
"max_concurrent_cdc_tasks": 2550,
10772+
"max_concurrent_backfill_tasks": 3076,
1076910773
"drop_large_objects": {},
1077010774
"stream_large_objects": {},
1077110775
},
@@ -10795,6 +10799,7 @@ def test_create_stream_rest_bad_request(
1079510799
},
1079610800
"exclude_objects": {},
1079710801
"max_concurrent_cdc_tasks": 2550,
10802+
"max_concurrent_backfill_tasks": 3076,
1079810803
},
1079910804
"postgresql_source_config": {
1080010805
"include_objects": {
@@ -10824,6 +10829,7 @@ def test_create_stream_rest_bad_request(
1082410829
"exclude_objects": {},
1082510830
"replication_slot": "replication_slot_value",
1082610831
"publication": "publication_value",
10832+
"max_concurrent_backfill_tasks": 3076,
1082710833
},
1082810834
},
1082910835
"destination_config": {
@@ -10996,6 +11002,7 @@ def test_update_stream_rest(request_type):
1099611002
},
1099711003
"exclude_objects": {},
1099811004
"max_concurrent_cdc_tasks": 2550,
11005+
"max_concurrent_backfill_tasks": 3076,
1099911006
"drop_large_objects": {},
1100011007
"stream_large_objects": {},
1100111008
},
@@ -11025,6 +11032,7 @@ def test_update_stream_rest(request_type):
1102511032
},
1102611033
"exclude_objects": {},
1102711034
"max_concurrent_cdc_tasks": 2550,
11035+
"max_concurrent_backfill_tasks": 3076,
1102811036
},
1102911037
"postgresql_source_config": {
1103011038
"include_objects": {
@@ -11054,6 +11062,7 @@ def test_update_stream_rest(request_type):
1105411062
"exclude_objects": {},
1105511063
"replication_slot": "replication_slot_value",
1105611064
"publication": "publication_value",
11065+
"max_concurrent_backfill_tasks": 3076,
1105711066
},
1105811067
},
1105911068
"destination_config": {
@@ -11319,6 +11328,7 @@ def test_update_stream_rest_bad_request(
1131911328
},
1132011329
"exclude_objects": {},
1132111330
"max_concurrent_cdc_tasks": 2550,
11331+
"max_concurrent_backfill_tasks": 3076,
1132211332
"drop_large_objects": {},
1132311333
"stream_large_objects": {},
1132411334
},
@@ -11348,6 +11358,7 @@ def test_update_stream_rest_bad_request(
1134811358
},
1134911359
"exclude_objects": {},
1135011360
"max_concurrent_cdc_tasks": 2550,
11361+
"max_concurrent_backfill_tasks": 3076,
1135111362
},
1135211363
"postgresql_source_config": {
1135311364
"include_objects": {
@@ -11377,6 +11388,7 @@ def test_update_stream_rest_bad_request(
1137711388
"exclude_objects": {},
1137811389
"replication_slot": "replication_slot_value",
1137911390
"publication": "publication_value",
11391+
"max_concurrent_backfill_tasks": 3076,
1138011392
},
1138111393
},
1138211394
"destination_config": {
@@ -13579,6 +13591,7 @@ def test_create_private_connection_rest_required_fields(
1357913591
# Check that path parameters and body parameters are not mixing in.
1358013592
assert not set(unset_fields) - set(
1358113593
(
13594+
"force",
1358213595
"private_connection_id",
1358313596
"request_id",
1358413597
)
@@ -13645,6 +13658,7 @@ def test_create_private_connection_rest_unset_required_fields():
1364513658
assert set(unset_fields) == (
1364613659
set(
1364713660
(
13661+
"force",
1364813662
"privateConnectionId",
1364913663
"requestId",
1365013664
)

0 commit comments

Comments
0 (0)
Morty Proxy This is a proxified and sanitized view of the page, visit original site.