diff --git a/CHANGELOG.md b/CHANGELOG.md index a612ddb0498c9..1e98b1bae1632 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,6 @@ # LocalStack Change Log +* v0.11.4: Add initial support for ACM API; build and push localstack-full image in CI, make light image the default; support client context passed to Node.js Lambda handler; increase max threads in thread pool to improve concurrency; support custom Lambda integration in API GW; fix HOSTNAME_EXTERNAL extraction to determine path-style addressing in S3; support "sms" as SNS subscriber protocol; support platform endpoints in SNS; fix RedrivePolicy for new SQS backend; fix triple logs in debug mode; fix format of Kinesis stream ARNs in CF; enhance implementation and tests for API GW HTTP/HTTP_PROXY support; support CF deletion of Lambda functions and IAM roles with inline policies; fix DynamoDB TransactWriteItems with ConditionCheck; add retries to fix intermittent SSL cert generation errors; fix DynamoDB stream MODIFY/INSERT event on UpdateItem for new item; fix DynamoDbStreams get-shardId-iterator with AT_SEQUENCE_NUMBER; fix proxy CORS headers; properly convert between Kinesis stream names and ARNs; fix generation of random password in SSM; add dummy AWS credentials to Lambda Docker containers; add test for VPN Gateway attachment.vpc-id filter; fix API key checks for API GW paths with path params; fix content-type in presigned S3 URLs; add support for pathPrefix in IAM list_users; fix SSL context creation error for multiple threads; enhance command line ports parsing; enhance CF support for ApiGateway::RestApi; add option to use default Lambda endpoint for StepFunctions; add tests to tag/untag SQS queues; add basic string functions in VLT templates; add tests for SQS FIFO queue with group/deduplication ID on multiple messages; fix invocation of destination Lambda for Logs subscription filters; add config option for DynamoDB Java heap size; add test for SQS DLQ when maxReceiveCount is reached; add missing region to EC2 getAccepterVpcInfo response; return raw response in edge proxy to fix gzip encoded S3 content; add test to trigger multiple SNS subscriptions in the presence of handler errors; fix adding multiple permissions to single Lambda function; return table name for DynamoDB table Ref in CF; make DDB stream creation asynchronous; replace ${filename} variable references in S3 presigned responses; fix HTML escaping of SQS message attributes; increase Quart server max content length; add SQS as target in EventBridge; fix shardId format for DDB Streams; add support for DynamoDB Global Tables; fix S3 Content-MD5 base64 checks; extend Java Lambda classpath; fix Docker port mapping for PORT_WEB_UI * v0.11.3: Switch to Quart Server for initial version of HTTP/2 support for API endpoints; move /health check endpoint to edge port; use moto instead of ElasticMQ as default backend for SQS; add thread synchronization to Lambda config initialization; point Lambda LOCALSTACK_HOSTNAME to main container directly; add CORS headers to all OPTIONS responses by default; upgrade DynamoDB to latest backend version; fix CF deploy issue with SNS resources, IAM role policy document; fix direct access to queue URLs with new SQS backend; encode account ID in SQS queue URLs; add handling of query parameters in SNS requests; use JSON safe dumping of bytes in API Gateway; determine proper external port in SQS URLs; fix handling of S3 LocationConstraint on bucket creation; add missing Record values to Kinesis stream listener response; fix DynamoDB DeleteEventSourceMappings in case tableARN is None; fix format of S3 InvalidRange error response; support CF condition based on AccountId; fix error responses for S3 Content-MD5 header; fix S3 bucket creation date for Java SDK v2; delete event source mappings on deletion of DynamoDB table; return proper response details on Lambda invocation error; store SSL certs to random tmp files in case of permission issues; enhance path matching for API Gateway invocations; fix edge mapping for CloudWatch, S3 HEAD requests, SQS requests; fix DynamoDB shard IDs; fix lower case for SQS->Lambda message attributes; pass HTTP_PROXY to Lambda if configured; skip Lambda invocation if event source mapping is in state Disabled; fix S3 CORS policy with single origin; fix S3 storage class on multipart upload; fix passed parameters on API Gateway proxy resource method call; cleanup asyncio thread pool on shutdown; fix S3 timestamp format; support expiry in S3 multipart POST; support Elasticsearch snapshot repo; fix binary MessageAttribute for SNS->SQS; make sync/async Lambda event sources configurable; Fix chunked encoding in S3 responses; add Firehose support for EventBridge targets; pass multivalue headers from API GW to Lambda; add persistence support for SNS; add API Key validation in API GW; fix handling of S3 multipart form upload via edge port; fix API GW response on DELETE Lambda integration; support StepFunctions targets in scheduled event rules; filter out Lambdas by region in ListFunctions; fix Terraform deletion of EC2 NAT Gateways; add support for IAM SimulatePrincipalPolicy; remove Python 2 tests in CI; add EventBridge schedule expression for HTTP subscribers; fix Windows host path for Lambda volume mounting; add UnsubscribeURL to SNS notification message * v0.11.2: Use dynamic instead of hardcoded backend service ports; use random port for multiserver; add API Gateway integration with DynamoDB; add Lambda support for the dotnetcore3.1 runtime; handle kms_key_arn/vpc_config in Lambda models; add bucket existence checks for S3 operations; refactor CLI and infra startup; fix mapping of CloudFormation account IDs; add persistence support to SQS API; remove obsolete subprocess32 dependency; optimize travis build for light image; add name for non-existent Lambda in calls to LAMBDA_FALLBACK_URL; add tests for CW getMetricData, API GW CreateModels, S3 listing 1000+ objects; add tests for EC2 create_vpc_endpoint, S3 put/get bucket versioning; add integration test for CloudWatch put_metric data; add test case for SQS queue RedrivePolicy attribute; fix health check for HTTPS endpoints with self-signed certs; mock simple response for EC2 reserved instance APIs; fix error handling on SQS sending of empty message batch * v0.11.1: Add support for Lambdas triggered by scheduled event rules; lazily install and allow selection of specific Elasticsearch versions; add ability to return multi valued headers from Lambda - API GW integration; fix CF deployment for EventPattern attribute in Events::Rule; add CF support to update instances of IAM::Role; implement Events put_events with SQS and Lambda targets; delete notification configurations on S3 bucket deletion; fix default RoleName in CF dependent IAM roles; support S3 expiry for pre-signed URLs and objects; add metadata headers in S3 GET responses; fix edge request forwarding for public S3 URLs; add missing S3 response headers for bytes ranges and request IDs; add util class to fix account IDs in STS listener; support "exists" operator for SNS filter policy; fix IAM detach_role_policy and add missing managed policies; fix tagging support for ES domains; print version number when starting up; fix response content for IAM error messages; update startup log messages with new edge port; fix requesting SSM params with leading slashes; fix resolving CloudFormation refs when updating resource IDs; add support for tags on IAM CreateUser calls; fix data type conversions in SNS->SQS message attributes; fix API Gateway put_integration to avoid responseTemplates containing null values; add test cases for STS get_federation_token; fix Docker port mapping conflicts by avoiding overlaps; add CF test for empty event rules names; delete SFN state machines via CF; consider None data when stripping chunk signatures in S3 requests; fix DynamoDB behavior for ReturnValues on PutItem; support custom Lambda endpoint for StepFunctions; add simple integration test for Serverless deployment; check BatchSize in Lambda event source mappings; add CFN support for Events::Rule, S3::BucketPolicy, Logs::LogGroup, and ElasticSearch/SecretsManager/KinesisFirehose resources; fix case sensitivity for Connection keep-alive header; put util jars at end of classpath for local Java Lambdas; add persistence for SSM and SecretsManager, introduce PersistingProxyListener; skip persistence replay if an API is disabled; optimize image size; create stripped-down light version of Docker image with lazy loading of ES libs; add documentation for USE_LIGHT_IMAGE; add checks for HOST_TMP_FOLDER when running in Docker diff --git a/localstack/constants.py b/localstack/constants.py index 0b9b8aef7802b..67c7b41c2db77 100644 --- a/localstack/constants.py +++ b/localstack/constants.py @@ -2,7 +2,7 @@ import localstack_client.config # LocalStack version -VERSION = '0.11.3' +VERSION = '0.11.4' # constant to represent the "local" region, i.e., local machine REGION_LOCAL = 'local' diff --git a/localstack/services/kinesis/kinesis_listener.py b/localstack/services/kinesis/kinesis_listener.py index 12275d1ea87f6..1557e79bd2378 100644 --- a/localstack/services/kinesis/kinesis_listener.py +++ b/localstack/services/kinesis/kinesis_listener.py @@ -2,7 +2,7 @@ import random from requests.models import Response from localstack import config -from localstack.utils.common import to_str, json_safe, clone, timestamp_millis +from localstack.utils.common import to_str, json_safe, clone, timestamp_millis, now_utc from localstack.utils.analytics import event_publisher from localstack.services.awslambda import lambda_api from localstack.services.generic_proxy import ProxyListener @@ -31,7 +31,7 @@ def forward_request(self, method, path, data, headers): consumer = clone(data) consumer['ConsumerStatus'] = 'ACTIVE' consumer['ConsumerARN'] = '%s/consumer/%s' % (data['StreamARN'], data['ConsumerName']) - consumer['ConsumerCreationTimestamp'] = timestamp_millis() + consumer['ConsumerCreationTimestamp'] = float(now_utc()) consumer = json_safe(consumer) STREAM_CONSUMERS.append(consumer) return {'Consumer': consumer} @@ -52,10 +52,11 @@ def consumer_matches(c): elif action == '%s.DescribeStreamConsumer' % ACTION_PREFIX: consumer_arn = data.get('ConsumerARN') or data['ConsumerName'] consumer_name = data.get('ConsumerName') or data['ConsumerARN'] + creation_timestamp = data.get('ConsumerCreationTimestamp') result = { 'ConsumerDescription': { 'ConsumerARN': consumer_arn, - # 'ConsumerCreationTimestamp': number, + 'ConsumerCreationTimestamp': creation_timestamp, 'ConsumerName': consumer_name, 'ConsumerStatus': 'ACTIVE', 'StreamARN': data.get('StreamARN') diff --git a/localstack/utils/cloudformation/template_deployer.py b/localstack/utils/cloudformation/template_deployer.py index 41de2ede34cbc..b1491700ce407 100644 --- a/localstack/utils/cloudformation/template_deployer.py +++ b/localstack/utils/cloudformation/template_deployer.py @@ -220,6 +220,28 @@ def replace(params, **kwargs): return replace +def get_ddb_provisioned_throughput(params, **kwargs): + args = params.get('ProvisionedThroughput') + if args: + if isinstance(args['ReadCapacityUnits'], str): + args['ReadCapacityUnits'] = int(args['ReadCapacityUnits']) + if isinstance(args['WriteCapacityUnits'], str): + args['WriteCapacityUnits'] = int(args['WriteCapacityUnits']) + return args + + +def get_ddb_global_sec_indexes(params, **kwargs): + args = params.get('GlobalSecondaryIndexes') + if args: + for index in args: + provisoned_throughput = index['ProvisionedThroughput'] + if isinstance(provisoned_throughput['ReadCapacityUnits'], str): + provisoned_throughput['ReadCapacityUnits'] = int(provisoned_throughput['ReadCapacityUnits']) + if isinstance(provisoned_throughput['WriteCapacityUnits'], str): + provisoned_throughput['WriteCapacityUnits'] = int(provisoned_throughput['WriteCapacityUnits']) + return args + + # maps resource types to functions and parameters for creation RESOURCE_TO_FUNCTION = { 'S3::Bucket': { @@ -406,9 +428,9 @@ def replace(params, **kwargs): 'TableName': 'TableName', 'AttributeDefinitions': 'AttributeDefinitions', 'KeySchema': 'KeySchema', - 'ProvisionedThroughput': 'ProvisionedThroughput', + 'ProvisionedThroughput': get_ddb_provisioned_throughput, 'LocalSecondaryIndexes': 'LocalSecondaryIndexes', - 'GlobalSecondaryIndexes': 'GlobalSecondaryIndexes', + 'GlobalSecondaryIndexes': get_ddb_global_sec_indexes, 'StreamSpecification': lambda params, **kwargs: ( common.merge_dicts(params.get('StreamSpecification'), {'StreamEnabled': True}, default=None)) }, diff --git a/tests/integration/test_cloudformation.py b/tests/integration/test_cloudformation.py index 0f9cdeafe0fa9..5a6944e1ed09b 100644 --- a/tests/integration/test_cloudformation.py +++ b/tests/integration/test_cloudformation.py @@ -557,8 +557,8 @@ def handler(event, context): - AttributeName: startTime KeyType: RANGE ProvisionedThroughput: - ReadCapacityUnits: 5 - WriteCapacityUnits: 5 + ReadCapacityUnits: '5' + WriteCapacityUnits: '5' StreamSpecification: StreamViewType: NEW_IMAGE GlobalSecondaryIndexes: @@ -571,8 +571,8 @@ def handler(event, context): Projection: ProjectionType: ALL ProvisionedThroughput: - ReadCapacityUnits: 5 - WriteCapacityUnits: 5 + ReadCapacityUnits: '5' + WriteCapacityUnits: '5' Outputs: Name: Value: @@ -1736,3 +1736,41 @@ def test_delete_stack_across_regions(self): ) cloudformation.delete_stack(StackName='myteststack') + + def test_globalindex_read_write_provisioned_throughput_dynamodb_table(self): + cf_client = aws_stack.connect_to_service('cloudformation') + ddb_client = aws_stack.connect_to_service('dynamodb') + stack_name = 'test_dynamodb' + + response = cf_client.create_stack( + StackName=stack_name, + TemplateBody=TEST_DEPLOY_BODY_3, + Parameters=[ + { + 'ParameterKey': 'tableName', + 'ParameterValue': 'dynamodb' + }, + { + 'ParameterKey': 'env', + 'ParameterValue': 'test' + } + ] + ) + self.assertEqual(response['ResponseMetadata']['HTTPStatusCode'], 200) + response = ddb_client.describe_table( + TableName='dynamodb-test' + ) + + if response['Table']['ProvisionedThroughput']: + throughput = response['Table']['ProvisionedThroughput'] + self.assertTrue(isinstance(throughput['ReadCapacityUnits'], int)) + self.assertTrue(isinstance(throughput['WriteCapacityUnits'], int)) + + for global_index in response['Table']['GlobalSecondaryIndexes']: + index_provisioned = global_index['ProvisionedThroughput'] + test_read_capacity = index_provisioned['ReadCapacityUnits'] + test_write_capacity = index_provisioned['WriteCapacityUnits'] + + self.assertTrue(isinstance(test_read_capacity, int)) + self.assertTrue(isinstance(test_write_capacity, int)) + cf_client.delete_stack(StackName=stack_name) diff --git a/tests/integration/test_kinesis.py b/tests/integration/test_kinesis.py index 42f056549c28e..0026385eb6e58 100644 --- a/tests/integration/test_kinesis.py +++ b/tests/integration/test_kinesis.py @@ -1,3 +1,4 @@ +from datetime import datetime import logging import unittest from localstack.utils.aws import aws_stack @@ -23,10 +24,14 @@ def assert_consumers(count): # create consumer and assert 1 consumer consumer_name = 'cons1' - client.register_stream_consumer(StreamARN=stream_arn, ConsumerName=consumer_name) + response = client.register_stream_consumer(StreamARN=stream_arn, ConsumerName=consumer_name) + self.assertEqual(response['Consumer']['ConsumerName'], consumer_name) + # boto3 converts the timestamp to datetime + self.assertTrue(isinstance(response['Consumer']['ConsumerCreationTimestamp'], datetime)) consumers = assert_consumers(1) self.assertEqual(consumers[0]['ConsumerName'], consumer_name) self.assertIn('/%s' % consumer_name, consumers[0]['ConsumerARN']) + self.assertTrue(isinstance(consumers[0]['ConsumerCreationTimestamp'], datetime)) # delete non-existing consumer and assert 1 consumer client.deregister_stream_consumer(StreamARN=stream_arn, ConsumerName='_invalid_')