diff --git a/.gitignore b/.gitignore index d9c900a..dedc2f9 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,4 @@ *.pyc -**/vendor/* \ No newline at end of file +**/vendor/* +target +test-suite/**/*.jar diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index bde2411..606bf24 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -3,11 +3,16 @@ Contributing to Lambda ### Prerequisites -This has only been tested on OSX and Linux. +These workflows have been tested on Linux and Mac. You must have a: -* Working Go 1.5 onwards installation. -* Working [Docker](http://docker.com) installation. -* GNU Make. +- Working `make` command +- Working [Go][go] >=1.5 installation, +- Working [Glide][glide] installation, +- Working [Docker][docker] installation. + +[go]: http://golang.org +[glide]: http://glide.sh +[docker]: http://www.docker.com To work on Java code, you'll need a working JDK and [Apache Maven](http://maven.apache.org). For node.js, any version of node >=0.10.0 will do. @@ -71,12 +76,17 @@ ironcli is built from source. To make sure you can hack on the `lambda` package within the vendoring workflow, you should do something like this. * Clone ironcli. -* Fork this repository. -* Edit the glide.yaml file to change the `lambda` package import path from - `github.com/iron-io/lambda/lambda` to `github.com//lambda/lambda`. - You'll also need to fix the imports in the ironcli lambda.go file. -* Run `glide i` to get the new package. -* Now `ironcli/vendor/github.com//lambda` will have your fork. Hack - in here and submit a PR. -* If you have a better idea of a workflow, we would appreciate leaving us - a note. +* Run `glide i` to install dependencies. +* Now `ironcli/vendor/github.com/iron-io/lambda` will have a clone of the + master branch of lambda. +* Create a new branch, hack hack hack. +* Make sure relevant code compiles and ironcli builds. +* Fork the lambda repository on Github. +* Add your fork as a remote: `git remote add git@github.com:/lambda` +* Push your branch to your fork: `git push ` +* Submit a PR. + +### Improving Lambda Docker images + +Simply clone this repository, make changes to the Dockerfile for individual +images in the `images/` directories and submit a Pull Request. diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..497dc87 --- /dev/null +++ b/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2016 Iron.io Inc. All rights reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/docs/aws.md b/docs/aws.md index 2e2f6d9..0da9f92 100644 --- a/docs/aws.md +++ b/docs/aws.md @@ -1,4 +1,10 @@ -## Using the AWS SDK from Lambda functions. +Interacting with AWS Services +============================= + +The node.js and Python stacks include SDKs to interact with other AWS services. +For Java you will need to include any such SDK in the JAR file. + +## Credentials Running Lambda functions outside of AWS means that we cannot automatically get access to other AWS resources based on Lambda subsuming the execution role @@ -8,10 +14,14 @@ these credentials explicitly. ### Using environment variables for the credentials -The easiest way to do this is to register the `AWS_ACCESS_KEY_ID` and -`AWS_SECRET_ACCESS_KEY` environment variables with IronWorker when registering -the Docker image. Then you may use the Environment Credentials loader provided -by the AWS SDK in various languages. Consider this node.js example: +The easiest way to do this is to pass the `AWS_ACCESS_KEY_ID` and +`AWS_SECRET_ACCESS_KEY` environment variables to Docker when you run the image. + +```sh +docker run -e AWS_ACCESS_KEY_ID= -e AWS_SECRET_ACCESS_KEY= +``` + +The various AWS SDKs will automatically pick these up. var AWS = require('aws-sdk'); AWS.config.region = 'us-west-2'; @@ -19,26 +29,315 @@ by the AWS SDK in various languages. Consider this node.js example: exports.run = function(event, context) { var s3bucket = new AWS.S3({ params: {Bucket: event.bucket}, - credentials: new AWS.EnvironmentCredentials('AWS') }); s3bucket.createBucket(function() { // Act on bucket here. }); } -We pass the S3 object a credentials created from the environment variables. +### Credentials on IronWorker Assuming you [packaged this function](./introduction.md) into a Docker image `iron/s3-write` and pushed it to Docker Hub. Instead of just registering with IronWorker as: - ironcli register iron/s3-write + iron register /s3-write do this instead: - ironcli register -e AWS_ACCESS_KEY_ID= \ - -e AWS_SECRET_ACCESS_KEY= \ - iron/s3-write + iron register -e AWS_ACCESS_KEY_ID= \ + -e AWS_SECRET_ACCESS_KEY= \ + /s3-write + +Alternatively, if you use `iron publish-function`, it will automatically +pick up the environment variables and forward them if valid ones are found. + +```sh +export AWS_ACCESS_KEY_ID= +export AWS_SECRET_ACCESS_KEY= +iron publish-function -function-name /s3-write +``` + +If you have an existing image with the same name registered with IronWorker, +the environment variables will not simply be updated. You need to first delete +the code from HUD and then publish the function again. This will unfortunately +result in a new webhook URL for the function. + +## Example: Using Lambda with IronWorker and Amazon Simple Notification Service + +Lambda's premise of server-less computing requires a few infrastructural pieces +other than just the Docker image. First there needs to be a platform that can +run these Docker images on demand. Second, we need some way to invoke the +Lambda function based on an external event. + +In this example, we will look at how to use IronWorker and Amazon Simple +Notification Service (SNS) to create a function that can search a given URL for +a user-specified keyword. You can build upon this, coupled with some storage +provider (like Amazon S3) to build a simple search engine. + +The concepts introduced here can be used with any infrastructure that let's you +start a Docker container on some event. It is not tied to IronWorker. + +The code for this example is located [here](../examples/sns/sns.js). + +### Setup + +Make sure you have an [IronWorker](https://www.iron.io/platform/ironworker/) +account. You can make one [here](https://www.iron.io/get-started/). You will +need a [Docker Hub](https://hub.docker.com) account to publish the Lambda function. + +Also set up an [AWS +account](http://docs.aws.amazon.com/sns/latest/dg/SNSBeforeYouBegin.html) and [create a SNS topic](http://docs.aws.amazon.com/sns/latest/dg/CreateTopic.html). Call this topic `sns-example`. Carefully note the region the topic was created in. The region is found in the topic ARN. + +You will also need credentials to use the AWS SDK. These credentials can be +obtained from the IAM and are in the form of an Access Key and Secret. See the +[AWS page](./aws.md) page for more information. + +### Function outline + +SNS can notify a variety of endpoints when a message is published to the topic. +One of these is an HTTPS URL. IronWorker provides an HTTPS URL that runs an +instance of the Docker image when the URL receives a POST. + +In this example, we will manually publish messages to SNS, which will trigger +the webhook, our Lambda function will fetch the URL passed in the message, +search for the keyword in the response, and print out the count. + +Here is the beginning of the function: + +```js +var http = require('http'); +var AWS = require('aws-sdk'); +AWS.config.region = 'us-west-1'; + +function searchString(context, text, key) { + // Global and Ignore case flags. + var regex = new RegExp(key, 'gi'); + + var results = []; + var m; + while ((m = regex.exec(text)) != null) { + results.push(m); + } + + console.log("Found", results.length, "instances of", key); + context.succeed(); +} + +function searchBody(context, res, key) { + if (res.statusCode === 200) { + var body = ""; + res.on('data', function(chunk) { body += chunk.toString(); }); + res.on('end', function() { searchString(context, body, key); }); + } else { + context.fail("Non-200 status code " + res.statusCode + " fetching '" + message.url + "'. Aborting."); + } +} +``` + +Here we set up the various functions that will implement our Lambda function's +logic. Set the AWS region to the region in the SNS topic ARN, otherwise our +function will fail. + +The `searchBody` function takes a node `http.ClientResponse` and gathers the +body data, then calls `searchString` to perform the regular expression match. +Finally each function invokes the `context.fail()` or `context.succeed()` +functions as appropriate. This is important, otherwise our function won't +terminate until it times out, even if execution was done. + +#### Handling SNS event types. + +SNS events send the payload as a JSON message to our webhook. These are passed +on to the Lambda function in the handler's `event` parameter. Each SNS message +contains a `Type` field. We are interested in two types - `Notification` and +`SubscriptionConfirmation`. The former is used to deliver published messages. + +Before SNS can start sending messages to the subscriber, the subscriber has to +confirm the subscription. This is to prevent abuse. The +`SubscriptionConfirmation` type is used for this. Our function will have to +deal with both. + +```js +exports.handler = function(event, context) { + if (event.Type == 'Notification') { + // ... + } + else if (event.Type == 'SubscriptionConfirmation') { + // ... + } else { + console.log("unknown event.Type", event.Type); + context.fail(); + } +}; +``` + +We can use the SDK to confirm the subscription. -Now, when you invoke the function, the AWS SDK will load the credentials from -the environment variables and your AWS API operations should work. +```js +var sns = new AWS.SNS(); +var params = { + Token: event.Token, + TopicArn: event.TopicArn, +}; +sns.confirmSubscription(params, function(err, data) { + if (err) { + console.log(err, err.stack); + context.fail(err); + } else { + console.log("Confirmed subscription", data); + console.log("Ready to process events."); + context.done(); + } +}); +``` + +The `Token` is unique and has to be sent to SNS to indicate that we are a valid +subscriber that the message was intended for. Once we confirm the subscription, +this run of the Lambda function is done and we can stop (`context.done()`). +SNS is now ready to run this Lambda function when we publish to the topic. + +Finally we come to the event type we expect to receive most often -- +`Notification`. In this case, we try to grab the url and keyword from the +message and run our earlier `searchBody()` function on it. + +```js +try { + var message = JSON.parse(event.Message); + if (typeof message.url == "string" && typeof message.keyword == "string") { + http.get(message.url, function(res) { searchBody(context, res, message.keyword); }) + .on('error', function(e) { + context.fail(e); + }); + } else { + context.fail("Invalid message " + event.Message); + } +} catch(e) { + context.fail(e); +} +``` + +### Trying it out + +With this function ready, we can Dockerize it and publish it to actually try it +out with SNS. + +```sh +iron lambda create-function -function-name /sns-example -runtime +nodejs -handler sns.handler sns.js +``` + +This will create a local docker image. The `publish-function` command will +upload this to Docker Hub and register it with IronWorker. + +To be able to use the AWS SDK, you'll also need to set two environment +variables. The values must be your AWS credentials. + +```sh +AWS_ACCESS_KEY_ID= +AWS_SECRET_ACCESS_KEY= + +iron publish-function -function-name /sns-example:latest +``` + +Visit the published function's code page in the [IronWorker control +panel](https://hud.iron.io). You should see a cloaked field called "Webhook +URL". Copy this URL. + +In the AWS SNS control panel, visit the `sns-example` topic. Click "Create +Subscription". Select the subscription type as HTTPS and paste the webhook URL. +Once you save this, the IronWorker task should have been launched and then +finished successfully with a "Confirmed subscription" message. + +Now you can click the blue "Publish to topic" button on the AWS SNS control +panel. Select the message format as JSON and the contents as (for example): + +```js +{ + "default": "{\"url\": \"http://www.econrates.com/reality/schul.html\", \"keyword\": \"blackbird\"}" +} +``` + +SNS will send the string in the `"default"` key to all subscribers. You should +be able to see that the IronWorker task has been run again. If everything went +well, it should have printed out a summary and exited successfully. + +That's it, a simple, notification based, Lambda function. + +## Example: Reading and writing to S3 Bucket + +This example demonstrates modifying S3 buckets and using the included +ImageMagick tools in a node.js function. Our function will fetch an image +stored in a key specified by the event, resize it to a width of 1024px and save +it to another key. + +The code for this example is located [here](../examples/s3/example.js). + +The event will look like: + +```js +{ + "bucket": "iron-lambda-demo-images", + "srcKey": "waterfall.jpg", + "dstKey": "waterfall-1024.jpg" +} +``` + +The setup, imports and SDK initialization. + +```js +var im = require('imagemagick'); +var fs = require('fs'); +var AWS = require('aws-sdk'); + +exports.run = function(event, context) { + var bucketName = event['bucket'] + var srcImageKey = event['srcKey'] + var dstImageKey = event['dstKey'] + + var s3 = new AWS.S3(); +} +``` + +First we retrieve the source and write it to a local file so ImageMagick can +work with it. + +```js +s3.getObject({ + Bucket: bucketName, + Key: srcImageKey + }, function (err, data) { + + if (err) throw err; + + var fileSrc = '/tmp/image-src.dat'; + var fileDst = '/tmp/image-dst.dat' + fs.writeFileSync(fileSrc, data.Body) + +}); +``` + +The actual resizing involves using the identify function to get the current +size (we only resize if the image is wider than 1024px), then doing the actual +conversion to `fileDst`. Finally we upload to S3. + +```js +im.identify(fileSrc, function(err, features) { + resizeIfRequired(err, features, fileSrc, fileDst, function(err, resized) { + if (err) throw err; + if (resized) { + s3.putObject({ + Bucket:bucketName, + Key: dstImageKey, + Body: fs.createReadStream(fileDst), + ContentType: 'image/jpeg', + ACL: 'public-read', + }, function (err, data) { + if (err) throw err; + context.done() + }); + } else { + context.done(); + } + }); +}); +``` diff --git a/docs/docker.md b/docs/docker.md new file mode 100644 index 0000000..b6346ce --- /dev/null +++ b/docs/docker.md @@ -0,0 +1,59 @@ +# Running the Lambda Docker images + +Docker images created by running the `create-function` subcommand on a Lambda +function are ready to execute. They do not need any command line arguments. + +The `test-function` subcommand can pass the correct parameters to `docker run` +to run those images with the payload and environment variables set up +correctly. If you would like more control, like mounting volumes, or adding +more environment variables this guide describes how to directly run these +images using `docker run`. + +## Payload + +The `payload` argument is written to a random, opaque directory on the host. +The file itself is called `payload.json`. This directory is mapped to the +`/mnt` volume in the container, so that the payload is available in +`/mnt/payload.json`. This is not REQUIRED, since the actual runtimes use the +`PAYLOAD_FILE` environment variable to discover the payload location. + +## Environment variables + +The `TASK_ID` variable maps to the AWS Request ID. This should be set to +something unique (a UUID, or an incrementing number). + +`test-function` runs a container with 300MB memory allocated to it. This same +information is available inside the container in the `TASK_MAXRAM` variable. +This value can be a number in bytes, or a number suffixed by `b`, `k`, `m`, `g` +for bytes, kilobytes, megabytes and gigabytes respectively. These are +case-insensitive. + +The following variables are set for AWS compatibility: +* `AWS_LAMBDA_FUNCTION_NAME` - The name of the docker image. +* `AWS_LAMBDA_FUNCTION_VERSION` - The default is `$LATEST`, but any string is + allowed. +* `AWS_ACCESS_KEY_ID` - Set this to the Access Key to allow the Lambda function + to use AWS APIs. +* `AWS_SECRET_ACCESS_KEY` - Set this to the Secret Key to allow the Lambda + function to use AWS APIs. + +## Running the container + +The default `test-function` can then be approximated as the following `docker +run` command: + +```sh +mkdir /tmp/payload_dir +echo "" >> /tmp/payload_dir/my_payload.json +docker run -v /tmp/payload_dir:/mnt \ + -m 1G \ + -e PAYLOAD_FILE=/mnt/my_payload.json \ + -e TASK_ID=$RANDOM \ + -e TASK_MAXRAM=1G \ + -e AWS_LAMBDA_FUNCTION_NAME=user/fancyfunction \ + -e AWS_LAMBDA_FUNCTION_VERSION=1.0 \ + -e AWS_ACCESS_KEY_ID= \ + -e AWS_SECRET_ACCESS_KEY= \ + --rm -it + user/fancyfunction +``` diff --git a/docs/environment.md b/docs/environment.md index c552c4b..b9f47eb 100644 --- a/docs/environment.md +++ b/docs/environment.md @@ -89,6 +89,82 @@ a JSON object with trace information. The Java8 runtime is significantly lacking at this piont and we **do not recommend** using it. +### Handler types + +There are some restrictions on the handler types supported. + +#### Only a void return type is allowed + +Since Lambda does not support request/response invocation, we explicitly +prohibit a non-void return type on the handler. + +#### JSON parse error stack differences + +AWS uses the Jackson parser, this project uses the GSON parser. So JSON parse +errors will have different traces. + +#### Single item vs. List + +Given a list handler like: + +```java +public static void myHandler(List l) { + // ... +} +``` + +If the payload is a single number, AWS Lambda will succeed and pass the handler +a list with a single item. This project will raise an exception. + +#### Collections of POJOs + +This project cannot currently deserialize a List or Map containing POJOs. For +example: + +```java +public class Handler { + public static MyPOJO { + private String attribute; + public void setAttribute(String a) { + attribute = a; + } + + public String getAttribute() { + return attribute; + } + } + + public static void myHandler(List l) { + // ... + } +} +``` + +This handler invoked with the below event will fail! + +```js +[{ "attribute": "value 1"}, { "attribute": "value 2" }] +``` + +#### Leveraging predefined types is not supported + +Using the types in `aws-lambda-java-core` to [implement handlers][predef] is +untested and unsupported right now. While the package is available in your +function, we have not tried it out. + +[predef]: http://docs.aws.amazon.com/lambda/latest/dg/java-handler-using-predefined-interfaces.html + +### Logging + +The [log4j and LambdaLogger +styles](http://docs.aws.amazon.com/lambda/latest/dg/java-logging.html) that log +to CloudWatch are not supported. + ### Context object -TODO +* `context.getFunctionName()` returns a String of the form of a docker image, + for example `iron/test-function`. +* `context.getFunctionVersion()` is always the string `"$LATEST"`. +* `context.getAwsRequestId()` reflects the environment variable `TASK_ID` which is + set to the task ID on IronWorker. If TASK_ID is empty, a new UUID is used. +* `getInvokedFunctionArn()`, `getLogGroupName()`, `getLogStreamName()`, `getIdentity()`, `getClientContext()`, `getLogger()` return `null`. diff --git a/docs/getting-started.md b/docs/getting-started.md index 98f1432..6922250 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -4,26 +4,8 @@ This guide will walk you through creating and testing a simple Lambda function. We will then upload it to IronWorker and run it. -## Prerequisites - -These workflows have been tested on Linux and Mac. You must have a: - -- Working [Go][go] >=1.5 installation, -- Working [Glide][glide] installation, -- Working [Docker][docker] installation. - -[go]: http://golang.org -[glide]: http://glide.sh -[docker]: http://www.docker.com - -We are going to use a development branch of `ironcli` instead of the official -release. TODO: `go install` will fail due to (lack of) vendoring. - - $ cd $GOPATH - $ go get github.com/iron-io/ironcli - $ cd src/github.com/iron-io/ironcli - $ git checkout -t origin/lambda - $ go install . +We need the the `ironcli` tool for the rest of this guide. You can install it +by following [these instructions](https://github.com/iron-io/ironcli). ## Creating the function @@ -53,7 +35,7 @@ Create an empty directory for your project and save this code in a file called Now let's use `ironcli`'s Lambda functionality to create a Docker image. We can then run the Docker image with a payload to execute the Lambda function. - $ $GOPATH/bin/ironcli lambda create-function --function-name irontest/node-exec:1 --runtime nodejs --handler node_exec.handler node_exec.js + $ iron lambda create-function --function-name irontest/node-exec:1 --runtime nodejs --handler node_exec.handler node_exec.js Image output Step 1 : FROM iron/lambda-nodejs ---> 66fb7af42230 Step 2 : ADD node_exec.js ./node_exec.js @@ -67,8 +49,8 @@ then run the Docker image with a payload to execute the Lambda function. As you can see, this is very similar to creating a Lambda function using the `aws` CLI tool. We name the function as we would name other Docker images. The -`1` indicates the version. You can use any string. This way you can make -changes to your code and tell IronWorker to run the newer code. The handler is +`1` indicates the version. You can use any string. This way you can configure +your deployment environment to use different versions. The handler is the name of the function to run, in the form that nodejs expects (`module.function`). Where you would package the files into a `.zip` to upload to Lambda, we just pass the list of files to `ironcli`. If you had node @@ -86,7 +68,7 @@ You should now see the generated Docker image. The `test-function` subcommand can launch the Dockerized function with the right parameters. - $ $GOPATH/bin/ironcli lambda test-function --function-name irontest/node-exec:1 --payload '{ "cmd": "echo Dockerized Lambda" }' + $ iron lambda test-function --function-name irontest/node-exec:1 --payload '{ "cmd": "echo Dockerized Lambda" }' Dockerized Lambda! You should see the output. Try changing the command to `date` or something more @@ -114,7 +96,7 @@ The `publish-function` command first uploads the image to Docker Hub, then registers it with IronWorker so you can queue tasks or launch a task in response to a webhook. - $ $GOPATH/bin/ironcli lambda publish-function --function-name irontest/node-exec:1 + $ iron lambda publish-function --function-name irontest/node-exec:1 -----> Configuring client Project '' with id='' -----> Registering worker 'irontest/node-exec' @@ -128,7 +110,7 @@ page. We can now run this from the command line. - $ $GOPATH/bin/ironcli worker queue -payload '{ "cmd": "echo Dockerized Lambda" }' -wait irontest/node-exec + $ iron worker queue -payload '{ "cmd": "echo Dockerized Lambda" }' -wait irontest/node-exec -----> Configuring client Project '' with id='' -----> Queueing task 'irontest/node-exec' @@ -148,8 +130,3 @@ Code page. $ curl -X POST -d '{ "cmd": "echo Dockerized Lambda" }' '' ---- - -TODO: -NOTE to devs: Perhaps prefetch Lambda function runners on to certain machines? - diff --git a/docs/import.md b/docs/import.md new file mode 100644 index 0000000..2ed9caa --- /dev/null +++ b/docs/import.md @@ -0,0 +1,42 @@ +Import existing AWS Lambda functions +==================================== + +The [ironcli](https://github.com/iron-io/ironcli/) tool includes a set of +commands to act on Lambda functions. Most of these are described in +[getting-started](./getting-started.md). One more subcommand is `aws-import`. + +If you have an existing AWS Lambda function, you can use this command to +automatically convert it to a Docker image that is ready to be deployed on +other platforms. + +### Credentials + +To use this, either have your AWS access key and secret key set in config +files, or in environment variables. In addition, you'll want to set a default +region. You can use the `aws` tool to set this up. Full instructions are in the +[AWS documentation][awscli]. + +[awscli]: http://docs.aws.amazon.com/cli/latest/userguide/cli-chap-getting-started.html#cli-config-files + +### Importing + +Assuming you have a lambda function named `my-function`, the following command: + +```sh +ironcli lambda aws-import my-function +``` + +will import the function code to a directory called `./my-function`. It will +then create a docker image called `my-function`. + +Using Lambda with Docker Hub and Iron Worker requires that the Docker image be +named `/`. This is used to uniquely identify +images on Docker Hub. Please use the `-image /` flag to `aws-import` to create a correctly named image. + +If you only want to download the code, pass the `-download-only` flag. The +`-region` and `-profile` flags are available similar to the `aws` tool to help +you tweak the settings on a command level. If you want to call the docker image +something other than `my-function`, pass the `-image ` flag. Finally, +you can import a different version of your lambda function than the latest one +by passing `-version .` diff --git a/docs/introduction.md b/docs/introduction.md index 0d7a906..cd1114f 100644 --- a/docs/introduction.md +++ b/docs/introduction.md @@ -22,8 +22,6 @@ easy to write containerized applications that will run anywhere without having to fiddle with Docker and get the various runtimes set up. Instead you can just write a simple function and have an "executable" ready to go. -TODO - ## How does it work? We provide base Docker images for the various runtimes that AWS Lambda @@ -38,18 +36,16 @@ The Docker container has to be run with a certain configuration, described ## Next steps -Write and package your Lambda functions with our [Getting started +Write, package and run your Lambda functions with our [Getting started guide](./getting-started.md). [Here is the environment](./environment.md) that Lambda provides. `ironcli lambda` lists the commands to work with Lambda functions locally. -There is a short guide to [using Lambda with IronWorker](./ironworker.md). -Non-AWS Lambda functions do have the disadvantage of not having deep -integration with other AWS services. Much of the push-based actions can be -solved by redirecting the event through [SNS and using webhooks](./sns.md). -AWS APIs are of course available for use through the AWS SDK available to the -function. We explain how to deal with authentication in [this guide](./aws.md). - -## Contributing +You can [import](./import.md) existing Lambda functions hosted on Amazon! +The Docker environment required to run Lambda functions is described +[here](./docker.md). -TODO +Non-AWS Lambda functions can continue to interact with AWS services. [Working +with AWS](./aws.md) describes how to access AWS credentials, interact with +services like S3 and how to launch a Lambda function due a notification from +SNS. diff --git a/docs/ironworker.md b/docs/ironworker.md deleted file mode 100644 index a62b424..0000000 --- a/docs/ironworker.md +++ /dev/null @@ -1,2 +0,0 @@ -Stuff related to IronWorker, like using versions, webhooks, uploading functions -etc. diff --git a/docs/sns.md b/docs/sns.md deleted file mode 100644 index 9196684..0000000 --- a/docs/sns.md +++ /dev/null @@ -1,204 +0,0 @@ -TODO: Should we add screenshots? - -Using Lambda with IronWorker and Amazon Simple Notification Service -=================================================================== - -Lambda's premise of server-less computing requires a few infrastructural pieces -other than just the Docker image. First there needs to be a platform that can -run these Docker images on demand. Second, we need some way to invoke the -Lambda function based on an external event. - -In this example, we will look at how to use IronWorker and Amazon Simple -Notification Service (SNS) to create a function that can search a given URL for -a user-specified keyword. You can build upon this, coupled with some storage -provider (like Amazon S3) to build a simple search engine. - -The concepts introduced here can be used with any infrastructure that let's you -start a Docker container on some event. It is not tied to IronWorker. - -The code for this example is located [here](../examples/sns/sns.js). - -## Setup - -Make sure you have an [IronWorker](https://www.iron.io/platform/ironworker/) -account. You can make one [here](https://www.iron.io/get-started/). You will -need a [Docker Hub](https://hub.docker.com) account to publish the Lambda function. - -Also set up an [AWS -account](http://docs.aws.amazon.com/sns/latest/dg/SNSBeforeYouBegin.html) and [create a SNS topic](http://docs.aws.amazon.com/sns/latest/dg/CreateTopic.html). Call this topic `sns-example`. Carefully note the region the topic was created in. The region is found in the topic ARN. - -You will also need credentials to use the AWS SDK. These credentials can be -obtained from the IAM and are in the form of an Access Key and Secret. See the -[AWS page](./aws.md) page for more information. - -## Function outline - -SNS can notify a variety of endpoints when a message is published to the topic. -One of these is an HTTPS URL. IronWorker provides an HTTPS URL that runs an -instance of the Docker image when the URL receives a POST. - -In this example, we will manually publish messages to SNS, which will trigger -the webhook, our Lambda function will fetch the URL passed in the message, -search for the keyword in the response, and print out the count. - -Here is the beginning of the function: - -```js -var http = require('http'); -var AWS = require('aws-sdk'); -AWS.config.region = 'us-west-1'; - -function searchString(context, text, key) { - // Global and Ignore case flags. - var regex = new RegExp(key, 'gi'); - - var results = []; - var m; - while ((m = regex.exec(text)) != null) { - results.push(m); - } - - console.log("Found", results.length, "instances of", key); - context.succeed(); -} - -function searchBody(context, res, key) { - if (res.statusCode === 200) { - var body = ""; - res.on('data', function(chunk) { body += chunk.toString(); }); - res.on('end', function() { searchString(context, body, key); }); - } else { - context.fail("Non-200 status code " + res.statusCode + " fetching '" + message.url + "'. Aborting."); - } -} -``` - -Here we set up the various functions that will implement our Lambda function's -logic. Set the AWS region to the region in the SNS topic ARN, otherwise our -function will fail. - -The `searchBody` function takes a node `http.ClientResponse` and gathers the -body data, then calls `searchString` to perform the regular expression match. -Finally each function invokes the `context.fail()` or `context.succeed()` -functions as appropriate. This is important, otherwise our function won't -terminate until it times out, even if execution was done. - -### Handling SNS event types. - -SNS events send the payload as a JSON message to our webhook. These are passed -on to the Lambda function in the handler's `event` parameter. Each SNS message -contains a `Type` field. We are interested in two types - `Notification` and -`SubscriptionConfirmation`. The former is used to deliver published messages. - -Before SNS can start sending messages to the subscriber, the subscriber has to -confirm the subscription. This is to prevent abuse. The -`SubscriptionConfirmation` type is used for this. Our function will have to -deal with both. - -```js -exports.handler = function(event, context) { - if (event.Type == 'Notification') { - // ... - } - else if (event.Type == 'SubscriptionConfirmation') { - // ... - } else { - console.log("unknown event.Type", event.Type); - context.fail(); - } -}; -``` - -We can use the SDK to confirm the subscription. - -```js -var sns = new AWS.SNS(); -var params = { - Token: event.Token, - TopicArn: event.TopicArn, -}; -sns.confirmSubscription(params, function(err, data) { - if (err) { - console.log(err, err.stack); - context.fail(err); - } else { - console.log("Confirmed subscription", data); - console.log("Ready to process events."); - context.done(); - } -}); -``` - -The `Token` is unique and has to be sent to SNS to indicate that we are a valid -subscriber that the message was intended for. Once we confirm the subscription, -this run of the Lambda function is done and we can stop (`context.done()`). -SNS is now ready to run this Lambda function when we publish to the topic. - -Finally we come to the event type we expect to receive most often -- -`Notification`. In this case, we try to grab the url and keyword from the -message and run our earlier `searchBody()` function on it. - -```js -try { - var message = JSON.parse(event.Message); - if (typeof message.url == "string" && typeof message.keyword == "string") { - http.get(message.url, function(res) { searchBody(context, res, message.keyword); }) - .on('error', function(e) { - context.fail(e); - }); - } else { - context.fail("Invalid message " + event.Message); - } -} catch(e) { - context.fail(e); -} -``` - -## Trying it out - -With this function ready, we can Dockerize it and publish it to actually try it -out with SNS. - -```sh -ironcli lambda create-function -function-name /sns-example -runtime -nodejs -handler sns.handler sns.js -``` - -This will create a local docker image. The `publish-function` command will -upload this to Docker Hub and register it with IronWorker. - -FIXME(nikhil): AWS credentials bit. - -To be able to use the AWS SDK, you'll also need to set two environment -variables. The values must be your AWS credentials. - -```sh -AWS_ACCESS_KEY_ID= -AWS_SECRET_ACCESS_KEY= - -ironcli publish-function -function-name /sns-example:latest -``` - -Visit the published function's code page in the [IronWorker control -panel](https://hud.iron.io). You should see a cloaked field called "Webhook -URL". Copy this URL. - -In the AWS SNS control panel, visit the `sns-example` topic. Click "Create -Subscription". Select the subscription type as HTTPS and paste the webhook URL. -Once you save this, the IronWorker task should have been launched and then -finished successfully with a "Confirmed subscription" message. - -Now you can click the blue "Publish to topic" button on the AWS SNS control -panel. Select the message format as JSON and the contents as (for example): - -```js -{ - "default": "{\"url\": \"http://www.econrates.com/reality/schul.html\", \"keyword\": \"blackbird\"}" -} -``` - -SNS will send the string in the `"default"` key to all subscribers. You should -be able to see that the IronWorker task has been run again. If everything went -well, it should have printed out a summary and exited successfully. - -That's it, a simple, notification based, Lambda function. diff --git a/examples/s3/Dockerfile b/examples/s3/Dockerfile new file mode 100644 index 0000000..528e257 --- /dev/null +++ b/examples/s3/Dockerfile @@ -0,0 +1,5 @@ +FROM iron/lambda-nodejs + +ADD example.js ./example.js + +CMD ["example.run"] diff --git a/examples/s3/Makefile b/examples/s3/Makefile new file mode 100644 index 0000000..188e742 --- /dev/null +++ b/examples/s3/Makefile @@ -0,0 +1,7 @@ +IMAGE=iron/lambda-node-aws-example + +create: Dockerfile + docker build -t $(IMAGE) . + +test: + docker run --rm -it -e PAYLOAD_FILE=/mnt/example-payload.json -e AWS_ACCESS_KEY_ID=change-here -e AWS_SECRET_ACCESS_KEY=change-here -v `pwd`:/mnt $(IMAGE) diff --git a/examples/s3/README.md b/examples/s3/README.md new file mode 100644 index 0000000..a0f93cb --- /dev/null +++ b/examples/s3/README.md @@ -0,0 +1,2 @@ +Example on how to use AWS S3 in a lambda function. + diff --git a/examples/s3/example-payload.json b/examples/s3/example-payload.json new file mode 100644 index 0000000..a832d6e --- /dev/null +++ b/examples/s3/example-payload.json @@ -0,0 +1,5 @@ +{ + "bucket": "iron-lambda-demo-images", + "srcKey": "waterfall.jpg", + "dstKey": "waterfall-1024.jpg" +} diff --git a/examples/s3/example.js b/examples/s3/example.js new file mode 100644 index 0000000..2f4c3de --- /dev/null +++ b/examples/s3/example.js @@ -0,0 +1,70 @@ +var im = require('imagemagick'); +var fs = require('fs'); +var AWS = require('aws-sdk'); + +// cb(err, resized) is called with true if resized. +function resizeIfRequired(err, features, fileSrc, fileDst, cb) { + if (err) { + cb(err, false); + return; + } + + var targetWidth = 1024; + if (features.width > targetWidth) + { + im.resize({ + srcPath : fileSrc, + dstPath : fileDst, + width : targetWidth, + format: 'jpg' + }, function(err) { + if (err) { + cb(err, false); + } else { + cb(null, true); + } + }); + } else { + cb(null, false); + } +} + +exports.run = function(event, context) { + var bucketName = event['bucket'] + var srcImageKey = event['srcKey'] + var dstImageKey = event['dstKey'] + + var s3 = new AWS.S3(); + + s3.getObject({ + Bucket: bucketName, + Key: srcImageKey + }, function (err, data) { + + if (err) throw err; + + var fileSrc = '/tmp/image-src.dat'; + var fileDst = '/tmp/image-dst.dat' + fs.writeFileSync(fileSrc, data.Body) + + im.identify(fileSrc, function(err, features) { + resizeIfRequired(err, features, fileSrc, fileDst, function(err, resized) { + if (err) throw err; + if (resized) { + s3.putObject({ + Bucket:bucketName, + Key: dstImageKey, + Body: fs.createReadStream(fileDst), + ContentType: 'image/jpeg', + ACL: 'public-read', + }, function (err, data) { + if (err) throw err; + context.succeed("Image updated"); + }); + } else { + context.succeed("Image not updated"); + } + }); + }); + }); +} diff --git a/images/examples/java/src/main/java/example/Hello.java b/images/examples/java/src/main/java/example/Hello.java index 7d4054c..3b98d2f 100644 --- a/images/examples/java/src/main/java/example/Hello.java +++ b/images/examples/java/src/main/java/example/Hello.java @@ -49,7 +49,7 @@ public ArrayList myHandlerList(ArrayList arrayList, Context return arrayList; } - public static void myHandlerPOJO(RequestClass request, Context context){ - System.out.println(String.format("Hello %s %s" , request.firstName, request.lastName)); + public static String myHandlerPOJO(RequestClass request, Context context){ + return String.format("Hello %s %s" , request.firstName, request.lastName); } } diff --git a/images/java/LambdaLauncher.sh b/images/java/LambdaLauncher.sh index 870412c..fc50294 100755 --- a/images/java/LambdaLauncher.sh +++ b/images/java/LambdaLauncher.sh @@ -17,6 +17,13 @@ else exit 1 fi +# set env variables from CONFIG_* prefixed ones +for i in $(env); do + if [[ $i == CONFIG_* ]]; then + export ${i:7} + fi +done + java -jar lambda.jar $2 exit 0 diff --git a/images/java/src/main/java/io/iron/lambda/ClassTypeHelper.java b/images/java/src/main/java/io/iron/lambda/ClassTypeHelper.java index dab8c3f..df8fb65 100644 --- a/images/java/src/main/java/io/iron/lambda/ClassTypeHelper.java +++ b/images/java/src/main/java/io/iron/lambda/ClassTypeHelper.java @@ -21,7 +21,8 @@ private static Set> getSimpleTypes() { ret.add(Integer.class); ret.add(int.class); ret.add(Boolean.class); + ret.add(boolean.class); return ret; } -} \ No newline at end of file +} diff --git a/images/java/src/main/java/io/iron/lambda/Launcher.java b/images/java/src/main/java/io/iron/lambda/Launcher.java index d94ce61..6d63ef7 100644 --- a/images/java/src/main/java/io/iron/lambda/Launcher.java +++ b/images/java/src/main/java/io/iron/lambda/Launcher.java @@ -8,19 +8,44 @@ import java.util.*; import java.util.stream.*; +import com.google.gson.JsonSyntaxException; import io.github.lukehutch.fastclasspathscanner.FastClasspathScanner; public class Launcher { + + PrintStream oldout; + + public Launcher() { + oldout = System.out; + System.setOut(System.err); + } + public static void main(String[] args) { String handler = args[0]; String payload = ""; - try { - String file = System.getenv("PAYLOAD_FILE"); - if (file != null) { + String file = System.getenv("PAYLOAD_FILE"); + if (file != null && !file.isEmpty()) { + try { payload = new String(Files.readAllBytes(Paths.get(file))); + } catch (IOException ioe) { + System.err.println("bootstrap: Could not read payload file " + ioe); + System.exit(1); + } + } else { + try { + ByteArrayOutputStream os = new ByteArrayOutputStream(); + byte[] buffer = new byte[4096]; + int n; + while ((n = System.in.read(buffer)) != -1) { + os.write(buffer, 0, n); + } + os.close(); + + payload = os.toString("UTF-8"); + } catch (IOException ioe) { + System.err.println("bootstrap: Could not read stdin " + ioe); + System.exit(1); } - } catch (IOException ioe) { - // Should probably log this somewhere useful but not in the output. } try { @@ -98,19 +123,31 @@ private void runMethod(Method lambdaMethod, String payload, Class cls, String ha // int, string, bool } else if (checkIfLambdaMethodRequiredPrimitiveType(parameterTypes)) { if (parameterTypes[0].getName().equals("java.lang.String")) { - result = callWithMaybeContext(lambdaMethod, lambdaInstance, contextRequired, payload, aws_ctx); + String p = ClassTypeHelper.gson.fromJson(payload, String.class); + result = callWithMaybeContext(lambdaMethod, lambdaInstance, contextRequired, p, aws_ctx); processed = true; - } else if (Objects.equals(parameterTypes[0].toString(), "int") || Objects.equals(parameterTypes[0].toString(), "Integer")) { - result = callWithMaybeContext(lambdaMethod, lambdaInstance, contextRequired, Integer.parseInt(payload), aws_ctx); + } else if (parameterTypes[0].toString().equals("int") || Objects.equals(parameterTypes[0].toString(), "Integer")) { + int i = ClassTypeHelper.gson.fromJson(payload, int.class); + result = callWithMaybeContext(lambdaMethod, lambdaInstance, contextRequired, i, aws_ctx); processed = true; - } else if (Objects.equals(parameterTypes[0].toString(), "boolean") || Objects.equals(parameterTypes[0].toString(), "Boolean")) { - result = callWithMaybeContext(lambdaMethod, lambdaInstance, contextRequired, Boolean.valueOf(payload), aws_ctx); + } else if (parameterTypes[0].toString().equals("boolean") || Objects.equals(parameterTypes[0].toString(), "Boolean")) { + boolean b = ClassTypeHelper.gson.fromJson(payload, boolean.class); + result = callWithMaybeContext(lambdaMethod, lambdaInstance, contextRequired, b, aws_ctx); processed = true; } } if (!processed) { System.err.println(String.format("Handler %s with simple, POJO, or IO(input/output) types not found", handlerName)); } + + try { + String jsonInString = ClassTypeHelper.gson.toJson(result); + oldout.println(jsonInString); + } + catch (Exception e) { + oldout.println("{\"error\": \"" + ClassTypeHelper.gson.toJson(e.toString()) + "\"}"); + System.exit(1); + } } private void launchMethod(String[] packageHandler, String payload) throws Exception { @@ -175,10 +212,6 @@ private void launchMethod(String[] packageHandler, String payload) throws Except }); Class returnType = matchedArray[0].getReturnType(); - if (!returnType.getTypeName().equals("void")) { - System.err.println(String.format("Handler can only have 'void' return type. Found '%s'.", returnType.getTypeName())); - System.exit(1); - } runMethod(matchedArray[0], payload, cls, handlerName); } diff --git a/images/node/README.md b/images/node/README.md deleted file mode 100644 index ee5afe8..0000000 --- a/images/node/README.md +++ /dev/null @@ -1,28 +0,0 @@ -Support for running nodejs Lambda functions. - -Create an image with - - docker build -t iron/node-lambda . - -This sets up a node stack and installs some deps to provide the Lambda runtime. - -Right now we use [node-lambda](https://github.com/motdotla/node-lambda) to exec -the handler and provide the function. Unfortunately it requires -a `package.json`, hence the `package.json.stupid`. Eventually we should break -the `node-lambda run` part of it out and use that. - -Running -------- - -Does not support payload (AWS Lambda 'event') yet. Expects the lambda zip file -to be called `function.zip` and available at `/mnt/function.zip`. HANDLER -should be set to `module.*export*`. So - - // fancy.js inside function.zip - exports.fancyFunction = function(event, context) {} - -would be launched as: - - docker run --rm -it -v /path/to/dir/containing/function.zip:/mnt -e HANDLER=fancy.fancyFunction iron/node-lambda - -In Lambda you'd submit these parameters in the call to `create-function`. diff --git a/images/node/bootstrap.js b/images/node/bootstrap.js index 5ea193d..a080ed6 100644 --- a/images/node/bootstrap.js +++ b/images/node/bootstrap.js @@ -2,6 +2,9 @@ var fs = require('fs'); +var oldlog = console.log +console.log = console.error + // Some notes on the semantics of the succeed(), fail() and done() methods. // Tests are the source of truth! // First call wins in terms of deciding the result of the function. BUT, @@ -64,11 +67,10 @@ var Context = function() { result = null } - var str; var failed = false; try { - str = JSON.stringify(result) - // Succeed does not output to log, it only responds to the HTTP request. + // Output result to log + oldlog(JSON.stringify(result)); } catch(e) { // Set X-Amz-Function-Error: Unhandled header console.log("Unable to stringify body as json: " + e); @@ -108,10 +110,10 @@ var Context = function() { } else { errstr = error.toString() } - console.log(JSON.stringify({"errorMessage": errstr })) + oldlog(JSON.stringify({"errorMessage": errstr })) } catch(e) { // Set X-Amz-Function-Error: Unhandled header - console.log(errstr) + oldlog(errstr) } } @@ -223,46 +225,92 @@ var makeCtx = function() { return ctx; } +var setEnvFromHeader = function () { + var headerPrefix = "CONFIG_"; + var newEnvVars = {}; + for (var key in process.env) { + if (key.indexOf(headerPrefix) == 0) { + newEnvVars[key.slice(headerPrefix.length)] = process.env[key]; + } + } + + for (var key in newEnvVars) { + process.env[key] = newEnvVars[key]; + } +} + + function run() { + setEnvFromHeader(); // FIXME(nikhil): Check for file existence and allow non-payload. - var payload = {}; var path = process.env["PAYLOAD_FILE"]; + var stream = process.stdin; if (path) { try { - var contents = fs.readFileSync(path, { encoding: 'utf8' }); - payload = JSON.parse(contents); + stream = fs.createReadStream(path); } catch(e) { - console.error("bootstrap: Error reading payload file", e) + console.error("bootstrap: Error opening payload file", e) + process.exit(1); } } - if (process.argv.length > 2) { - var handler = process.argv[2]; - var parts = handler.split('.'); - // FIXME(nikhil): Error checking. - var script = parts[0]; - var entry = parts[1]; + var input = ""; + stream.setEncoding('utf8'); + stream.on('data', function(chunk) { + input += chunk; + }); + + stream.on('error', function(err) { + console.error("bootstrap: Error reading payload stream", err); + process.exit(1); + }); + + stream.on('end', function() { + var payload = {} try { - var mod = require('./'+script); - if (mod[entry] === undefined) { - throw "Handler '" + entry + "' missing on module '" + script + "'" + if (input.length > 0) { + payload = JSON.parse(input); } + } catch(e) { + console.error("bootstrap: Error parsing JSON", e); + process.exit(1); + } - if (typeof mod[entry] !== 'function') { - throw "TypeError: " + (typeof mod[entry]) + " is not a function" - } + if (process.argv.length > 2) { + var handler = process.argv[2]; + var parts = handler.split('.'); + // FIXME(nikhil): Error checking. + var script = parts[0]; + var entry = parts[1]; + var started = false; + try { + var mod = require('./'+script); + var func = mod[entry]; + if (func === undefined) { + oldlog("Handler '" + entry + "' missing on module '" + script + "'"); + return; + } - mod[entry](payload, makeCtx()) - } catch(e) { - if (typeof e === 'string') { - console.log(e) - } else { - console.log(e.message) + if (typeof func !== 'function') { + throw "TypeError: " + (typeof func) + " is not a function"; + } + started = true; + mod[entry](payload, makeCtx()) + } catch(e) { + if (typeof e === 'string') { + oldlog(e) + } else { + oldlog(e.message) + } + if (!started) { + oldlog("Process exited before completing request\n") + } } + } else { + console.error("bootstrap: No script specified") + process.exit(1); } - } else { - console.error("bootstrap: No script specified") - } + }) } run() diff --git a/images/python/Dockerfile b/images/python/Dockerfile index 79dc6b5..fc8330e 100644 --- a/images/python/Dockerfile +++ b/images/python/Dockerfile @@ -13,8 +13,8 @@ RUN \ && apk --no-cache add py-curl \ && easy_install-2.7 "urlgrabber==3.9.1" \ \ - && easy_install-2.7 "botocore==1.3.22" \ - && easy_install-2.7 "boto3==1.2.3" \ + && easy_install-2.7 "botocore==1.4.17" \ + && easy_install-2.7 "boto3==1.3.1" \ \ && rm -rf /tmp/* diff --git a/images/python/bootstrap.py b/images/python/bootstrap.py index 449a7cd..b575caf 100644 --- a/images/python/bootstrap.py +++ b/images/python/bootstrap.py @@ -8,6 +8,8 @@ import time import uuid +oldstdout = sys.stdout +sys.stdout = sys.stderr debugging = False @@ -33,7 +35,7 @@ def __init__(self): self.memory_limit_in_mb = int(getTASK_MAXRAM() / 1024 / 1024) def get_remaining_time_in_millis(self): - remaining = plannedEnd - time.time() + remaining = plannedEnd - int(time.time()) if remaining < 0: remaining = 0 return remaining * 1000 @@ -112,8 +114,10 @@ def getPAYLOAD_FILE(): def getTASK_TIMEOUT(): - return os.environ.get('TASK_TIMEOUT') or 3600 - + try: + return int(os.environ.get('TASK_TIMEOUT', '')) + except ValueError: + return 3600 def getTASK_MAXRAM(): # IronWorker uses MAXMEM, Hybrid uses MAXRAM. @@ -190,7 +194,19 @@ def filter(self, record): } logging.config.dictConfig(loggingConfig) -plannedEnd = time.time() + getTASK_TIMEOUT() +def setEnvFromHeader(): + headerPrefix = "CONFIG_" + newEnvVars = {} + for key, value in os.environ.iteritems(): + if key.startswith(headerPrefix): + newEnvVars[key[len(headerPrefix):]] = value + + for key, value in newEnvVars.iteritems(): + os.environ[key] = value + + +setEnvFromHeader() +plannedEnd = int(time.time()) + getTASK_TIMEOUT() debugging and print ('os.environ = ', os.environ) debugging and print ('/mnt content = ', os.listdir("/mnt")) @@ -211,11 +227,6 @@ def filter(self, record): if handlerName is None: stopWithError("handlerName arg is not specified") -if payloadFileName is None: - stopWithError("PAYLOAD_FILE variable is not specified") - -if not os.path.isfile(payloadFileName): - stopWithError("No payload present") handlerParts = string.rsplit(handlerName, ".", 2) @@ -232,15 +243,20 @@ def filter(self, record): stopWithError("Function name is not defined") try: - with file(payloadFileName) as f: + if payloadFileName: + payloadFile = file(payloadFileName, 'r') + else: + payloadFile = sys.stdin + + with payloadFile as f: payload = f.read() -except: - stopWithError("Failed to read {payloadFileName}".format(payloadFileName=payloadFileName)) -debugging and print ('payload loaded') +except Exception, e: + stopWithError("Failed to read {payloadFileName}. err={err}".format(payloadFileName=(payloadFileName or ''), err=e)) try: - payload = json.loads(payload) + if len(payload) > 0: + payload = json.loads(payload) except: debugging and print ('payload is ') and print (payload) stopWithError('Payload is not JSON') @@ -260,7 +276,7 @@ def filter(self, record): try: result = caller.call(payload, context) - #FIXME where to put result in async mode? + oldstdout.write(json.dumps(result)) except Exception as e: stopWithError(e) diff --git a/lambda/lambda.go b/lambda/lambda.go index 0277dc5..e56b7f0 100644 --- a/lambda/lambda.go +++ b/lambda/lambda.go @@ -258,6 +258,15 @@ func RunImageWithPayload(imageName string, payload string) error { envs = append(envs, "AWS_LAMBDA_FUNCTION_VERSION=$LATEST") envs = append(envs, "TASK_ID="+uuid.NewV4().String()) envs = append(envs, fmt.Sprintf("TASK_MAXRAM=%d", allocatedMemory)) + // Try to forward AWS credentials. + { + creds := credentials.NewEnvCredentials() + v, err := creds.Get() + if err == nil { + envs = append(envs, "AWS_ACCESS_KEY_ID="+v.AccessKeyID) + envs = append(envs, "AWS_SECRET_ACCESS_KEY="+v.SecretAccessKey) + } + } opts := docker.CreateContainerOptions{ Config: &docker.Config{ @@ -323,8 +332,7 @@ func RunImageWithPayload(imageName string, payload string) error { // Registers public docker image named `imageNameVersion` as a IronWorker called `imageName`. // For example, // RegisterWithIron("foo/myimage:1", credentials.NewEnvCredentials()) will register a worker called "foo/myimage" that will use Docker Image "foo/myimage:1". -// The AWS credentials are required to configure environment variables for the image so that the AWS APIs can be used successfully. -func RegisterWithIron(imageNameVersion string, awsCredentials *credentials.Credentials) error { +func RegisterWithIron(imageNameVersion string) error { tokens := strings.Split(imageNameVersion, ":") if len(tokens) != 2 || tokens[0] == "" || tokens[1] == "" { return errors.New("Invalid image name. Should be of the form \"name:version\".") @@ -332,25 +340,29 @@ func RegisterWithIron(imageNameVersion string, awsCredentials *credentials.Crede imageName := tokens[0] - creds, err := awsCredentials.Get() - if err != nil { - return errors.New(fmt.Sprintf("Could not extract AWS credentials to register environment variables with IronWorker: %s", err)) - } - // Worker API doesn't have support for register yet, but we use it to extract the configuration. w := worker.New() url := fmt.Sprintf("https://%s/2/projects/%s/codes?oauth=%s", w.Settings.Host, w.Settings.ProjectId, w.Settings.Token) - marshal, err := json.Marshal(map[string]interface{}{ + registerOpts := map[string]interface{}{ "name": imageName, "image": imageNameVersion, "env_vars": map[string]string{ - "AWS_ACCESS_KEY_ID": creds.AccessKeyID, - "AWS_SECRET_ACCESS_KEY": creds.SecretAccessKey, "AWS_LAMBDA_FUNCTION_NAME": imageName, "AWS_LAMBDA_FUNCTION_VERSION": "1", // FIXME: swapi does not allow $ right now. }, - }) + } + + // Try to forward AWS credentials. + { + creds := credentials.NewEnvCredentials() + v, err := creds.Get() + if err == nil { + registerOpts["env_vars"].(map[string]string)["AWS_ACCESS_KEY_ID"] = v.AccessKeyID + registerOpts["env_vars"].(map[string]string)["AWS_SECRET_ACCESS_KEY"] = v.SecretAccessKey + } + } + marshal, err := json.Marshal(registerOpts) var body bytes.Buffer mw := multipart.NewWriter(&body) jsonWriter, err := mw.CreateFormField("data") diff --git a/test-suite/glide.lock b/test-suite/glide.lock index 27a50d1..f8d09ac 100644 --- a/test-suite/glide.lock +++ b/test-suite/glide.lock @@ -1,8 +1,8 @@ -hash: 3acc61a3ed582a0bed9b45f7ca21745e319397b3685a57d2b33ee6cf32870eaf -updated: 2016-03-10T15:41:06.138951394+05:30 +hash: 144bd93c4247f3246e459717b36598951283499fa842ff3bed8ad23dfea2478d +updated: 2016-10-18T11:20:43.851037791-07:00 imports: - name: github.com/aws/aws-sdk-go - version: 2e7cf03d7f5c8a4b4c9f7341ddf1e13102845cf2 + version: 8f2725740345a0561fa09669f5f75f905404ef8b subpackages: - aws - aws/awserr @@ -25,10 +25,31 @@ imports: - aws/ec2metadata - private/protocol/json/jsonutil - private/protocol/rest +- name: github.com/Azure/go-ansiterm + version: fa152c58bc15761d0200cb75fe958b89a9d4888e + subpackages: + - winterm +- name: github.com/docker/docker + version: 20f81dde9bd97c86b2d0e33bbbf1388018611929 + subpackages: + - pkg/jsonmessage + - pkg/jsonlog + - pkg/term + - pkg/system + - pkg/term/windows +- name: github.com/docker/engine-api + version: 4290f40c056686fcaa5c9caf02eac1dde9315adf + subpackages: + - types/filters + - types/swarm +- name: github.com/docker/go-units + version: f2145db703495b2e525c59662db69a7344b00bb8 - name: github.com/go-ini/ini - version: 776aa739ce9373377cd16f526cdf06cb4c89b40f + version: 6e4869b434bd001f6983749881c7ead3545887d8 +- name: github.com/hashicorp/go-cleanhttp + version: ad28ea4487f05916463e2423a55166280e8254b5 - name: github.com/iron-io/iron_go3 - version: 4aec4b86e69b521ea83e4434e274630585b73b54 + version: cd9cc95ce2d2bb25d2e4e10cd62fff1d97ad1906 subpackages: - worker - api @@ -38,11 +59,21 @@ imports: subpackages: - lambda - name: github.com/jmespath/go-jmespath - version: 0b12d6b521d83fc7f755e7cfc1b1fbdd35a01a74 + version: bd40a432e4c76585ef6b72d3fd96fb9b6dc7b68d +- name: github.com/opencontainers/runc + version: bf77e5976ad42b6167a167a9f3093748f7c154db + subpackages: + - libcontainer/user - name: github.com/satori/go.uuid - version: e673fdd4dea8a7334adbbe7f57b7e4b00bdc5502 + version: b061729afc07e77a8aa4fad0a2fd840958f1942a - name: github.com/sendgrid/sendgrid-go version: 0bf6332788d0230b7da84a1ae68d7531073200e1 - name: github.com/sendgrid/smtpapi-go - version: 08102729bcf72bfe74dde6c0da8e0423e94090ca + version: 072b6f477c501c30348f442559997853f12b364e +- name: github.com/Sirupsen/logrus + version: 3ec0642a7fb6488f65b06f9040adc67e3990296a +- name: golang.org/x/sys + version: 002cbb5f952456d0c50e0d2aff17ea5eca716979 + subpackages: + - unix devImports: [] diff --git a/test-suite/glide.yaml b/test-suite/glide.yaml index daa9e6c..cfc5b3d 100644 --- a/test-suite/glide.yaml +++ b/test-suite/glide.yaml @@ -17,3 +17,15 @@ import: - package: github.com/satori/go.uuid - package: github.com/sendgrid/sendgrid-go version: ~2.0.0 +- package: github.com/docker/docker + version: ~1.10.3 + subpackages: + - pkg/jsonmessage +- package: github.com/docker/engine-api + subpackages: + - types/filters + - types/swarm +- package: github.com/hashicorp/go-cleanhttp +- package: github.com/opencontainers/runc + subpackages: + - libcontainer/user diff --git a/test-suite/iron.go b/test-suite/iron.go index 68bcd6c..7e63128 100644 --- a/test-suite/iron.go +++ b/test-suite/iron.go @@ -5,12 +5,10 @@ import ( "bytes" "encoding/json" "fmt" - "io" "log" "os" "regexp" "strings" - "sync" "time" "github.com/iron-io/iron_go3/worker" @@ -37,31 +35,32 @@ func cleanIronGeneric(output []byte) []byte { return output } -func cleanPython27IronOutput(output string) (string, error) { +func cleanIronTaskIdAndTimestamp(output string) (string, error) { var buf bytes.Buffer - var requestId string = "" + var taskId string = "" // expecting request id as hex of bson_id requestIdPattern, _ := regexp.Compile("[a-f0-9]{24}") scanner := bufio.NewScanner(strings.NewReader(output)) for scanner.Scan() { line := scanner.Text() - if requestId == "" { + if taskId == "" { parts := strings.Fields(line) // generic logging through logger.info, logger.warning & etc if len(parts) >= 3 { requestIdCandidate := parts[2] if requestIdPattern.MatchString(requestIdCandidate) { - requestId = requestIdCandidate + taskId = requestIdCandidate } } } - line = util.RemoveTimestampAndRequestIdFromLogLine(line, requestId) - - buf.WriteString(line) - buf.WriteRune('\n') + line, isOk := util.RemoveTimestampAndRequestIdFromIronLogLine(line, taskId) + if isOk { + buf.WriteString(line) + buf.WriteRune('\n') + } if err := scanner.Err(); err != nil { return "", err } @@ -70,57 +69,77 @@ func cleanPython27IronOutput(output string) (string, error) { return buf.String(), nil } -func cleanIron(runtime string, output []byte) ([]byte, error) { +func cleanIron(output []byte) ([]byte, error) { output = cleanIronGeneric(output) - switch runtime { - case "python2.7": - cleaned, err := cleanPython27IronOutput(string(output)) - return []byte(cleaned), err - default: - return output, nil - } + cleaned, err := cleanIronTaskIdAndTimestamp(string(output)) + return []byte(cleaned), err } -func runOnIron(w *worker.Worker, wg *sync.WaitGroup, test *util.TestDescription, result chan<- io.Reader) { - var imagePrefix string - if imagePrefix = os.Getenv("IRON_LAMBDA_TEST_IMAGE_PREFIX"); imagePrefix == "" { - log.Fatalf("IRON_LAMBDA_TEST_IMAGE_PREFIX not set") - } +//Returns a result and a debug channels. The channels are closed on test run finalization +func runOnIron(w *worker.Worker, test *util.TestDescription) (<-chan string, <-chan string) { + result := make(chan string, 1) + debug := make(chan string, 1) + go func() { + defer close(result) + defer close(debug) + var imagePrefix string + if imagePrefix = os.Getenv("IRON_LAMBDA_TEST_IMAGE_PREFIX"); imagePrefix == "" { + log.Fatalf("IRON_LAMBDA_TEST_IMAGE_PREFIX not set") + } - var output bytes.Buffer - defer func() { - result <- &output - wg.Done() - }() + payload, _ := json.Marshal(test.Event) + timeout := time.Duration(test.Timeout) * time.Second - payload, _ := json.Marshal(test.Event) - timeout := time.Duration(test.Timeout) * time.Second + debug <- "Enqueuing the task" + taskids, err := w.TaskQueue(worker.Task{ + Cluster: "internal", + CodeName: fmt.Sprintf("%s/%s", imagePrefix, test.Name), + Payload: string(payload), + Timeout: &timeout, + }) - taskids, err := w.TaskQueue(worker.Task{ - Cluster: "internal", - CodeName: fmt.Sprintf("%s/%s", imagePrefix, test.Name), - Payload: string(payload), - Timeout: &timeout, - }) + if err != nil { + debug <- fmt.Sprintf("Error queueing task %s", err) + return + } - if err != nil { - output.WriteString(fmt.Sprintf("Error queueing task %s %s", test.Name, err)) - return - } + if len(taskids) < 1 { + debug <- "Something went wrong, empty taskids list" + return + } - if len(taskids) < 1 { - output.WriteString(fmt.Sprintf("Something went wrong, empty taskids list", test.Name)) - return - } + end := time.After(timeout) + taskid := taskids[0] + debug <- fmt.Sprintf("Task Id: %s", taskid) - taskid := taskids[0] + debug <- "Waiting for task" + select { + case <-w.WaitForTask(taskid): + case <-end: + debug <- fmt.Sprintf("Task timed out %s", taskid) + return + } - <-w.WaitForTask(taskid) - iron_log := <-w.WaitForTaskLog(taskid) - cleanedLog, err := cleanIron(test.Runtime, iron_log) - if err != nil { - output.WriteString(fmt.Sprintf("Error processing a log for task %s %s", test.Name, err)) - } else { - output.Write(cleanedLog) - } + var iron_log []byte + debug <- "Waiting for task log" + select { + case _iron_log, wait_log_ok := <-w.WaitForTaskLog(taskid): + if !wait_log_ok { + debug <- fmt.Sprintf("Something went wrong, no task log %s", taskid) + return + } + iron_log = _iron_log + case <-end: + debug <- fmt.Sprintf("Task timed out to get task log or the log is empty %s", taskid) + return + } + + cleanedLog, err := cleanIron(iron_log) + if err != nil { + debug <- fmt.Sprintf("Error processing a log %s", test.Name) + } else { + result <- string(cleanedLog) + } + }() + return result, debug } diff --git a/test-suite/lambda.go b/test-suite/lambda.go index 6aa9ccc..ca3d913 100644 --- a/test-suite/lambda.go +++ b/test-suite/lambda.go @@ -6,10 +6,8 @@ import ( "encoding/json" "errors" "fmt" - "io" "log" "strings" - "sync" "time" "github.com/aws/aws-sdk-go/aws" @@ -27,49 +25,12 @@ func indexOf(a string, list []string) int { return -1 } -func cleanNodeJsAwsOutput(output string) (string, error) { - var buf bytes.Buffer - if strings.HasPrefix(output, "START RequestId:") { - scanner := bufio.NewScanner(strings.NewReader(output)) - if scanner.Scan() { - firstLine := scanner.Text() - fields := strings.Fields(firstLine) - if len(fields) > 2 { - id := fields[2] - for scanner.Scan() { - line := strings.TrimSpace(scanner.Text()) - if strings.HasPrefix(line, "END") { - return buf.String(), nil - } - - // Remove timestamp - idx := strings.IndexByte(line, 'Z') - if idx >= 0 { - untimed := strings.TrimSpace(line[idx+1:]) - unprefix := strings.TrimPrefix(untimed, id) - buf.WriteString(strings.TrimSpace(unprefix)) - buf.WriteRune('\n') - } else { - buf.WriteString(line) - buf.WriteRune('\n') - } - } - if err := scanner.Err(); err != nil { - return "", err - } - } - } - } - - return "", errors.New(fmt.Sprintf("Don't know how to clean '%s'", output)) -} - // Processes all requests log lines inside the log and succedes only with the latest one // The log line format: [some data] [timestamp] [request_id] [some other data] // Request start format: START RequestId: [request_id] [some data] // Request end format: END RequestId: [request_id] [some data] // AWS report format: REPORT RequestId: [request_id] [some data] -func cleanPython27AwsOutput(output string) (string, error) { +func cleanLambda(output string) (string, error) { var buf bytes.Buffer var requestId string = "" knownRequestIds := make(map[string]bool, 0) @@ -116,10 +77,11 @@ func cleanPython27AwsOutput(output string) (string, error) { } } - line = util.RemoveTimestampAndRequestIdFromLogLine(line, requestId) - - buf.WriteString(line) - buf.WriteRune('\n') + line, isOk := util.RemoveTimestampAndRequestIdFromAwsLogLine(line, requestId) + if isOk { + buf.WriteString(line) + buf.WriteRune('\n') + } if err := scanner.Err(); err != nil { return "", err } @@ -133,81 +95,165 @@ func cleanPython27AwsOutput(output string) (string, error) { } -func cleanAwsGeneric(old_output, output string) (string, error) { - if old_output == output { - return "", errors.New("No change in the log") - } +func findFirstRequestIdFromLog(output string) string { + scanner := bufio.NewScanner(strings.NewReader(output)) + for scanner.Scan() { + line := scanner.Text() + fields := strings.Fields(line) + + //processing START, END and REPORT log lines + requestIdFieldIndex := indexOf("RequestId:", fields) + if (requestIdFieldIndex > 0) && (requestIdFieldIndex+1 < len(fields)) { + requestIdInLine := fields[requestIdFieldIndex+1] + prefix := fields[requestIdFieldIndex-1] - if strings.HasPrefix(output, old_output) { - return output[len(old_output):], nil + if prefix == "START" { + return requestIdInLine + } + } } - return output, nil + + return "" } -func cleanAws(old_output, output, runtime string) (string, error) { - output, err := cleanAwsGeneric(old_output, output) - if err != nil { - return "", err - } +func isEndOfRequestMarkerPresent(output, requestId string) bool { + scanner := bufio.NewScanner(strings.NewReader(output)) + for scanner.Scan() { + line := scanner.Text() + fields := strings.Fields(line) - switch runtime { - case "nodejs": - return cleanNodeJsAwsOutput(output) - case "python2.7": - return cleanPython27AwsOutput(output) - case "java8": - return cleanNodeJsAwsOutput(output) - default: - return output, nil + //processing START, END and REPORT log lines + requestIdFieldIndex := indexOf("RequestId:", fields) + if (requestIdFieldIndex > 0) && (requestIdFieldIndex+1 < len(fields)) { + requestIdInLine := fields[requestIdFieldIndex+1] + prefix := fields[requestIdFieldIndex-1] + + if prefix == "END" && requestIdInLine == requestId { + return true + } + } } + return false } -func runOnLambda(l *lambda.Lambda, cw *cloudwatchlogs.CloudWatchLogs, wg *sync.WaitGroup, test *util.TestDescription, result chan<- io.Reader) { - var output bytes.Buffer - defer func() { - result <- &output - wg.Done() - }() - - name := test.Name +func getLogAndDetectRequestComplete(logGetter func() (string, error), old_output, requestId string) (string, string, bool, error) { - old_invocation_log, err := getLog(cw, name) + output, err := logGetter() if err != nil { - old_invocation_log = "" + return "", "", false, err } - payload, _ := json.Marshal(test.Event) + log := strings.TrimPrefix(output, old_output) - invoke_input := &lambda.InvokeInput{ - FunctionName: aws.String(name), - InvocationType: aws.String("Event"), - Payload: payload, + if requestId == "" { + requestId = findFirstRequestIdFromLog(log) } - _, err = l.Invoke(invoke_input) - if err != nil { - output.WriteString(fmt.Sprintf("Error invoking function %s %s", name, err)) - return + + if requestId != "" { + marker := isEndOfRequestMarkerPresent(strings.TrimPrefix(output, old_output), requestId) + return output, requestId, marker, nil } - latency := 1 //1 second for network and infrastructure timeouts - timeout := time.Duration(test.Timeout+latency) * time.Second + return log, "", false, nil +} - time.Sleep(timeout) +//Returns a result and a debug channels. The channels are closed on test run finalization +func runOnLambda(l *lambda.Lambda, cw *cloudwatchlogs.CloudWatchLogs, test *util.TestDescription) (<-chan string, <-chan string) { + result := make(chan string, 1) + debug := make(chan string, 1) + go func() { + defer close(result) + defer close(debug) - invocation_log, err := getLog(cw, name) - if err != nil { - output.WriteString(fmt.Sprintf("Error getting log %s %s", name, err)) - return - } + name := test.Name - final, err := cleanAws(old_invocation_log, invocation_log, test.Runtime) + debug <- "Getting old log" + old_invocation_log, err := getLog(cw, name) + if err != nil { + old_invocation_log = "" + } - if err != nil { - output.WriteString(fmt.Sprintf("Error cleaning log %s %s", name, err)) - return - } + payload, _ := json.Marshal(test.Event) + + invoke_input := &lambda.InvokeInput{ + FunctionName: aws.String(name), + InvocationType: aws.String("Event"), + Payload: payload, + } + debug <- "Enqueuing task" + _, err = l.Invoke(invoke_input) + if err != nil { + debug <- fmt.Sprintf("Error invoking function %s ", err) + return + } - output.WriteString(final) + timeout := time.Duration(test.Timeout) * time.Second + + debug <- "Waiting for task" + now := time.Now() + elapsed := time.After(timeout) + ticker := time.NewTicker(3 * time.Second) + defer ticker.Stop() + + log := "" + completed := false + requestId := "" + logGetter := func() (string, error) { + return getLog(cw, name) + } + + // waiting for test.Timeout or for the full log whichever occurs first + logWaitLoop: + for { + select { + case <-elapsed: + break logWaitLoop + case <-ticker.C: + log, requestId, completed, err = getLogAndDetectRequestComplete(logGetter, old_invocation_log, requestId) + if err != nil { + debug <- fmt.Sprintf("Error getting log %s ", err) + return + } + if completed { + break logWaitLoop + } + } + } + + if !completed { + log, requestId, completed, err = getLogAndDetectRequestComplete(logGetter, old_invocation_log, requestId) + if err != nil { + debug <- fmt.Sprintf("Error getting log %s ", err) + return + } + if !completed { + if requestId != "" { + debug <- fmt.Sprintf("Request Id: %s", requestId) + } + debug <- time.Now().Sub(now).String() + logLines := strings.Split(log, "\n") + switch len(logLines) { + case 0: + debug <- "Log for current test run is empty" + case 1: + debug <- fmt.Sprintf("Log does not contain entries for current test run:\n%s", logLines[0]) + default: + debug <- fmt.Sprintf("Log does not contain entries for current test run:\n%s\n...\n%s", logLines[0], logLines[len(logLines)-1]) + } + return + } + } + debug <- fmt.Sprintf("Request Id: %s", requestId) + final, err := cleanLambda(log) + + if err != nil { + debug <- fmt.Sprintf("Error cleaning log %s", err) + return + } + + result <- final + }() + return result, debug } func getLog(cw *cloudwatchlogs.CloudWatchLogs, name string) (string, error) { @@ -247,6 +293,8 @@ func getLog(cw *cloudwatchlogs.CloudWatchLogs, name string) (string, error) { LogStreamName: stream.LogStreamName, LogGroupName: group.LogGroupName, StartFromHead: aws.Bool(true), + // allow 3 minute local vs server time out of sync + StartTime: aws.Int64(time.Now().Add(-3*time.Minute).Unix() * 1000), } events, err := cw.GetLogEvents(get_log_input) diff --git a/test-suite/main.go b/test-suite/main.go index 469b672..7b7cb9f 100644 --- a/test-suite/main.go +++ b/test-suite/main.go @@ -4,13 +4,11 @@ import ( "bytes" "flag" "fmt" - "io" "io/ioutil" "log" "os" "path/filepath" "strings" - "sync" "time" "github.com/aws/aws-sdk-go/aws" @@ -20,7 +18,6 @@ import ( "github.com/aws/aws-sdk-go/service/lambda" "github.com/iron-io/iron_go3/worker" "github.com/iron-io/lambda/test-suite/util" - "github.com/satori/go.uuid" "github.com/sendgrid/sendgrid-go" ) @@ -155,65 +152,159 @@ Runs all tests. If filter is passed, only runs tests matching filter. Filter is if err != nil { log.Fatal(err) } + if len(tests) == 0 { + log.Fatal("No tests to run") + } - endMarker := uuid.NewV4().String() - testResults := make(chan []string) - endMarkerCount := 0 + // expected duration for all tests to run in a sequential way + // after the fullTimeout expires no test result is accepted and `Timeout` message is reported for a test + fullTimeout := 0 for _, test := range tests { - endMarkerCount++ - go runTest(endMarker, testResults, test, w, cw, l) + fullTimeout += test.Timeout + 5 } - for endMarkerCount > 0 { - lines := <-testResults - for _, line := range lines { - if line == endMarker { - endMarkerCount-- - } else { - log.Println(line) + concurrency := util.NewSemaphore(5) // using a limit, otherwise AWS fails with `ThrottlingException: Rate exceeded` on log retrieval + + endOfTime := time.Now().Add(time.Duration(fullTimeout) * time.Second) + var testResults <-chan []string = nil + for _, test := range tests { + r := runTest(test, w, cw, l, endOfTime, concurrency) + + // forwarding messages from all tests to a single channel + testResults = util.JoinChannels(testResults, r) + } + + passed, failed := make([]string, 0, len(tests)), make([]string, 0, len(tests)) + + for { + lines, ok := <-testResults + if !ok { + break + } + + if len(lines) > 0 { + if strings.HasPrefix(lines[0], "PASS ") { + passed = append(passed, lines[0]) + } + if strings.HasPrefix(lines[0], "FAIL ") { + failed = append(failed, lines[0]) } } - } -} -func runTest(endMarker string, result chan<- []string, test *util.TestDescription, w *worker.Worker, cw *cloudwatchlogs.CloudWatchLogs, l *lambda.Lambda) { - defer func() { - result <- []string{endMarker} - }() + for _, line := range lines { + log.Println(line) + } + } - testName := test.Name + log.Println(fmt.Sprintf("Total %d passed and %d failed tests", len(passed), len(failed))) + for _, line := range passed { + log.Println(line) + } + for _, line := range failed { + log.Println(line) + } - result <- []string{ - fmt.Sprintf("Starting test %s", testName), + if len(failed) > 0 { + os.Exit(1) } +} - awschan := make(chan io.Reader, 1) - ironchan := make(chan io.Reader, 1) +//Returns a channel with a test run result and debug messages +func runTest(test *util.TestDescription, w *worker.Worker, cw *cloudwatchlogs.CloudWatchLogs, l *lambda.Lambda, waitEnd time.Time, s util.Semaphore) <-chan []string { + result := make(chan []string) - var wg sync.WaitGroup - wg.Add(2) - go runOnLambda(l, cw, &wg, test, awschan) - go runOnIron(w, &wg, test, ironchan) + go func() { + defer close(result) - wg.Wait() + s.Lock() + defer s.Unlock() - awsreader := <-awschan - awss, _ := ioutil.ReadAll(awsreader) + testName := test.Name - ironreader := <-ironchan - irons, _ := ioutil.ReadAll(ironreader) + result <- []string{ + fmt.Sprintf("Starting test %s", testName), + } + + endOfWait := time.NewTimer(waitEnd.Sub(time.Now())) + + awschan, awsdbg := runOnLambda(l, cw, test) + ironchan, irondbg := runOnIron(w, test) + + // redirecting every debug message from test runs to result without waiting test results + // before closing `result` aslo waits for closing of debug channels + defer util.ForwardInBackground("DBG AWS Lambda "+testName+" ", awsdbg, result)() + defer util.ForwardInBackground("DBG Iron "+testName+" ", irondbg, result)() + + // waiting for test results or for the timeout whichever occurs first + var awss, irons *bytes.Buffer + elapsed := false + for !elapsed && (awschan != nil || ironchan != nil) { + select { + case data, ok := <-awschan: + { + if ok { + if awss == nil { + awss = &bytes.Buffer{} + } + awss.WriteString(data) + } else { + awschan = nil + } + } + case data, ok := <-ironchan: + { + if ok { + if irons == nil { + irons = &bytes.Buffer{} + } + irons.WriteString(data) + } else { + ironchan = nil + } + } + case <-endOfWait.C: + elapsed = true + } + } - if !bytes.Equal(awss, irons) { delimiter := "==========================================" - result <- []string{ - fmt.Sprintf("FAIL %s Output does not match!", testName), - fmt.Sprintf("AWS lambda output\n%s\n%s\n%s", delimiter, awss, delimiter), - fmt.Sprintf("Iron output\n%s\n%s\n%s", delimiter, irons, delimiter), + awsOutputStr, awsOutput := "No AWS lambda output", "" + if awss != nil { + awsOutput = string(awss.Bytes()) + awsOutputStr = fmt.Sprintf("AWS lambda output\n%s\n%s\n%s", delimiter, awsOutput, delimiter) } - notifyFailure(testName) - } else { - result <- []string{ - fmt.Sprintf("PASS %s", testName), + ironOutputStr, ironOutput := "No Iron output", "" + if irons != nil { + ironOutput = string(irons.Bytes()) + ironOutputStr = fmt.Sprintf("Iron output\n%s\n%s\n%s", delimiter, ironOutput, delimiter) } - } + + if elapsed { + result <- []string{ + fmt.Sprintf("FAIL %s Timeout elapsed!", testName), + awsOutputStr, + ironOutputStr, + } + notifyFailure(testName) + } else if awsOutput != ironOutput || awss == nil || irons == nil { + result <- []string{ + fmt.Sprintf("FAIL %s Output does not match!", testName), + awsOutputStr, + ironOutputStr, + } + notifyFailure(testName) + } else { + if awss == nil { + panic(testName + " " + awsOutputStr) + } + if irons == nil { + panic(testName + " " + ironOutputStr) + } + result <- []string{ + fmt.Sprintf("PASS %s", testName), + } + } + }() + + return result } diff --git a/test-suite/test.js b/test-suite/test.js new file mode 100644 index 0000000..20ce375 --- /dev/null +++ b/test-suite/test.js @@ -0,0 +1,15 @@ +var st = require('stack-trace'); + +function b() { + throw new Error("YO"); +} + +function a() { + b(); +} + +try { + a(); +} catch(e) { + console.log(st.parse(e)) +} diff --git a/test-suite/tests/java/test-resolution-type-bool/Makefile b/test-suite/tests/java/test-resolution-type-bool/Makefile new file mode 100644 index 0000000..e2feec3 --- /dev/null +++ b/test-suite/tests/java/test-resolution-type-bool/Makefile @@ -0,0 +1,4 @@ +build: + mvn package + cp target/test-resolution-type-bool-1.0.jar ./test-build.jar + go run ../../../tools/local-image/main.go $(PWD) diff --git a/test-suite/tests/java/test-resolution-type-bool/lambda.test b/test-suite/tests/java/test-resolution-type-bool/lambda.test new file mode 100644 index 0000000..61ffee4 --- /dev/null +++ b/test-suite/tests/java/test-resolution-type-bool/lambda.test @@ -0,0 +1,7 @@ +{ + "handler": "lambdatest.Hello::myHandler", + "runtime": "java8", + "name": "resolution-type-bool", + "description": "Pass boolean as a string and see which one is chosen. Looks like the first one in order is picked.", + "event": "true" +} diff --git a/test-suite/tests/java/test-resolution-type-bool/pom.xml b/test-suite/tests/java/test-resolution-type-bool/pom.xml new file mode 100644 index 0000000..b696aaa --- /dev/null +++ b/test-suite/tests/java/test-resolution-type-bool/pom.xml @@ -0,0 +1,46 @@ + + 4.0.0 + + io.iron.lambda.tests + test-resolution-type-bool + jar + 1.0 + + + + com.amazonaws + aws-lambda-java-core + 1.1.0 + + + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.0 + + 1.8 + 1.8 + + + + org.apache.maven.plugins + maven-shade-plugin + 2.4.3 + + false + + + + package + + shade + + + + + + + diff --git a/test-suite/tests/java/test-resolution-type-bool/src/main/java/lambdatest/Hello.java b/test-suite/tests/java/test-resolution-type-bool/src/main/java/lambdatest/Hello.java new file mode 100644 index 0000000..35cd112 --- /dev/null +++ b/test-suite/tests/java/test-resolution-type-bool/src/main/java/lambdatest/Hello.java @@ -0,0 +1,20 @@ +package lambdatest; + +import com.amazonaws.services.lambda.runtime.Context; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +public class Hello { + public static void myHandler(boolean b) { + System.out.println("Boolean " + b); + } + + public static void myHandler(String s) { + System.out.println("String " + s); + } +} diff --git a/test-suite/tests/java/test-resolution-type-int/Makefile b/test-suite/tests/java/test-resolution-type-int/Makefile new file mode 100644 index 0000000..352ff20 --- /dev/null +++ b/test-suite/tests/java/test-resolution-type-int/Makefile @@ -0,0 +1,4 @@ +build: + mvn package + cp target/test-resolution-type-int-1.0.jar ./test-build.jar + go run ../../../tools/local-image/main.go $(PWD) diff --git a/test-suite/tests/java/test-resolution-type-int/lambda.test b/test-suite/tests/java/test-resolution-type-int/lambda.test new file mode 100644 index 0000000..943a188 --- /dev/null +++ b/test-suite/tests/java/test-resolution-type-int/lambda.test @@ -0,0 +1,7 @@ +{ + "handler": "lambdatest.Hello::myHandler", + "runtime": "java8", + "name": "resolution-type-int", + "description": "Pass a bad number as a string to a int handler. Should fail due to JSON parse error. Unfortunately output will not match due to different JSON libs.", + "event": "4b5d" +} diff --git a/test-suite/tests/java/test-resolution-type-int/pom.xml b/test-suite/tests/java/test-resolution-type-int/pom.xml new file mode 100644 index 0000000..7e282e0 --- /dev/null +++ b/test-suite/tests/java/test-resolution-type-int/pom.xml @@ -0,0 +1,46 @@ + + 4.0.0 + + io.iron.lambda.tests + test-resolution-type-int + jar + 1.0 + + + + com.amazonaws + aws-lambda-java-core + 1.1.0 + + + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.0 + + 1.8 + 1.8 + + + + org.apache.maven.plugins + maven-shade-plugin + 2.4.3 + + false + + + + package + + shade + + + + + + + diff --git a/test-suite/tests/java/test-resolution-type-int/src/main/java/lambdatest/Hello.java b/test-suite/tests/java/test-resolution-type-int/src/main/java/lambdatest/Hello.java new file mode 100644 index 0000000..b15dd94 --- /dev/null +++ b/test-suite/tests/java/test-resolution-type-int/src/main/java/lambdatest/Hello.java @@ -0,0 +1,16 @@ +package lambdatest; + +import com.amazonaws.services.lambda.runtime.Context; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +public class Hello { + public static void myHandler(int b) { + System.out.println("Integer " + b); + } +} diff --git a/test-suite/tests/java/test-resolution-type-list-number/Makefile b/test-suite/tests/java/test-resolution-type-list-number/Makefile new file mode 100644 index 0000000..d6b0526 --- /dev/null +++ b/test-suite/tests/java/test-resolution-type-list-number/Makefile @@ -0,0 +1,4 @@ +build: + mvn package + cp target/test-resolution-type-list-number-1.0.jar ./test-build.jar + go run ../../../tools/local-image/main.go $(PWD) diff --git a/test-suite/tests/java/test-resolution-type-list-number/lambda.test b/test-suite/tests/java/test-resolution-type-list-number/lambda.test new file mode 100644 index 0000000..5dc1eaa --- /dev/null +++ b/test-suite/tests/java/test-resolution-type-list-number/lambda.test @@ -0,0 +1,7 @@ +{ + "handler": "lambdatest.Hello::myHandler", + "runtime": "java8", + "name": "resolution-type-list-number", + "description": "Pass list of numbers, should work.", + "event": [1, 3.14, -2, 21453523, -2e24] +} diff --git a/test-suite/tests/java/test-resolution-type-list-number/pom.xml b/test-suite/tests/java/test-resolution-type-list-number/pom.xml new file mode 100644 index 0000000..103377c --- /dev/null +++ b/test-suite/tests/java/test-resolution-type-list-number/pom.xml @@ -0,0 +1,46 @@ + + 4.0.0 + + io.iron.lambda.tests + test-resolution-type-list-number + jar + 1.0 + + + + com.amazonaws + aws-lambda-java-core + 1.1.0 + + + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.0 + + 1.8 + 1.8 + + + + org.apache.maven.plugins + maven-shade-plugin + 2.4.3 + + false + + + + package + + shade + + + + + + + diff --git a/test-suite/tests/java/test-resolution-type-list-number/src/main/java/lambdatest/Hello.java b/test-suite/tests/java/test-resolution-type-list-number/src/main/java/lambdatest/Hello.java new file mode 100644 index 0000000..a50826b --- /dev/null +++ b/test-suite/tests/java/test-resolution-type-list-number/src/main/java/lambdatest/Hello.java @@ -0,0 +1,18 @@ +package lambdatest; + +import com.amazonaws.services.lambda.runtime.Context; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +public class Hello { + public static void myHandler(List l) { + for (Double i : l) { + System.out.println(i); + } + } +} diff --git a/test-suite/tests/java/test-resolution-type-list-pojo/Makefile b/test-suite/tests/java/test-resolution-type-list-pojo/Makefile new file mode 100644 index 0000000..09b360a --- /dev/null +++ b/test-suite/tests/java/test-resolution-type-list-pojo/Makefile @@ -0,0 +1,4 @@ +build: + mvn package + cp target/test-resolution-type-list-pojo-1.0.jar ./test-build.jar + go run ../../../tools/local-image/main.go $(PWD) diff --git a/test-suite/tests/java/test-resolution-type-list-pojo/lambda.test b/test-suite/tests/java/test-resolution-type-list-pojo/lambda.test new file mode 100644 index 0000000..9d9b560 --- /dev/null +++ b/test-suite/tests/java/test-resolution-type-list-pojo/lambda.test @@ -0,0 +1,22 @@ +{ + "handler": "lambdatest.Hello::myHandler", + "runtime": "java8", + "name": "resolution-type-list-pojo", + "description": "Pass a list of deserializable POJO.", + "event": [{ + "symbol": "Fe", + "name": "Iron", + "atomicNumber": 26, + "foundOnEarth": true, + "configuration": [2, 8, 14, 2], + "halfLife": { "54": 3.1e22, "55": 2.73, "56": "stable" } + }, + { + "symbol": "Ti", + "name": "Titanium", + "atomicNumber": 22, + "foundOnEarth": true, + "configuration": [2, 8, 10, 2], + "halfLife": { "44": 63, "46": "stable" } + }] +} diff --git a/test-suite/tests/java/test-resolution-type-list-pojo/pom.xml b/test-suite/tests/java/test-resolution-type-list-pojo/pom.xml new file mode 100644 index 0000000..2c588c5 --- /dev/null +++ b/test-suite/tests/java/test-resolution-type-list-pojo/pom.xml @@ -0,0 +1,46 @@ + + 4.0.0 + + io.iron.lambda.tests + test-resolution-type-list-pojo + jar + 1.0 + + + + com.amazonaws + aws-lambda-java-core + 1.1.0 + + + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.0 + + 1.8 + 1.8 + + + + org.apache.maven.plugins + maven-shade-plugin + 2.4.3 + + false + + + + package + + shade + + + + + + + diff --git a/test-suite/tests/java/test-resolution-type-list-pojo/src/main/java/lambdatest/Hello.java b/test-suite/tests/java/test-resolution-type-list-pojo/src/main/java/lambdatest/Hello.java new file mode 100644 index 0000000..c1f59b2 --- /dev/null +++ b/test-suite/tests/java/test-resolution-type-list-pojo/src/main/java/lambdatest/Hello.java @@ -0,0 +1,82 @@ +package lambdatest; + +import com.amazonaws.services.lambda.runtime.Context; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +public class Hello { + public static class Atom { + String symbol; + String name; + int atomicNumber; + boolean foundOnEarth; + List configuration; + Map halfLife; + + public String getSymbol() { + return symbol; + } + + public String getName() { + return name; + } + + public int getAtomicNumber() { + return atomicNumber; + } + + public boolean getFoundOnEarth() { + return foundOnEarth; + } + + public List getConfiguration() { + return configuration; + } + + public Map getHalfLife() { + return halfLife; + } + + public void setSymbol(String s) { + symbol = s; + } + + public void setName(String n) { + name = n; + } + + public void setAtomicNumber(int a) { + atomicNumber = a; + } + + public void setFoundOnEarth(boolean f) { + foundOnEarth = f; + } + + public void setConfiguration(List l) { + configuration = l; + } + + public void setHalfLife(Map m) { + halfLife = m; + } + + } + + public static void myHandler(List l, Context c) { + for (Atom a : l) { + System.out.println("Symbol: " + a.symbol); + System.out.println("Name: " + a.name); + System.out.println("Atomic Number: " + a.atomicNumber); + System.out.println("Found on Earth: " + a.foundOnEarth); + System.out.println("Configuration: " + a.configuration); + System.out.println("Half lives: " + a.halfLife); + System.out.println("---"); + } + } +} diff --git a/test-suite/tests/java/test-resolution-type-pojo/Makefile b/test-suite/tests/java/test-resolution-type-pojo/Makefile new file mode 100644 index 0000000..90bdcbc --- /dev/null +++ b/test-suite/tests/java/test-resolution-type-pojo/Makefile @@ -0,0 +1,4 @@ +build: + mvn package + cp target/test-resolution-type-pojo-1.0.jar ./test-build.jar + go run ../../../tools/local-image/main.go $(PWD) diff --git a/test-suite/tests/java/test-resolution-type-pojo/lambda.test b/test-suite/tests/java/test-resolution-type-pojo/lambda.test new file mode 100644 index 0000000..2cbb395 --- /dev/null +++ b/test-suite/tests/java/test-resolution-type-pojo/lambda.test @@ -0,0 +1,14 @@ +{ + "handler": "lambdatest.Hello::myHandler", + "runtime": "java8", + "name": "resolution-type-pojo", + "description": "Pass a deserializable POJO.", + "event": { + "symbol": "Fe", + "name": "Iron", + "atomicNumber": 26, + "foundOnEarth": true, + "configuration": [2, 8, 14, 2], + "halfLife": { "54": 3.1e22, "55": 2.73, "56": "stable" } + } +} diff --git a/test-suite/tests/java/test-resolution-type-pojo/pom.xml b/test-suite/tests/java/test-resolution-type-pojo/pom.xml new file mode 100644 index 0000000..1b8e2da --- /dev/null +++ b/test-suite/tests/java/test-resolution-type-pojo/pom.xml @@ -0,0 +1,46 @@ + + 4.0.0 + + io.iron.lambda.tests + test-resolution-type-pojo + jar + 1.0 + + + + com.amazonaws + aws-lambda-java-core + 1.1.0 + + + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.0 + + 1.8 + 1.8 + + + + org.apache.maven.plugins + maven-shade-plugin + 2.4.3 + + false + + + + package + + shade + + + + + + + diff --git a/test-suite/tests/java/test-resolution-type-pojo/src/main/java/lambdatest/Hello.java b/test-suite/tests/java/test-resolution-type-pojo/src/main/java/lambdatest/Hello.java new file mode 100644 index 0000000..a243edf --- /dev/null +++ b/test-suite/tests/java/test-resolution-type-pojo/src/main/java/lambdatest/Hello.java @@ -0,0 +1,79 @@ +package lambdatest; + +import com.amazonaws.services.lambda.runtime.Context; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +public class Hello { + public static class Atom { + String symbol; + String name; + int atomicNumber; + boolean foundOnEarth; + List configuration; + Map halfLife; + + public String getSymbol() { + return symbol; + } + + public String getName() { + return name; + } + + public int getAtomicNumber() { + return atomicNumber; + } + + public boolean getFoundOnEarth() { + return foundOnEarth; + } + + public List getConfiguration() { + return configuration; + } + + public Map getHalfLife() { + return halfLife; + } + + public void setSymbol(String s) { + symbol = s; + } + + public void setName(String n) { + name = n; + } + + public void setAtomicNumber(int a) { + atomicNumber = a; + } + + public void setFoundOnEarth(boolean f) { + foundOnEarth = f; + } + + public void setConfiguration(List l) { + configuration = l; + } + + public void setHalfLife(Map m) { + halfLife = m; + } + + } + + public static void myHandler(Atom a, Context c) { + System.out.println("Symbol: " + a.symbol); + System.out.println("Name: " + a.name); + System.out.println("Atomic Number: " + a.atomicNumber); + System.out.println("Found on Earth: " + a.foundOnEarth); + System.out.println("Configuration: " + a.configuration); + System.out.println("Half lives: " + a.halfLife); + } +} diff --git a/test-suite/tests/node/test-context-done-succeed/test.js b/test-suite/tests/node/test-context-done-succeed/test.js index 958edf8..24c9e23 100644 --- a/test-suite/tests/node/test-context-done-succeed/test.js +++ b/test-suite/tests/node/test-context-done-succeed/test.js @@ -2,4 +2,5 @@ var fs = require('fs'); exports.run = function(event, context) { fs.readFile("./test.js", context.done); + console.log("done"); } diff --git a/test-suite/tests/node/test-context-fail-undefined/test.js b/test-suite/tests/node/test-context-fail-undefined/test.js index 8848532..0037961 100644 --- a/test-suite/tests/node/test-context-fail-undefined/test.js +++ b/test-suite/tests/node/test-context-fail-undefined/test.js @@ -1,3 +1,4 @@ exports.run = function(event, context) { context.fail(); + console.log("done"); } diff --git a/test-suite/tests/node/test-context-succeed-undefined/test.js b/test-suite/tests/node/test-context-succeed-undefined/test.js index 344980f..6ac3418 100644 --- a/test-suite/tests/node/test-context-succeed-undefined/test.js +++ b/test-suite/tests/node/test-context-succeed-undefined/test.js @@ -1,3 +1,4 @@ exports.run = function(event, context) { context.succeed(); + console.log('done') } diff --git a/test-suite/tests/node/test-context/test.js b/test-suite/tests/node/test-context/test.js index 0de4e8f..dc748bb 100644 --- a/test-suite/tests/node/test-context/test.js +++ b/test-suite/tests/node/test-context/test.js @@ -38,4 +38,6 @@ exports.run = function(event, context) { assert.ok(typeof context.awsRequestId == 'string' && context.awsRequestId.length > 0, "context.awsRequestId is defined."); assert.ok(typeof context.memoryLimitInMB == 'string' && parseInt(context.memoryLimitInMB) >= 0, "context.memoryLimitInMB is a string representing a non-negative number."); + + context.done('ok') } diff --git a/test-suite/tests/node/test-uuid/test.js b/test-suite/tests/node/test-uuid/test.js index 236e5f6..9e07e5f 100644 --- a/test-suite/tests/node/test-uuid/test.js +++ b/test-suite/tests/node/test-uuid/test.js @@ -1,4 +1,5 @@ var uuid = require('uuid') exports.run = function(event, context) { - context.succeed(uuid.v4()) + context.succeed(uuid.v4()) + console.log("done"); } diff --git a/test-suite/tests/node/test-whitespace/test.js b/test-suite/tests/node/test-whitespace/test.js index 45de8ed..b7175aa 100644 --- a/test-suite/tests/node/test-whitespace/test.js +++ b/test-suite/tests/node/test-whitespace/test.js @@ -1,4 +1,5 @@ var i = require('./this is an import'); exports.run = function(event, context) { + console.log("start") context.succeed(i.answer); } diff --git a/test-suite/tests/python/test-awssdk/lambda.test b/test-suite/tests/python/test-awssdk/lambda.test index 604aac7..1c3bc69 100644 --- a/test-suite/tests/python/test-awssdk/lambda.test +++ b/test-suite/tests/python/test-awssdk/lambda.test @@ -4,7 +4,6 @@ "name": "awssdk", "description": "Test that AWS SDK is available and works by attempting to create a bucket and write to it on S3.", "event": { - "bucket": "lambda-python2.7-test-bucket", - "region": "us-west-2" + "bucket": "lambda-python2.7-test-bucket-1" } } diff --git a/test-suite/tests/python/test-awssdk/test.py b/test-suite/tests/python/test-awssdk/test.py index e6e219b..701f6d2 100644 --- a/test-suite/tests/python/test-awssdk/test.py +++ b/test-suite/tests/python/test-awssdk/test.py @@ -7,20 +7,14 @@ def run(event, context): print ("Test start") s3 = boto3.resource('s3') - bucket = s3.Bucket(Bucket=event['bucket']) + print ("Creating object ...") + obj = s3.Object( + bucket_name=event['bucket'], + key='myKey-' + str(uuid.uuid4()) + ) - if bucket.creation_date is None: - print ("Creating bucket ...") - bucket.create( - ACL='private', - CreateBucketConfiguration={'LocationConstraint': event['region']} - ) - bucket.wait_until_exists() - - print ("Adding object ...") - obj = bucket.put_object( - Key='myKey-'+str(uuid.uuid4()), - Body='Hello!') + print ("Putting object value ...") + obj.put(Body='Hello!') print ("Deleting object ...") obj.delete() diff --git a/test-suite/tests/python/test-context-log-func/lambda.test b/test-suite/tests/python/test-context-log-func/lambda.test index f31ed9c..1a324da 100644 --- a/test-suite/tests/python/test-context-log-func/lambda.test +++ b/test-suite/tests/python/test-context-log-func/lambda.test @@ -3,6 +3,5 @@ "runtime": "python2.7", "name": "context-log-func", "description": "Test context.log method compatibility.", - "event" : {}, - "timeout" : 5 + "event" : {} } diff --git a/test-suite/tests/python/test-context-meta/lambda.test b/test-suite/tests/python/test-context-meta/lambda.test index a54021c..31ddb47 100644 --- a/test-suite/tests/python/test-context-meta/lambda.test +++ b/test-suite/tests/python/test-context-meta/lambda.test @@ -3,6 +3,5 @@ "runtime": "python2.7", "name": "context-meta", "description": "Test context object compatibility.", - "event" : {}, - "timeout" : 5 + "event" : {} } diff --git a/test-suite/tests/python/test-context-values/lambda.test b/test-suite/tests/python/test-context-values/lambda.test index 5f96f7c..40fbc5e 100644 --- a/test-suite/tests/python/test-context-values/lambda.test +++ b/test-suite/tests/python/test-context-values/lambda.test @@ -3,6 +3,5 @@ "runtime": "python2.7", "name": "context-values", "description": "Test context object compatibility #2.", - "event" : {}, - "timeout" : 5 + "event" : {} } diff --git a/test-suite/tests/python/test-context-values/test.py b/test-suite/tests/python/test-context-values/test.py index ca48a88..f44b5e8 100644 --- a/test-suite/tests/python/test-context-values/test.py +++ b/test-suite/tests/python/test-context-values/test.py @@ -5,7 +5,7 @@ def run(event, context): print("Function name is set:", not (context.function_name is None)) print("Function version is set:", not (context.function_version is None)) - print("Memory limit in MB (grater or equal than 100):", context.memory_limit_in_mb > 100) + print("Memory limit in MB is positive:", context.memory_limit_in_mb > 0) print("AWS request ID is set:", not (context.aws_request_id is None)) remaining1 = context.get_remaining_time_in_millis() diff --git a/test-suite/tests/python/test-event-as-a-dict/lambda.test b/test-suite/tests/python/test-event-as-a-dict/lambda.test index bdff1a7..44894dc 100644 --- a/test-suite/tests/python/test-event-as-a-dict/lambda.test +++ b/test-suite/tests/python/test-event-as-a-dict/lambda.test @@ -11,6 +11,5 @@ "msg":"i'm an object" }, "arr" : [1, "a string 2", null] - }, - "timeout" : 5 + } } diff --git a/test-suite/tests/python/test-event-integer/lambda.test b/test-suite/tests/python/test-event-integer/lambda.test index 96e876e..1d99a91 100644 --- a/test-suite/tests/python/test-event-integer/lambda.test +++ b/test-suite/tests/python/test-event-integer/lambda.test @@ -3,6 +3,5 @@ "runtime": "python2.7", "name": "event-integer", "description": "Integer payload", - "event" : 4294967296, - "timeout" : 5 + "event" : 4294967296 } diff --git a/test-suite/tests/python/test-event-meta/lambda.test b/test-suite/tests/python/test-event-meta/lambda.test index 1d721dc..697a0bf 100644 --- a/test-suite/tests/python/test-event-meta/lambda.test +++ b/test-suite/tests/python/test-event-meta/lambda.test @@ -3,6 +3,5 @@ "runtime": "python2.7", "name": "event-meta", "description": "Extracts event attributes", - "event" : {}, - "timeout" : 5 + "event" : {} } diff --git a/test-suite/tests/python/test-event-null/lambda.test b/test-suite/tests/python/test-event-null/lambda.test index 0bc4034..0efed71 100644 --- a/test-suite/tests/python/test-event-null/lambda.test +++ b/test-suite/tests/python/test-event-null/lambda.test @@ -3,6 +3,5 @@ "runtime": "python2.7", "name": "event-null", "description": "Test behavior when event set to null", - "event" : null, - "timeout" : 5 + "event" : null } diff --git a/test-suite/tests/python/test-event-undefined/lambda.test b/test-suite/tests/python/test-event-undefined/lambda.test index a0da9e9..e91d2b7 100644 --- a/test-suite/tests/python/test-event-undefined/lambda.test +++ b/test-suite/tests/python/test-event-undefined/lambda.test @@ -2,6 +2,5 @@ "handler": "test.run", "runtime": "python2.7", "name": "event-undefined", - "description": "Test behavior when event it not specified", - "timeout" : 5 + "description": "Test behavior when event is not specified" } diff --git a/test-suite/tests/python/test-logging-checkoutput/lambda.test b/test-suite/tests/python/test-logging-checkoutput/lambda.test index 8acf199..987141b 100644 --- a/test-suite/tests/python/test-logging-checkoutput/lambda.test +++ b/test-suite/tests/python/test-logging-checkoutput/lambda.test @@ -3,6 +3,5 @@ "runtime": "python2.7", "name": "context-logging-checkoutput", "description": "Test default logging format compatibility.", - "event" : {}, - "timeout" : 5 + "event" : {} } diff --git a/test-suite/tests/python/test-logging-meta/lambda.test b/test-suite/tests/python/test-logging-meta/lambda.test index 34489dd..c3a413d 100644 --- a/test-suite/tests/python/test-logging-meta/lambda.test +++ b/test-suite/tests/python/test-logging-meta/lambda.test @@ -3,6 +3,5 @@ "runtime": "python2.7", "name": "logging-meta", "description": "Test default logging config compatibility.", - "event" : {}, - "timeout" : 5 + "event" : {} } diff --git a/test-suite/tests/python/test-versions/lambda.test b/test-suite/tests/python/test-versions/lambda.test index 55a0bd7..dfc9243 100644 --- a/test-suite/tests/python/test-versions/lambda.test +++ b/test-suite/tests/python/test-versions/lambda.test @@ -3,6 +3,5 @@ "runtime": "python2.7", "name": "versions", "description": "Test installed package versions.", - "event" : {}, - "timeout" : 15 + "event" : {} } diff --git a/test-suite/tools/add-test/main.go b/test-suite/tools/add-test/main.go index 0ef00e1..0845a42 100644 --- a/test-suite/tools/add-test/main.go +++ b/test-suite/tools/add-test/main.go @@ -16,6 +16,7 @@ import ( "github.com/aws/aws-sdk-go/aws/credentials" "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/service/lambda" + "github.com/docker/docker/pkg/jsonmessage" iron_lambda "github.com/iron-io/lambda/lambda" "github.com/iron-io/lambda/test-suite/util" "github.com/satori/go.uuid" @@ -100,6 +101,7 @@ func createLambdaFunction(l *lambda.Lambda, code []byte, runtime, role, name, ha configInput := &lambda.UpdateFunctionConfigurationInput{ FunctionName: aws.String(name), + Handler: aws.String(handler), Timeout: aws.Int64(int64(timeout)), } resp, err = l.UpdateFunctionConfiguration(configInput) @@ -144,6 +146,24 @@ func addToLambda(dir string) error { return err } +type DockerJsonWriter struct { + under io.Writer + w io.Writer +} + +func NewDockerJsonWriter(under io.Writer) *DockerJsonWriter { + r, w := io.Pipe() + go func() { + err := jsonmessage.DisplayJSONMessagesStream(r, under, 1, true, nil) + log.Fatal(err) + }() + return &DockerJsonWriter{under, w} +} + +func (djw *DockerJsonWriter) Write(p []byte) (int, error) { + return djw.w.Write(p) +} + func addToIron(dir string) error { desc, err := util.ReadTestDescription(dir) if err != nil { @@ -158,13 +178,13 @@ func addToIron(dir string) error { return err } - opts := iron_lambda.PushImageOptions{imageNameVersion, os.Stdout, false} + opts := iron_lambda.PushImageOptions{imageNameVersion, NewDockerJsonWriter(os.Stdout), true} err = iron_lambda.PushImage(opts) if err != nil { return err } - return iron_lambda.RegisterWithIron(imageNameVersion, credentials.NewEnvCredentials()) + return iron_lambda.RegisterWithIron(imageNameVersion) } type RegisterOn struct { diff --git a/test-suite/util/channel-utils.go b/test-suite/util/channel-utils.go new file mode 100644 index 0000000..c90ee85 --- /dev/null +++ b/test-suite/util/channel-utils.go @@ -0,0 +1,58 @@ +package util + +import ( + "fmt" +) + +// Forwards messages from source channel to target channnel with prefix +// Target <- + Source +// Return func could be used to forward remaining messages to the output channel before closing it +// Example: +// defer close(target) +// source := someFunc() +// defer util.ForwardInBackground("Prefix", source, target)() +func ForwardInBackground(prefix string, source <-chan string, target chan<- []string) func() { + f := func() { + for data := range source { + prefixed := fmt.Sprintf("%s%s", prefix, data) + target <- []string{prefixed} + } + } + go func() { + defer func() { recover() }() + f() + }() + + return f +} + +// Concatenate two channels in async way, i.e., any message comming from a or b passes to result without any change +func JoinChannels(a <-chan []string, b <-chan []string) <-chan []string { + if a == nil { + return b + } + if b == nil { + return a + } + r := make(chan []string) + go func() { + defer close(r) + for a != nil || b != nil { + select { + case item, ok := <-a: + if ok { + r <- item + } else { + a = nil + } + case item, ok := <-b: + if ok { + r <- item + } else { + b = nil + } + } + } + }() + return r +} diff --git a/test-suite/util/semaphore.go b/test-suite/util/semaphore.go new file mode 100644 index 0000000..e5db7f3 --- /dev/null +++ b/test-suite/util/semaphore.go @@ -0,0 +1,24 @@ +package util + +type empty struct{} +type Semaphore chan empty + +func NewSemaphore(size int) Semaphore { + return make(Semaphore, size) +} + +func (s Semaphore) Lock() { + s <- empty{} +} + +func (s Semaphore) Unlock() { + <-s +} + +func (s Semaphore) Available() int { + return len(s) +} + +func (s Semaphore) Size() int { + return cap(s) +} diff --git a/test-suite/util/util.go b/test-suite/util/util.go index 990dbfd..701968a 100644 --- a/test-suite/util/util.go +++ b/test-suite/util/util.go @@ -7,6 +7,7 @@ import ( "io/ioutil" "os" "path/filepath" + "regexp" "strings" iron_lambda "github.com/iron-io/lambda/lambda" @@ -21,7 +22,7 @@ type TestDescription struct { // The test's timeout in seconds, valid timeout as imposed by Lambda // is between 1 and 300 inclusive. - // If no Timeout is specified the 30 sec default is used + // If no Timeout is specified the 60 sec default is used Timeout int } @@ -40,7 +41,7 @@ func ReadTestDescription(dir string) (*TestDescription, error) { desc.Name = fmt.Sprintf("lambda-test-suite-%s-%s", normalizedRuntime, desc.Name) if desc.Timeout == 0 { - desc.Timeout = 30 + desc.Timeout = 60 } else if desc.Timeout < 1 { desc.Timeout = 1 } else if desc.Timeout > 300 { @@ -104,22 +105,39 @@ func MakeImage(dir string, desc *TestDescription, imageNameVersion string) error return err } -func RemoveTimestampAndRequestIdFromLogLine(line, requestId string) string { - if requestId != "" { - parts := strings.Fields(line) - - // assume timestamp is before request_id - for i, p := range parts { - if p == requestId { - ts := parts[i-1] - if strings.HasSuffix(ts, "Z") && strings.HasPrefix(ts, "20") { - line = strings.Replace(line, ts, "", 1) - } - line = strings.Replace(line, parts[i], "", 1) - break +func RemoveTimestampAndRequestIdFromAwsLogLine(line, requestId string) (string, bool) { + return removeTimestampAndRequestIdFromLogLine(line, requestId, awsRequestIdRegexp) +} +func RemoveTimestampAndRequestIdFromIronLogLine(line, requestId string) (string, bool) { + return removeTimestampAndRequestIdFromLogLine(line, requestId, ironRequestIdRegexp) +} + +var ironRequestIdRegexp *regexp.Regexp = regexp.MustCompile("^[0-9a-fA-F]{24}$") +var awsRequestIdRegexp *regexp.Regexp = regexp.MustCompile("^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$") +var timestampRegexp *regexp.Regexp = regexp.MustCompile("^20\\d{2}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}\\.\\d{3,6}Z") + +func removeTimestampAndRequestIdFromLogLine(line, requestId string, requestIdRegexp *regexp.Regexp) (string, bool) { + sep := "\t" + parts := strings.Split(line, sep) + + // assume timestamp is before request_id + for i, p := range parts { + if p == requestId { + hasTimeStamp := i > 0 && timestampRegexp.MatchString(parts[i-1]) + if hasTimeStamp { + parts = append(parts[:i-1], parts[i+1:]...) + } else { + parts = append(parts[:i], parts[i+1:]...) } + return strings.Join(parts, sep), true } } - return line + //remove a log line from another request_id + for _, p := range parts { + if requestIdRegexp.MatchString(p) { + return "", false + } + } + return line, true }