Merge pull request aws-samples#1609 from awsrahulrsr/main

Author: boyney123

kinesis-lambda-terraform/README.md

@@ -0,0 +1,86 @@
+# AWS Kinesis Data Streams to AWS Lambda
+
+This pattern creates an AWS Kinesis Data Stream, a stream consumer, and an AWS Lambda function. When data is added to the stream, the Lambda function is invoked.
+
+Learn more about this pattern at Serverless Land Patterns: [https://serverlessland.com/patterns/kinesis-to-lambda-terraform](https://serverlessland.com/patterns/kinesis-to-lambda-terraform)
+
+Important: this application uses various AWS services and there are costs associated with these services after the Free Tier usage - please see the [AWS Pricing page](https://aws.amazon.com/pricing/) for details. You are responsible for any AWS costs incurred. No warranty is implied in this example.
+
+## Requirements
+
+* [Create an AWS account](https://portal.aws.amazon.com/gp/aws/developer/registration/index.html) if you do not already have one and log in. The IAM user that you use must have sufficient permissions to make necessary AWS service calls and manage AWS resources.
+* [AWS CLI](https://docs.aws.amazon.com/cli/latest/userguide/install-cliv2.html) installed and configured
+* [Git Installed](https://git-scm.com/book/en/v2/Getting-Started-Installing-Git)
+* [Terraform](https://learn.hashicorp.com/tutorials/terraform/install-cli?in=terraform/aws-get-started) installed
+
+## Deployment Instructions
+
+1. Clone the project to your local working directory
+
+   ```sh
+   git clone https://github.com/aws-samples/serverless-patterns/ 
+   ```
+
+2. Change the working directory to this pattern's directory
+
+   ```sh
+   cd serverless-patterns/kinesis-lambda-terraform
+   ```
+
+1. From the command line, initialize terraform to  to downloads and installs the providers defined in the configuration:
+    ```
+    terraform init
+    ```
+1. From the command line, apply the configuration in the main.tf file:
+    ```
+    terraform apply
+    ```
+1. During the prompts:
+
+   - Enter yes
+
+## Testing
+
+Use the Amazon Kinesis Data Generator for testing. The easiest way to use this tool is to use the [hosted generator](https://awslabs.github.io/amazon-kinesis-data-generator/web/producer.html) and follow the [setup instructions](https://awslabs.github.io/amazon-kinesis-data-generator/web/help.html).
+
+After you have the generator configured, you should have a custom URL to generate data for your Kinesis data stream. In your configuration steps, you created a username and password. Log in to the generator using those credentials.
+
+When you are logged in, you can generate data for your stream test.
+
+1. Choose the region you deployed the application to
+2. For Stream/delivery stream, select your stream.
+3. For Records per second, keep the default value of 100.
+4. On the Template 1 tab, name the template Sensor1.
+5. Use the following template:
+    ```JSON
+    {
+        "sensorId": {{random.number(50)}},
+        "currentTemperature": {{random.number(
+            {
+                "min":10,
+                "max":150
+            }
+        )}},
+        "status": "{{random.arrayElement(
+            ["OK","FAIL","WARN"]
+        )}}"
+    }
+    ```
+6. Choose Send Data.
+7. After several seconds, choose Stop Sending Data.
+
+## Cleanup
+
+1. Delete all created resources by terraform
+    ```bash
+    terraform destroy
+    ```
+2. During the prompts:
+    * Enter yes
+3. Confirm all created resources has been deleted
+    ```bash
+    terraform show
+    ```
+
+----
+Copyright 2023 Amazon.com, Inc. or its affiliates. All Rights Reserved.

kinesis-lambda-terraform/example-pattern.json

@@ -0,0 +1,58 @@
+{
+    "title": "Kinesis Data Streams With Lambda Consumer",
+    "description": "When data is loaded into Kinesis data stream, the configured lambda function is invoked.",
+    "language": "",
+    "level": "200",
+    "framework": "Terraform",
+    "introBox": {
+      "headline": "How it works",
+      "text": [
+        "This pattern creates a Kinesis Data Stream and lambda function (consumer) with event source mapping. ",
+        "When the data is loaded into the stream, the lambda function is invoked. "
+      ]
+    },
+    "gitHub": {
+      "template": {
+        "repoURL": "https://github.com/aws-samples/serverless-patterns/tree/main/kinesis-lambda-terraform",
+        "templateURL": "serverless-patterns/kinesis-lambda-terraform",
+        "projectFolder": "kinesis-lambda-terraform",
+        "templateFile": "main.tf"
+      }
+    },
+    "resources": {
+      "bullets": [
+        {
+          "text": "Kinesis Data Streams with Lambda consumers",
+          "link": "https://docs.aws.amazon.com/lambda/latest/dg/with-kinesis.html"
+        },
+        {
+          "text": "Lambda event source mapping",
+          "link": "https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/lambda_event_source_mapping.html"
+        }
+      ]
+    },
+    "deploy": {
+      "text": [
+        "terraform apply"
+      ]
+    },
+    "testing": {
+      "text": [
+        "See the GitHub repo for detailed testing instructions."
+      ]
+    },
+    "cleanup": {
+      "text": [
+        "Delete the stack: <code>terraform destroy</code>."
+      ]
+    },
+    "authors": [
+      {
+        "name": "Rahul Sringeri",
+        "image": "",
+        "bio": "Technical Account Manager at AWS for strategic accounts.",
+        "linkedin": "",
+        "twitter": ""
+      }
+    ]
+  }

kinesis-lambda-terraform/main.tf

@@ -0,0 +1,57 @@
+terraform {
+  required_providers {
+    aws = {
+      source  = "hashicorp/aws"
+      version = "~> 4.22"
+    }
+  }
+
+  required_version = ">= 0.14.9"
+}
+
+provider "aws" {
+  region = "us-east-1"  # Change to your desired region
+}
+resource "aws_kinesis_stream" "sample_stream" {
+  name             = "sample-stream"
+  shard_count      = 1
+  retention_period = 24
+}
+resource "aws_lambda_function" "sample_lambda" {
+  filename         = "sample_lambda.zip"  # Path to your Lambda code ZIP file
+  function_name    = "sample-lambda"
+  role             = aws_iam_role.lambda_role.arn
+  handler          = "index.handler"
+  runtime          = "nodejs14.x"  # Change to your preferred runtime
+}
+resource "aws_iam_role" "lambda_role" {
+  name = "lambda-role"
+  assume_role_policy = jsonencode({
+    Version = "2012-10-17",
+    Statement = [
+      {
+        Action = "sts:AssumeRole",
+        Effect = "Allow",
+        Principal = {
+          Service = "lambda.amazonaws.com"
+        }
+      }
+    ]
+  })
+}
+
+resource "aws_lambda_event_source_mapping" "sample_mapping" {
+  event_source_arn = aws_kinesis_stream.sample_stream.arn
+  function_name    = aws_lambda_function.sample_lambda.arn
+  starting_position = "LATEST"
+}
+
+output "kinesis_data_stream" {
+    value = aws_kinesis_stream.sample_stream.arn
+    description = "Kinesis data stream with shards"
+}
+
+output "consumer_function" {
+  value = aws_lambda_function.sample_function.arn
+  description = "Consumer Function function name"
+}

kinesis-lambda-terraform/src/app.js

@@ -0,0 +1,15 @@
+//Sample lambda code
+
+exports.handler = async (event) => {
+    console.log('Received Kinesis records:', JSON.stringify(event, null, 2));
+    // Process each Kinesis record
+    event.Records.forEach((record) => {
+      const payload = Buffer.from(record.kinesis.data, 'base64').toString('utf-8');
+      console.log('Record payload:', payload);
+      // Add your processing logic here
+    });
+    return {
+      statusCode: 200,
+      body: 'Processed Kinesis records successfully',
+    };
+  };