📚 examples

Author: Guy7B

.gitignore

@@ -4,4 +4,4 @@ dist
 local
 .vscode
 .DS_Store
-examples
+.serverless

examples/serverless-localstack/.env.local

@@ -0,0 +1,10 @@
+LOCALSTACK_URL=http://localstack:4566
+LOCALSTACK_HOSTNAME=localstack
+DEPLOYMENT_STAGE=local
+AWS_REGION=us-east-1
+AWS_ACCESS_KEY_ID=dummy
+AWS_SECRET_ACCESS_KEY=dummy
+AWS_SESSION_TOKEN=dummy
+AWS_CUSTOM_ENDPOINT=http://localhost:4566
+LOG_LEVEL=DEBUG
+PREPARATION_DATA_BUCKET=transformations-preparation-data-local

examples/serverless-localstack/EXERCISE-prepare-data/v4_using_preparation_data.js

@@ -0,0 +1,81 @@
+// Adding a new field "hasWikiPage"
+// "hasWikiPage" is a boolean field that is set to true if the item has a wiki page
+// It is calculated with a prepare function that fetches the wiki page status for each item
+
+const { utils } = require('dynamo-data-transform');
+
+const userAgentHeader = {
+  'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.138 Safari/537.36',
+};
+
+const fetch = (...args) => import('node-fetch').then(({ default: nodeFetch }) => nodeFetch(
+  ...args,
+  {
+    headers: userAgentHeader,
+  },
+));
+
+const TABLE_NAME = 'UsersExample';
+
+const transformUp = async ({ ddb, preparationData, isDryRun }) => {
+  const addHasWikiPage = (hasWikiDict) => (item) => {
+    const valueFromPreparation = hasWikiDict[`${item.PK}-${item.SK}`];
+    const updatedItem = valueFromPreparation ? {
+      ...item,
+      hasWikiPage: valueFromPreparation,
+    } : item;
+    return updatedItem;
+  };
+
+  return utils.transformItems(
+    ddb,
+    TABLE_NAME,
+    addHasWikiPage(JSON.parse(preparationData)),
+    isDryRun,
+  );
+};
+
+const transformDown = async ({ ddb, isDryRun }) => {
+  const removeHasWikiPage = (item) => {
+    const { hasWikiPage, ...oldItem } = item;
+    return oldItem;
+  };
+
+  return utils.transformItems(ddb, TABLE_NAME, removeHasWikiPage, isDryRun);
+};
+
+const prepare = async ({ ddb }) => {
+  let lastEvalKey;
+  let preparationData = {};
+
+  let scannedAllItems = false;
+
+  while (!scannedAllItems) {
+    const { Items, LastEvaluatedKey } = await utils.getItems(ddb, lastEvalKey, TABLE_NAME);
+    lastEvalKey = LastEvaluatedKey;
+
+    const currentPreparationData = await Promise.all(Items.map(async (item) => {
+      const wikiItemUrl = `https://en.wikipedia.org/wiki/${item.name}`;
+      const currWikiResponse = await fetch(wikiItemUrl);
+      return {
+        [`${item.PK}-${item.SK}`]: currWikiResponse.status === 200,
+      };
+    }));
+
+    preparationData = {
+      ...preparationData,
+      ...currentPreparationData.reduce((acc, item) => ({ ...acc, ...item }), {}),
+    };
+
+    scannedAllItems = !lastEvalKey;
+  }
+
+  return preparationData;
+};
+
+module.exports = {
+  transformUp,
+  transformDown,
+  prepare,
+  transformationNumber: 4,
+};

examples/serverless-localstack/README.md

@@ -0,0 +1,90 @@
+# This repository shows the usage of the dynamo-data-transform package.
+
+## Quick Start
+
+### Initialize localstack container and dynamodb GUI
+*Please make sure you have installed and running docker on your machine.
+Run:
+```bash
+docker compose up
+```
+
+### Install dependencies
+```bash
+npm install
+```
+
+### Deploy service to localstack
+```bash
+npm start
+```
+
+Note that after running the above command, all the transformation scripts in data-transformations folder will be executed.
+See the UsersExample table here:
+http://localhost:8001/
+
+
+
+#### Serverless Plugin
+```bash
+npx sls dynamodt --help
+```
+
+#### Interactive Cli
+```bash
+npm install -g dynamo-data-transform
+
+ddt -i
+```
+
+#### Standalone npm package usage:
+init - `ddt init -tableNames "Users"`
+up - `ddt up`
+down - `ddt down -t Users`
+prepare - `ddt prepare -t Users --tNumber 3`
+history - `ddt history -t Users`
+```
+
+
+
+## Exercises:
+1.  For understanding how to prepare data for transformation, take a look at "v4_using_preparation_data.js"
+    Move the file "v4_using_preparation_data.js" to the data-transformations folder.
+    ```bash
+    mv EXERCISE-prepare-data/v4_using_preparation_data.js data-transformations/UsersExample
+    ```
+    Let's check the prepare script results
+    ```bash
+    npx sls dynamodt prepare --table UsersExample --tNumber 4 --dry
+    ```
+    The results in the console should be:
+    ```js
+    {
+        'USER#21-NAME#Bradley Wiggins: true,
+        'USER#34-NAME#Chaos': true,
+        'USER#32-NAME#Knuckles': true,
+        'USER#29-NAME#Plankton': true,
+        ...
+    }
+    ```
+    Now lets prepare some data for the transformation. Run the same command as before but without --dry.
+    ```bash
+    npx sls dynamodt prepare --table UsersExample --tNumber 4
+    ```
+    Let's run the pending transformation script, currently it is "v4_using_preparation_data.js"
+    ```bash
+    npx sls dynamodt up --stage local
+    ```
+    Now open the dynamodb GUI and check the data.
+    http://localhost:8001/
+    
+2.  Rollback the last transformation
+    ```bash
+    npx sls dynamodt down --stage local --table UsersExample --dry
+    ```
+    Now you will see in the console that "hasWikiPage" field was removed from each item.
+    Lets rollback the last transformation for real.
+    ```bash
+    npx sls dynamodt down --stage local --table UsersExample
+    ```
+    Now open the dynamodb GUI and check the data.

examples/serverless-localstack/data-transformations/UsersExample/v1_insert_users.js

@@ -0,0 +1,24 @@
+// Seed users data transformation
+const { utils } = require('dynamo-data-transform');
+
+const { USERS_DATA } = require('../../usersData');
+
+const TABLE_NAME = 'UsersExample';
+
+/**
+ * @param {DynamoDBDocumentClient} ddb - dynamo db document client https://github.com/aws/aws-sdk-js-v3/tree/main/clients/client-dynamodb
+ * @param {boolean} isDryRun - true if this is a dry run
+ */
+const transformUp = async ({ ddb, isDryRun }) => {
+  return utils.insertItems(ddb, TABLE_NAME, USERS_DATA, isDryRun);
+};
+
+const transformDown = async ({ ddb, isDryRun }) => {
+  return utils.deleteItems(ddb, TABLE_NAME, USERS_DATA, isDryRun);
+};
+
+module.exports = {
+  transformUp,
+  transformDown,
+  transformationNumber: 1,
+};

examples/serverless-localstack/data-transformations/UsersExample/v2_add_random_number_field.js

@@ -0,0 +1,27 @@
+// Adding a "randotNumber" field to each item
+const { utils } = require('dynamo-data-transform');
+
+const TABLE_NAME = 'UsersExample';
+
+const transformUp = async ({ ddb, isDryRun }) => {
+  const addRandotNumberField = (item) => {
+    const updatedItem = { ...item, randotNumber: Math.random() };
+    return updatedItem;
+  };
+  return utils.transformItems(ddb, TABLE_NAME, addRandotNumberField, isDryRun);
+};
+
+const transformDown = async ({ ddb, isDryRun }) => {
+  const removeRandotNumberField = (item) => {
+    const { randotNumber, ...oldItem } = item;
+    return oldItem;
+  };
+  return utils.transformItems(ddb, TABLE_NAME, removeRandotNumberField, isDryRun);
+};
+
+module.exports = {
+  transformUp,
+  transformDown,
+  // prepare, // pass this function only if you need preparation data for the transformation
+  transformationNumber: 2,
+};

examples/serverless-localstack/data-transformations/UsersExample/v3_split_fullname.js

@@ -0,0 +1,32 @@
+// Split full name of every record into first and last name
+const { utils } = require('dynamo-data-transform');
+
+const TABLE_NAME = 'UsersExample';
+
+const transformUp = async ({ ddb, isDryRun }) => {
+  const addFirstAndLastName = (item) => {
+    // Just for the example:
+    // Assume that the FullName has one space between first and last name
+    const [firstName, ...lastName] = item.name.split(' ');
+    return {
+      ...item,
+      firstName,
+      lastName: lastName.join(' '),
+    };
+  };
+  return utils.transformItems(ddb, TABLE_NAME, addFirstAndLastName, isDryRun);
+};
+
+const transformDown = async ({ ddb, isDryRun }) => {
+  const removeFirstAndLastName = (item) => {
+    const { firstName, lastName, ...oldItem } = item;
+    return oldItem;
+  };
+  return utils.transformItems(ddb, TABLE_NAME, removeFirstAndLastName, isDryRun);
+};
+
+module.exports = {
+  transformUp,
+  transformDown,
+  transformationNumber: 3,
+};

examples/serverless-localstack/docker-compose.yml

@@ -0,0 +1,44 @@
+version: '3.9'
+
+services:
+  dynamodb:
+    image: aaronshaf/dynamodb-admin
+    ports:
+      - "8001:8001"
+    environment:
+      - DYNAMO_ENDPOINT=http://localstack:4566
+    networks:
+      - localstack-net
+
+  localstack:
+    image: localstack/localstack:0.14.2
+    ports:
+      - "4566-4583:4566-4583"
+      - "${PORT_WEB_UI-4666}:${PORT_WEB_UI-8080}"
+      - "8080:8080"
+      - "4510:4510"
+    environment:
+      - SERVICES=${SERVICES-iam,s3,s3api,lambda,apigateway,apigatewaymanagementapi,cloudwatch,events,cloudformation,sts,cognito,secretsmanager,dynamodb,ecr,sqs,ssm,sns}
+      - DATA_DIR=${DATA_DIR- }
+      - LOCALSTACK_API_KEY=${LOCALSTACK_API_KEY- }
+      - PORT_WEB_UI=8080
+      - START_WEB=0
+      - LAMBDA_REMOTE_DOCKER=true
+      - LAMBDA_EXECUTOR=docker-reuse
+      - LAMBDA_REMOVE_CONTAINERS=true
+      - DOCKER_HOST=unix:///var/run/docker.sock
+      - LAMBDA_DOCKER_NETWORK=localstack-net
+      - KINESIS_ERROR_PROBABILITY=${KINESIS_ERROR_PROBABILITY- }
+      - LS_LOG=info
+      - DEBUG=0
+      - TMPDIR=./tmp/localstack
+      - HOST_TMP_FOLDER=${TMPDIR}
+    volumes:
+      - "/var/run/docker.sock:/var/run/docker.sock"
+      - "${TMPDIR:-/tmp/localstack}"
+    networks:
+      - localstack-net
+
+networks:
+  localstack-net:
+    name: localstack-net