✨ First public version of dynamo data transformation

Author: Guy7B

.eslintrc.js

@@ -28,6 +28,16 @@ module.exports = {
     'import/no-dynamic-require': 'off',
     'global-require': 'off',
     'no-restricted-syntax': 'off',
+    'max-len': [
+      'error',
+      {
+        code: 120,
+        ignoreComments: true,
+        ignoreRegExpLiterals: true,
+        ignoreStrings: true,
+        ignoreTemplateLiterals: true,
+      },
+    ],
   },
   overrides: [
     {
@@ -37,6 +47,13 @@ module.exports = {
         'import/no-unresolved': 'off',
       },
     },
+    {
+      files: ['*.js'],
+      rules: {
+        'import/no-extraneous-dependencies': 'off',
+        'import/no-unresolved': 'off',
+      },
+    },
   ],
 
 };

.gitignore

@@ -3,6 +3,5 @@ node_modules
 dist
 local
 .vscode
-dynamodb-migrations-cli
 .DS_Store
 examples

.npmignore

@@ -2,4 +2,5 @@
 /.github
 /guy
 /.vscode
-/docs/images
+/docs/images
+/examples

README.md

@@ -130,7 +130,7 @@ Implement these functions:
 
 1. Preparing data from external resources for the data transformation can be done by using `sls dynamodt prepare`
 
-    Run `sls dynamodt prepare --mNumber <transformation_number> --table <table>`\
+    Run `sls dynamodt prepare --tNumber <transformation_number> --table <table>`\
     The data will be stored in a S3 bucket  \
     The data will be decrypted while running the data transformation script.
 
@@ -155,8 +155,20 @@ First of all, keep in mind that our mission is to prevent downtime while executi
 - To be continued...
 
 ### Troubleshooting
-To be continued
-
+Credentials error:
+#### Required environment variables: #Move it to the tool docs
+```bash
+# configure aws credentials aws configure or use environment variables
+export AWS_ACCESS_KEY_ID=<your-access-key-id>
+export AWS_SECRET_ACCESS_KEY=<your-secret-access-key>
+export AWS_DEFAULT_REGION=<your-region>
+
+#[OPTIONAL] 
+# Required for preparing data for the migration
+# The default is migrations-preparation-data
+# Make sure you have created the bucket before running the migration
+export PREPARATION_DATA_BUCKET=<your-bucket-name>
+```
 
 ### Data Transformation Script Format (e.g v1_script.js)
 ```js
@@ -182,7 +194,7 @@ const prepare = async ({ ddb, isDryRun }) => {
 module.exports = {
   transformUp,
   transformDown,
-  prepare,
+  prepare, // optional
   transformationNumber: 1,
 }
 ```
@@ -196,7 +208,7 @@ https://github.com/jitsecurity/dynamo-data-transform/tree/main/examples/serverle
 #### Insert records
 
 ```js
-// Seed users data migration
+// Seed users data transformation
 const { utils } = require('dynamo-data-transform');
 const { USERS_DATA } = require('../../usersData');
 
@@ -217,38 +229,38 @@ const transformDown = async ({ ddb, isDryRun }) => {
 module.exports = {
   transformUp,
   transformDown,
-  migrationNumber: 1,
+  transformationNumber: 1,
 };
 ```
 
 #### Add a new field to each record
 ```js
-// Adding a "randomNumber" field to each item
+// Adding a "randotNumber" field to each item
 const { utils } = require('dynamo-data-transform');
 
 const TABLE_NAME = 'UsersExample';
 
 const transformUp = async ({ ddb, isDryRun }) => {
-  const addRandomNumberField = (item) => {
-    const updatedItem = { ...item, randomNumber: Math.random() };
+  const addRandotNumberField = (item) => {
+    const updatedItem = { ...item, randotNumber: Math.random() };
     return updatedItem;
   };
-  return utils.transformItems(ddb, TABLE_NAME, addRandomNumberField, isDryRun);
+  return utils.transformItems(ddb, TABLE_NAME, addRandotNumberField, isDryRun);
 };
 
 const transformDown = async ({ ddb, isDryRun }) => {
-  const removeRandomNumberField = (item) => {
-    const { randomNumber, ...oldItem } = item;
+  const removeRandotNumberField = (item) => {
+    const { randotNumber, ...oldItem } = item;
     return oldItem;
   };
-  return utils.transformItems(ddb, TABLE_NAME, removeRandomNumberField, isDryRun);
+  return utils.transformItems(ddb, TABLE_NAME, removeRandotNumberField, isDryRun);
 };
 
 module.exports = {
   transformUp,
   transformDown,
-  // prepare, // pass this function only if you need preparation data for the migration
-  migrationNumber: 2,
+  // prepare, // pass this function only if you need preparation data for the transformation
+  transformationNumber: 2,
 };
 ```
 
@@ -333,7 +345,7 @@ module.exports = {
   transformUp,
   transformDown,
   prepare,
-  migrationNumber: 3,
+  transformationNumber: 4,
 };
 ```
 

cli/Components/App.js

@@ -14,8 +14,8 @@ const App = () => {
 
   return (
     <>
-      <BigTextErrorBoundary text={'Data Migration Tool'} key={'keyfortitle'}>
-        <Title text='Data Migration Tool' />
+      <BigTextErrorBoundary text='Dynamo Data Transform' key='keyfortitle'>
+        <Title text='Dynamo Data Transform' />
       </BigTextErrorBoundary>
       <Newline />
       {selection ? (

cli/Components/CommandForm.js

@@ -47,7 +47,7 @@ const CommandForm = ({ selection, setSelection }) => {
     script(params).then(() => {
       setSelection(null);
     }).catch((error) => {
-      console.error(error, `An error has occured while running migration (${selection}).`);
+      console.error(error, `An error has occured while running transformation (${selection}).`);
     });
   };
 

cli/cli-commands.js

@@ -7,8 +7,8 @@ const COMMAND_OPTIONS = {
   table: {
     type: 'string', name: 'table', label: 'Specify table name', initialValue: '',
   },
-  mNumber: {
-    type: 'string', name: 'mNumber', label: 'Specify the version of current migration number',
+  tNumber: {
+    type: 'string', name: 'tNumber', label: 'Specify the version of current transformation number',
   },
   tableNames: {
     type: 'string', name: 'tableNames', label: 'Specify table names e.g "table1, table2"', initialValue: '',
@@ -24,11 +24,11 @@ const CLI_COMMANDS = {
 };
 
 const HELP_COMMANDS = {
-  up: 'ddm up',
-  down: 'ddm down --table <table>',
-  history: 'ddm history --table <table>',
-  prepare: 'ddm prepare --table <table> --mNumber <migration_number>',
-  init: 'ddm init --tableNames <table_names>',
+  up: 'ddt up',
+  down: 'ddt down --table <table>',
+  history: 'ddt history --table <table>',
+  prepare: 'ddt prepare --table <table> --tNumber <transformation_number>',
+  init: 'ddt init --tableNames <table_names>',
 };
 
 const CLI_FORM = {
@@ -49,7 +49,7 @@ const CLI_FORM = {
     title: 'Preparetion Parameters',
     fields: [
       COMMAND_OPTIONS.table,
-      COMMAND_OPTIONS.mNumber,
+      COMMAND_OPTIONS.tNumber,
       COMMAND_OPTIONS.dry,
     ],
   },

cli/index.js

@@ -9,7 +9,7 @@ const { HELP_COMMANDS } = require('./cli-commands');
 const commandAliases = {
   dry: 'd',
   table: 't',
-  mNumber: 'm',
+  tNumber: 'tnum',
   tableNames: 'n',
   interactive: 'i',
 };
@@ -18,7 +18,7 @@ const options = parseArgs(process.argv.slice(2),{
   alias: commandAliases,
   boolean: ['dry', 'interactive', 'help'],
   string: ['table', 'tableNames'],
-  number: ['mNumber'],
+  number: ['tNumber'],
 });
 
 (() => {
@@ -44,7 +44,7 @@ const options = parseArgs(process.argv.slice(2),{
       console.info(`"${command}" command run successfully.`);
       process.exit(0);
     }).catch((error) => {
-      console.error(error, `An error has occured while running migration (${command}).`);
+      console.error(error, `An error has occured while running transformation (${command}).`);
       process.exit(1);
     });
   }

index.js

@@ -1,3 +1,5 @@
-const SlsPluginDynamodbDataMigrations = require('./serverless-plugin/sls-plugin');
+const SlsDynamoDataTransformationsPlugin = require('./serverless-plugin/sls-plugin');
+const utils = require('./src/utils');
 
-module.exports = SlsPluginDynamodbDataMigrations;
+module.exports = SlsDynamoDataTransformationsPlugin;
+module.exports.utils = utils;

serverless-plugin/commands.js

@@ -13,8 +13,8 @@ const COMMAND_OPTIONS = {
     required: true,
     type: 'string',
   },
-  mNumber: {
-    usage: 'Specify the version of current migration (e.g. "--mNumber 1")',
+  tNumber: {
+    usage: 'Specify the version of current transformation (e.g. "--tNumber 1")',
     required: true,
     type: 'number',
   },
@@ -24,39 +24,39 @@ const COMMAND_OPTIONS = {
 };
 
 module.exports = {
-  migration: {
-    usage: 'Runs database migrations',
+  dynamodt: {
+    usage: 'Run data transformations',
     commands: {
       init: {
-        usage: `sls migration init - ${COMMAND_DESCRIPTION.init}`,
+        usage: `sls dynamodt init - ${COMMAND_DESCRIPTION.init}`,
         lifecycleEvents: ['init'],
       },
       up: {
-        usage: `sls migration up - ${COMMAND_DESCRIPTION.up}`,
-        lifecycleEvents: ['migrate'],
+        usage: `sls dynamodt up - ${COMMAND_DESCRIPTION.up}`,
+        lifecycleEvents: ['transform'],
         options: {
           dry: COMMAND_OPTIONS.dry,
         },
       },
       prepare: {
-        usage: `"sls migration prepare --table {{TABLE_NAME}} --mNumber 1" - ${COMMAND_DESCRIPTION.prepare}`,
+        usage: `"sls dynamodt prepare --table {{TABLE_NAME}} --tNumber 1" - ${COMMAND_DESCRIPTION.prepare}`,
         lifecycleEvents: ['prepare'],
         options: {
-          mNumber: COMMAND_OPTIONS.mNumber,
+          tNumber: COMMAND_OPTIONS.tNumber,
           dry: COMMAND_OPTIONS.dry,
           table: COMMAND_OPTIONS.table,
         },
       },
       down: {
-        usage: `sls migration down --table {{TABLE_NAME}} - ${COMMAND_DESCRIPTION.down}`,
+        usage: `sls dynamodt down --table {{TABLE_NAME}} - ${COMMAND_DESCRIPTION.down}`,
         lifecycleEvents: ['rollback'],
         options: {
           table: COMMAND_OPTIONS.table,
           dry: COMMAND_OPTIONS.dry,
         },
       },
       history: {
-        usage: `sls migration history --table {{TABLE_NAME}} -  ${COMMAND_DESCRIPTION.history}`,
+        usage: `sls dynamodt history --table {{TABLE_NAME}} -  ${COMMAND_DESCRIPTION.history}`,
         lifecycleEvents: ['history'],
         options: {
           table: COMMAND_OPTIONS.table,