💥 Change data transformation script function parameters

Author: Guy7B

README.md

@@ -105,7 +105,7 @@ dynamodb-data-migrations <command> --help
 ```
 
 ## What it does behind the scenes
-- When migration is running for the first time a Record in your table is created. \
+- When a data migration is running for the first time a Record in your table is created. \
   This record is for tracking the executed migrations on a specific table.
 
 
@@ -131,13 +131,13 @@ Implement these functions:
 
 1. Preparing data from external resources for the migration can be done by using `sls migration prepare`
 
-    Run `sls migration prepare -p $(pwd)/migrations/{{YOUR_TABLE_NAME}}/v1.js`\
+    Run `sls migration prepare --mNumber <migration_number> --table <table>`\
     The data will be stored encrypted in this format: {{FILE_VERSION}}.{{ENV}}.encrypted (e.g v1.local.encrypted) \
     The data will be decrypted while running the migration script.
 
 1. **Final Step** Create a pull request. \
    Note that the migration runs after an sls deploy command it is integrated \
-   with lifecycle hooks of serverless `after:deploy:deploy` hook.
+   with lifecycle of serverless `after:deploy:deploy` hook.
 
 #### The Third Phase (Use The New Resources/Data):
 1. Adjust your code to work with the new data. \
@@ -160,6 +160,19 @@ To be continued
 
 
 ### Examples
+Examples of data migration code:
+
+#### Insert records
+https://github.com/jitsecurity/dynamodb-data-migrations/tree/main/examples/demo/data-migrations/v1_insert_users.js
+
+#### Add a new field to each record
+https://github.com/jitsecurity/dynamodb-data-migrations/tree/main/examples/demo/data-migrations/v2_add_random_number_field.js
+
+#### Add field using preparation data (s3 bucket)
+https://github.com/jitsecurity/dynamodb-data-migrations/tree/main/examples/demo/data-migrations/v3_using_preparation_data.js
+
+#### Split fullname into first and last name
+https://github.com/jitsecurity/dynamodb-data-migrations/tree/main/examples/demo/data-migrations/v4_split_fullname.js
 
 - Required functions to implement by the plugin user examples
 1. async transformUp
@@ -206,5 +219,3 @@ const transformDown = async (ddb, isDryRun) => {
   } while (lastEvalKey)
 }
 ```
-- To be continued
-<!-- - `sls migration up --stage local` -  -->

src/command-handlers/down.js

@@ -28,7 +28,7 @@ const down = async ({ table, dry: isDryRun }) => {
     console.info('Running data migration script using preparation data');
   }
 
-  await transformDown(ddb, preparationData, isDryRun);
+  await transformDown({ ddb, preparationData, isDryRun });
 
   if (!isDryRun) {
     await rollbackMigration(ddb, latestDataMigrationNumber, table);

src/command-handlers/prepare.js

@@ -14,7 +14,7 @@ const prepareHandler = async ({ table, mNumber, dry: isDryRun }) => {
 
   const ddb = getDynamoDBClient();
   const prepatationDataPath = `${table}/${MIGRATION_NUMBER_PREFIX}${mNumber}`;
-  const preparationData = await prepare(ddb, isDryRun);
+  const preparationData = await prepare({ ddb, isDryRun });
   if (isDryRun) {
     console.info(preparationData, 'preparationData');
     console.info("It's a dry run");

src/command-handlers/up.js

@@ -20,7 +20,7 @@ const executeDataMigration = async (ddb, migration, table, isDryRun) => {
       console.info('Running data migration script using preparation data');
     }
 
-    const transformationResponse = await transformUp(ddb, preparationData, isDryRun);
+    const transformationResponse = await transformUp({ ddb, preparationData, isDryRun });
     if (!isDryRun) {
       await syncMigrationRecord(ddb, migrationNumber, table, transformationResponse?.transformed);
     } else {

src/config/constants.js

@@ -4,10 +4,13 @@ const MIGRATION_SCRIPT_EXTENSION = '.js';
 const MIGRATION_NUMBER_PREFIX = 'v';
 const MIGRATION_NAME_SEPARATOR = '_';
 
+const DATA_MIGRATIONS_KEY_ID = 'DataMigrations';
+
 module.exports = {
   DATA_MIGRATIONS_FOLDER_NAME,
   BASE_MIGRATIONS_FOLDER_PATH,
   MIGRATION_SCRIPT_EXTENSION,
   MIGRATION_NUMBER_PREFIX,
   MIGRATION_NAME_SEPARATOR,
+  DATA_MIGRATIONS_KEY_ID,
 };

src/config/migration-template-file.js

@@ -2,23 +2,28 @@ const { utils } = require('@jitsecurity/dynamodb-data-migrations');
 
 const TABLE_NAME = '{{YOUR_TABLE_NAME}}';
 
-const up = (item) => {
-  // Adding a new field to the item example
-  const updatedItem = { ...item, newField: 'value' };
-  return updatedItem;
+/**
+ * @param {DynamoDBDocumentClient} ddb - dynamo db client of @aws-sdk https://github.com/aws/aws-sdk-js-v3/tree/main/clients/client-dynamodb
+ * @param {boolean} isDryRun
+ * @returns {{transformed: number}}
+ *
+ */
+const transformUp = async ({ ddb, isDryRun }) => {
+  // Replace this with your own logic
+  const addNewFieldToItem = (item) => {
+    const updatedItem = { ...item, newField: 'value' };
+    return updatedItem;
+  };
+  return utils.transformItems(ddb, isDryRun, TABLE_NAME, addNewFieldToItem);
 };
 
-const transformUp = async (ddb, preparationData, isDryRun) => {
-  return utils.transformItems(ddb, isDryRun, TABLE_NAME, up);
-};
-
-const down = (item) => {
-  const { newField, ...oldItem } = item;
-  return oldItem;
-};
-
-const transformDown = async (ddb, isDryRun) => {
-  return utils.transformItems(ddb, isDryRun, TABLE_NAME, down);
+const transformDown = async ({ ddb, isDryRun }) => {
+  // Replace this function with your own logic
+  const removeFieldFromItem = (item) => {
+    const { newField, ...oldItem } = item;
+    return oldItem;
+  };
+  return utils.transformItems(ddb, isDryRun, TABLE_NAME, removeFieldFromItem);
 };
 
 module.exports = {
@@ -27,3 +32,9 @@ module.exports = {
   // prepare, // pass this function only if you need preparation data for the migration
   migrationNumber: 1,
 };
+
+/**
+ * For more data migration scripts examples, see:
+ * https://github.com/jitsecurity/dynamodb-data-migrations/tree/main/examples
+ *
+ */

src/services/dynamodb/migrations-executions-manager.js

@@ -1,8 +1,7 @@
 const { GetCommand, PutCommand } = require('@aws-sdk/lib-dynamodb');
+const { DATA_MIGRATIONS_KEY_ID } = require('../../config/constants');
 const getTableKeySchema = require('../../utils/getTableKeySchema');
 
-const DATA_MIGRATIONS_KEY_ID = 'DataMigrations';
-
 const getDataMigrationKey = async (table) => {
   const keySchema = await getTableKeySchema(table);
   const dataMigrationKey = keySchema.reduce((acc, val) => ({

src/utils/getItems.js

@@ -1,4 +1,9 @@
 const { ScanCommand } = require('@aws-sdk/lib-dynamodb');
+const { DATA_MIGRATIONS_KEY_ID } = require('../config/constants');
+
+const filterMigrationRecord = (items) => {
+  return items.filter((item) => !Object.values(item).includes(DATA_MIGRATIONS_KEY_ID));
+};
 
 const getItems = async (ddb, lastEvalKey, tableName) => {
   const params = {
@@ -8,7 +13,10 @@ const getItems = async (ddb, lastEvalKey, tableName) => {
   };
 
   const scanCommand = new ScanCommand(params);
-  return ddb.send(scanCommand);
+  const scanResponse = await ddb.send(scanCommand);
+  scanResponse.Items = filterMigrationRecord(scanResponse.Items);
+
+  return scanResponse;
 };
 
 module.exports = getItems;

src/utils/index.js

@@ -2,14 +2,14 @@ const getItems = require('./getItems');
 const transformItems = require('./transformItems');
 const transactWriteItems = require('./transactWriteItems');
 const batchWriteItems = require('./batchWriteItems');
-const seedItems = require('./seedItems');
+const insertItems = require('./insertItems');
 const deleteItems = require('./deleteItems');
 
 module.exports = {
   getItems,
   transactWriteItems,
   transformItems,
   batchWriteItems,
-  seedItems,
+  insertItems,
   deleteItems,
 };

src/utils/seedItems.js src/utils/insertItems.jssrc/utils/seedItems.js renamed to src/utils/insertItems.js

@@ -3,7 +3,7 @@ const { getUnprocessedItems } = require('./responseUtils');
 
 const MAX_ITEMS_PER_BATCH = 25;
 
-const seedItems = async (ddb, tableName, items, isDryRun) => {
+const insertItems = async (ddb, tableName, items, isDryRun) => {
   if (isDryRun) {
     console.info(`Dry run: would have seeded ${items.length} items to ${tableName}`, items);
     return;
@@ -31,4 +31,4 @@ const seedItems = async (ddb, tableName, items, isDryRun) => {
   }
 };
 
-module.exports = seedItems;
+module.exports = insertItems;