chore: made implementation consistent with other modules

Author: naman-contentstack

.talismanrc

@@ -1,4 +1,12 @@
 fileignoreconfig:
-  - filename: pnpm-lock.yaml
-    checksum: 9ad01ca900e007d8ea4f5ffcef9f166665aba852b3b9a0a5f4506c04fd362138
+  - filename: packages/contentstack-asset-management/src/import/fields.ts
+    checksum: e46c3eb94bba78ae06af0139a1b0fd4113c3dcfe879ed40cfe77659781526d5c
+  - filename: packages/contentstack-asset-management/src/import/asset-types.ts
+    checksum: 3525703fd2ac0f7ab3e963966c0abfa53d87c09f73dabab0889c7f41a5f1b003
+  - filename: packages/contentstack-import/src/types/default-config.ts
+    checksum: 1c09acba953cfd7058a3e0d63f0a9bfbb8f28e903538eaa015fdc611402bbd4f
+  - filename: packages/contentstack-asset-management/src/types/asset-management-api.ts
+    checksum: 9394e67f4d0dea9ad27a8592adf99441f40731b788335cd7699c78205bdaad58
+  - filename: packages/contentstack-import/src/import/modules/assets.ts
+    checksum: 98cea49283eb5d975168dc51e27b0e2fb541a9ed9b4368206f99b434e104f096
 version: '1.0'

packages/contentstack-asset-management/src/constants/index.ts

@@ -1,19 +1,36 @@
-export const BATCH_SIZE = 50;
-export const CHUNK_FILE_SIZE_MB = 1;
+/** Fallback when export/import do not pass `chunkWriteBatchSize`. */
+export const FALLBACK_AM_CHUNK_WRITE_BATCH_SIZE = 50;
+/** Fallback when export/import do not pass `chunkFileSizeMb`. */
+export const FALLBACK_AM_CHUNK_FILE_SIZE_MB = 1;
+/** Fallback when import does not pass `apiConcurrency`. */
+export const FALLBACK_AM_API_CONCURRENCY = 5;
+/** @deprecated Use FALLBACK_AM_API_CONCURRENCY */
+export const DEFAULT_AM_API_CONCURRENCY = FALLBACK_AM_API_CONCURRENCY;
 
-/** Default parallel AM API calls when import caller does not set apiConcurrency. */
-export const DEFAULT_AM_API_CONCURRENCY = 5;
+/** Fallback strip lists when import options omit `fieldsImportInvalidKeys` / `assetTypesImportInvalidKeys`. */
+export const FALLBACK_FIELDS_IMPORT_INVALID_KEYS = [
+  'created_at',
+  'created_by',
+  'updated_at',
+  'updated_by',
+  'is_system',
+  'asset_types_count',
+] as const;
+export const FALLBACK_ASSET_TYPES_IMPORT_INVALID_KEYS = [
+  'created_at',
+  'created_by',
+  'updated_at',
+  'updated_by',
+  'is_system',
+  'category',
+  'preview_image_url',
+  'category_detail',
+] as const;
 
-/**
- * Mapper output paths — must stay aligned with contentstack-import `PATH_CONSTANTS`
- * (`mapper` / `assets` / uid, url, space-uid file names).
- */
-export const IMPORT_ASSETS_MAPPER_DIR_SEGMENTS = ['mapper', 'assets'] as const;
-export const IMPORT_ASSETS_MAPPER_FILES = {
-  UID_MAPPING: 'uid-mapping.json',
-  URL_MAPPING: 'url-mapping.json',
-  SPACE_UID_MAPPING: 'space-uid-mapping.json',
-} as const;
+/** @deprecated Use FALLBACK_AM_CHUNK_WRITE_BATCH_SIZE */
+export const BATCH_SIZE = FALLBACK_AM_CHUNK_WRITE_BATCH_SIZE;
+/** @deprecated Use FALLBACK_AM_CHUNK_FILE_SIZE_MB */
+export const CHUNK_FILE_SIZE_MB = FALLBACK_AM_CHUNK_FILE_SIZE_MB;
 
 /**
  * Main process name for Asset Management 2.0 export (single progress bar).

packages/contentstack-asset-management/src/export/base.ts

@@ -5,8 +5,11 @@ import { FsUtility, log, CLIProgressManager, configHandler } from '@contentstack
 import type { AssetManagementAPIConfig } from '../types/asset-management-api';
 import type { ExportContext } from '../types/export-types';
 import { AssetManagementAdapter } from '../utils/asset-management-api-adapter';
-import { AM_MAIN_PROCESS_NAME } from '../constants/index';
-import { BATCH_SIZE, CHUNK_FILE_SIZE_MB } from '../constants/index';
+import {
+  AM_MAIN_PROCESS_NAME,
+  FALLBACK_AM_CHUNK_FILE_SIZE_MB,
+  FALLBACK_AM_CHUNK_WRITE_BATCH_SIZE,
+} from '../constants/index';
 
 export type { ExportContext };
 
@@ -83,17 +86,19 @@ export class AssetManagementExportAdapter extends AssetManagementAdapter {
       await writeFile(pResolve(dir, indexFileName), '{}');
       return;
     }
+    const chunkMb = this.exportContext.chunkFileSizeMb ?? FALLBACK_AM_CHUNK_FILE_SIZE_MB;
+    const batchSize = this.exportContext.chunkWriteBatchSize ?? FALLBACK_AM_CHUNK_WRITE_BATCH_SIZE;
     const fs = new FsUtility({
       basePath: dir,
       indexFileName,
-      chunkFileSize: CHUNK_FILE_SIZE_MB,
+      chunkFileSize: chunkMb,
       moduleName,
       fileExt: 'json',
       metaPickKeys,
       keepMetadata: true,
     });
-    for (let i = 0; i < items.length; i += BATCH_SIZE) {
-      const batch = items.slice(i, i + BATCH_SIZE);
+    for (let i = 0; i < items.length; i += batchSize) {
+      const batch = items.slice(i, i + batchSize);
       fs.writeIntoFile(batch as Record<string, string>[], { mapKeyVal: true });
     }
     fs.completeFile(true);

packages/contentstack-asset-management/src/export/spaces.ts

@@ -29,8 +29,18 @@ export class ExportSpaces {
   }
 
   async start(): Promise<void> {
-    const { linkedWorkspaces, exportDir, branchName, assetManagementUrl, org_uid, apiKey, context, securedAssets } =
-      this.options;
+    const {
+      linkedWorkspaces,
+      exportDir,
+      branchName,
+      assetManagementUrl,
+      org_uid,
+      apiKey,
+      context,
+      securedAssets,
+      chunkWriteBatchSize,
+      chunkFileSizeMb,
+    } = this.options;
 
     if (!linkedWorkspaces.length) {
       log.debug('No linked workspaces to export', context);
@@ -60,6 +70,8 @@ export class ExportSpaces {
       spacesRootPath,
       context,
       securedAssets,
+      chunkWriteBatchSize,
+      chunkFileSizeMb,
     };
 
     const sharedFieldsDir = pResolve(spacesRootPath, 'fields');

packages/contentstack-asset-management/src/import/asset-types.ts

@@ -4,10 +4,9 @@ import { log } from '@contentstack/cli-utilities';
 
 import type { AssetManagementAPIConfig, ImportContext } from '../types/asset-management-api';
 import { AssetManagementImportAdapter } from './base';
-import { PROCESS_NAMES, PROCESS_STATUS } from '../constants/index';
+import { FALLBACK_ASSET_TYPES_IMPORT_INVALID_KEYS, PROCESS_NAMES, PROCESS_STATUS } from '../constants/index';
 import { runInBatches } from '../utils/concurrent-batch';
-
-const STRIP_KEYS = ['created_at', 'created_by', 'updated_at', 'updated_by', 'is_system', 'category', 'preview_image_url', 'category_detail'];
+import { readChunkedJsonItems } from '../utils/chunked-json-read';
 
 /**
  * Reads shared asset types from `spaces/asset_types/asset-types.json` and POSTs
@@ -28,8 +27,10 @@ export default class ImportAssetTypes extends AssetManagementImportAdapter {
   async start(): Promise<void> {
     await this.init();
 
+    const stripKeys = this.importContext.assetTypesImportInvalidKeys ?? [...FALLBACK_ASSET_TYPES_IMPORT_INVALID_KEYS];
     const dir = this.getAssetTypesDir();
-    const items = await this.readAllChunkedJson<Record<string, unknown>>(dir, 'asset-types.json');
+    const indexName = this.importContext.assetTypesFileName ?? 'asset-types.json';
+    const items = await readChunkedJsonItems<Record<string, unknown>>(dir, indexName, this.importContext.context);
 
     if (items.length === 0) {
       log.debug('No shared asset types to import', this.importContext.context);
@@ -64,8 +65,8 @@ export default class ImportAssetTypes extends AssetManagementImportAdapter {
 
       const existing = existingByUid.get(uid);
       if (existing) {
-        const exportedClean = omit(assetType, STRIP_KEYS);
-        const existingClean = omit(existing, STRIP_KEYS);
+        const exportedClean = omit(assetType, stripKeys);
+        const existingClean = omit(existing, stripKeys);
         if (!isEqual(exportedClean, existingClean)) {
           log.warn(
             `Asset type "${uid}" already exists in the target org with a different definition. Skipping — to apply the exported definition, delete the asset type from the target org first.`,
@@ -78,7 +79,7 @@ export default class ImportAssetTypes extends AssetManagementImportAdapter {
         continue;
       }
 
-      toCreate.push({ uid, payload: omit(assetType, STRIP_KEYS) as Record<string, unknown> });
+      toCreate.push({ uid, payload: omit(assetType, stripKeys) as Record<string, unknown> });
     }
 
     await runInBatches(toCreate, this.apiConcurrency, async ({ uid, payload }) => {

packages/contentstack-asset-management/src/import/assets.ts

@@ -5,6 +5,7 @@ import { log } from '@contentstack/cli-utilities';
 import type { AssetManagementAPIConfig, ImportContext } from '../types/asset-management-api';
 import { AssetManagementImportAdapter } from './base';
 import { getArrayFromResponse } from '../utils/export-helpers';
+import { readChunkedJsonItems } from '../utils/chunked-json-read';
 import { runInBatches } from '../utils/concurrent-batch';
 import { PROCESS_NAMES, PROCESS_STATUS } from '../constants/index';
 
@@ -42,10 +43,11 @@ export default class ImportAssets extends AssetManagementImportAdapter {
    */
   private async loadExportedAssetItems(spaceDir: string): Promise<AssetRecord[] | null> {
     const assetsDir = pResolve(spaceDir, 'assets');
-    if (!existsSync(join(assetsDir, 'assets.json'))) {
+    const assetsIndex = this.importContext.assetsFileName ?? 'assets.json';
+    if (!existsSync(join(assetsDir, assetsIndex))) {
       return null;
     }
-    return this.readAllChunkedJson<AssetRecord>(assetsDir, 'assets.json');
+    return readChunkedJsonItems<AssetRecord>(assetsDir, assetsIndex, this.importContext.context);
   }
 
   /**
@@ -95,14 +97,15 @@ export default class ImportAssets extends AssetManagementImportAdapter {
     // 1. Import folders
     // -----------------------------------------------------------------------
     const folderUidMap: Record<string, string> = {};
-    const foldersFilePath = join(assetsDir, 'folders.json');
+    const foldersFileName = this.importContext.foldersFileName ?? 'folders.json';
+    const foldersFilePath = join(assetsDir, foldersFileName);
 
     if (existsSync(foldersFilePath)) {
       let foldersData: unknown;
       try {
         foldersData = JSON.parse(readFileSync(foldersFilePath, 'utf8'));
       } catch (e) {
-        log.debug(`Could not read folders.json: ${e}`, this.importContext.context);
+        log.debug(`Could not read ${foldersFileName}: ${e}`, this.importContext.context);
       }
 
       if (foldersData) {

packages/contentstack-asset-management/src/import/base.ts

@@ -1,9 +1,10 @@
 import { resolve as pResolve } from 'node:path';
-import { FsUtility, log, CLIProgressManager, configHandler } from '@contentstack/cli-utilities';
+import { CLIProgressManager, configHandler } from '@contentstack/cli-utilities';
 
 import type { AssetManagementAPIConfig, ImportContext } from '../types/asset-management-api';
 import { AssetManagementAdapter } from '../utils/asset-management-api-adapter';
-import { AM_MAIN_PROCESS_NAME, DEFAULT_AM_API_CONCURRENCY } from '../constants/index';
+import { AM_MAIN_PROCESS_NAME, FALLBACK_AM_API_CONCURRENCY } from '../constants/index';
+import { readChunkedJsonItems } from '../utils/chunked-json-read';
 
 export type { ImportContext };
 
@@ -63,39 +64,21 @@ export class AssetManagementImportAdapter extends AssetManagementAdapter {
 
   /** Parallel AM API limit for import batches. */
   protected get apiConcurrency(): number {
-    return this.importContext.apiConcurrency ?? DEFAULT_AM_API_CONCURRENCY;
+    return this.importContext.apiConcurrency ?? FALLBACK_AM_API_CONCURRENCY;
   }
 
   protected getAssetTypesDir(): string {
-    return pResolve(this.importContext.spacesRootPath, 'asset_types');
+    return pResolve(this.importContext.spacesRootPath, this.importContext.assetTypesDir ?? 'asset_types');
   }
 
   protected getFieldsDir(): string {
-    return pResolve(this.importContext.spacesRootPath, 'fields');
+    return pResolve(this.importContext.spacesRootPath, this.importContext.fieldsDir ?? 'fields');
   }
 
   /**
-   * Reads all items from a FsUtility chunked JSON store (index file + chunk files).
-   * Returns a flat array of all items across all chunks.
+   * Reads all items from a chunked JSON store via {@link readChunkedJsonItems} (FsUtility).
    */
   protected async readAllChunkedJson<T = Record<string, unknown>>(dir: string, indexFileName: string): Promise<T[]> {
-    try {
-      const fs = new FsUtility({ basePath: dir, indexFileName });
-      const indexer = fs.indexFileContent;
-      const items: T[] = [];
-      for (const _ in indexer) {
-        const chunk = await fs.readChunkFiles.next().catch((err: unknown): null => {
-          log.debug(`Error reading chunk: ${err}`, this.importContext.context);
-          return null;
-        });
-        if (chunk) {
-          items.push(...(Object.values(chunk as Record<string, T>)));
-        }
-      }
-      return items;
-    } catch (err) {
-      log.debug(`readAllChunkedJson failed for ${dir}/${indexFileName}: ${err}`, this.importContext.context);
-      return [];
-    }
+    return readChunkedJsonItems<T>(dir, indexFileName, this.importContext.context);
   }
 }

packages/contentstack-asset-management/src/import/fields.ts

@@ -4,10 +4,9 @@ import { log } from '@contentstack/cli-utilities';
 
 import type { AssetManagementAPIConfig, ImportContext } from '../types/asset-management-api';
 import { AssetManagementImportAdapter } from './base';
-import { PROCESS_NAMES, PROCESS_STATUS } from '../constants/index';
+import { FALLBACK_FIELDS_IMPORT_INVALID_KEYS, PROCESS_NAMES, PROCESS_STATUS } from '../constants/index';
 import { runInBatches } from '../utils/concurrent-batch';
-
-const STRIP_KEYS = ['created_at', 'created_by', 'updated_at', 'updated_by', 'is_system', 'asset_types_count'];
+import { readChunkedJsonItems } from '../utils/chunked-json-read';
 
 /**
  * Reads shared fields from `spaces/fields/fields.json` and POSTs each to the
@@ -28,8 +27,10 @@ export default class ImportFields extends AssetManagementImportAdapter {
   async start(): Promise<void> {
     await this.init();
 
+    const stripKeys = this.importContext.fieldsImportInvalidKeys ?? [...FALLBACK_FIELDS_IMPORT_INVALID_KEYS];
     const dir = this.getFieldsDir();
-    const items = await this.readAllChunkedJson<Record<string, unknown>>(dir, 'fields.json');
+    const indexName = this.importContext.fieldsFileName ?? 'fields.json';
+    const items = await readChunkedJsonItems<Record<string, unknown>>(dir, indexName, this.importContext.context);
 
     if (items.length === 0) {
       log.debug('No shared fields to import', this.importContext.context);
@@ -64,8 +65,8 @@ export default class ImportFields extends AssetManagementImportAdapter {
 
       const existing = existingByUid.get(uid);
       if (existing) {
-        const exportedClean = omit(field, STRIP_KEYS);
-        const existingClean = omit(existing, STRIP_KEYS);
+        const exportedClean = omit(field, stripKeys);
+        const existingClean = omit(existing, stripKeys);
         if (!isEqual(exportedClean, existingClean)) {
           log.warn(
             `Field "${uid}" already exists in the target org with a different definition. Skipping — to apply the exported definition, delete the field from the target org first.`,
@@ -78,7 +79,7 @@ export default class ImportFields extends AssetManagementImportAdapter {
         continue;
       }
 
-      toCreate.push({ uid, payload: omit(field, STRIP_KEYS) as Record<string, unknown> });
+      toCreate.push({ uid, payload: omit(field, stripKeys) as Record<string, unknown> });
     }
 
     await runInBatches(toCreate, this.apiConcurrency, async ({ uid, payload }) => {