diff --git a/.talismanrc b/.talismanrc index b12912c7..09ab9653 100644 --- a/.talismanrc +++ b/.talismanrc @@ -1,18 +1,12 @@ fileignoreconfig: - - filename: pnpm-lock.yaml - checksum: 9c19eb613068c193fac35e72327198fbc86e759968391f07cc876c56b2b1a63d - - filename: packages/contentstack-export/test/unit/export/modules/base-class.test.ts - checksum: bd2b28305fff90ca26bce56b2c5c61751a62225d310a2553874e9ec009ed78e8 - - filename: packages/contentstack-export/test/unit/export/modules/assets.test.ts - checksum: 73ff01e2d19c8d1384dca2ee7087f8c19e0b1fac6b29c75a02ca523a36b7cb92 - - filename: packages/contentstack-export/src/types/default-config.ts - checksum: bf399466aae808342ec013c0179fbc24ac2d969c77fdbef47a842b12497d507e - - filename: packages/contentstack-export/src/types/index.ts - checksum: fa36c236abac338b03bf307102a99f25dddac9afe75b6b34fb82e318e7759799 - - filename: packages/contentstack-export/src/config/index.ts - checksum: 1eb407ee0bd21597d8a4c673fce99d60fafd151ac843c33dac52ffdcc73e8107 - - filename: packages/contentstack-export/test/unit/export/modules/stack.test.ts - checksum: 79876b8f635037a2d8ba38dac055e7625bf85db6a3cf5729434e6a97e44857d6 - - filename: packages/contentstack-export/src/export/modules/stack.ts - checksum: 375c0c5f58d43430b355050d122d3283083ca91891abe8105a4b4fd9433ece97 + - filename: packages/contentstack-asset-management/src/export/base.ts + checksum: 01cb2158fea9cbe05449c04efb42d8d416d24868d411bd7300d8fa99c9a4ab01 + - filename: packages/contentstack-asset-management/test/unit/export/base.test.ts + checksum: 164fc2e5a4337a2739903499b66eecc66a85bb9b50aa2e71079bdd046a195a94 + - filename: packages/contentstack-asset-management/src/import/fields.ts + checksum: 1cd52254ddbfd186d8ade2c73fc799dd1caa0f10bdd3c6b151621c27207ee173 + - filename: packages/contentstack-import/src/utils/asset-management-import-options.ts + checksum: 96b61b683109be2cc2dbab5231e1ded19282fbf176cd8492c75cc4861f2efea0 + - filename: packages/contentstack-asset-management/src/import/asset-types.ts + checksum: fa2aeea704fd259628f9c86eacd3cf9f6f12543b06b387bd65db7df6b6f5fc49 version: '1.0' diff --git a/packages/contentstack-asset-management/package.json b/packages/contentstack-asset-management/package.json index 2ac90a3d..f41f31f1 100644 --- a/packages/contentstack-asset-management/package.json +++ b/packages/contentstack-asset-management/package.json @@ -29,7 +29,7 @@ ], "license": "MIT", "dependencies": { - "@contentstack/cli-utilities": "~2.0.0-beta" + "@contentstack/cli-utilities": "~2.0.0-beta.5" }, "oclif": { "commands": "./lib/commands", diff --git a/packages/contentstack-asset-management/src/constants/index.ts b/packages/contentstack-asset-management/src/constants/index.ts index 96ff8490..9d6bca63 100644 --- a/packages/contentstack-asset-management/src/constants/index.ts +++ b/packages/contentstack-asset-management/src/constants/index.ts @@ -1,5 +1,32 @@ -export const BATCH_SIZE = 50; -export const CHUNK_FILE_SIZE_MB = 1; +/** Fallback when export/import do not pass `chunkFileSizeMb`. */ +export const FALLBACK_AM_CHUNK_FILE_SIZE_MB = 1; +/** Fallback when import does not pass `apiConcurrency`. */ +export const FALLBACK_AM_API_CONCURRENCY = 5; +/** @deprecated Use FALLBACK_AM_API_CONCURRENCY */ +export const DEFAULT_AM_API_CONCURRENCY = FALLBACK_AM_API_CONCURRENCY; + +/** Fallback strip lists when import options omit `fieldsImportInvalidKeys` / `assetTypesImportInvalidKeys`. */ +export const FALLBACK_FIELDS_IMPORT_INVALID_KEYS = [ + 'created_at', + 'created_by', + 'updated_at', + 'updated_by', + 'is_system', + 'asset_types_count', +] as const; +export const FALLBACK_ASSET_TYPES_IMPORT_INVALID_KEYS = [ + 'created_at', + 'created_by', + 'updated_at', + 'updated_by', + 'is_system', + 'category', + 'preview_image_url', + 'category_detail', +] as const; + +/** @deprecated Use FALLBACK_AM_CHUNK_FILE_SIZE_MB */ +export const CHUNK_FILE_SIZE_MB = FALLBACK_AM_CHUNK_FILE_SIZE_MB; /** * Main process name for Asset Management 2.0 export (single progress bar). @@ -17,10 +44,15 @@ export const PROCESS_NAMES = { AM_FIELDS: 'Fields', AM_ASSET_TYPES: 'Asset types', AM_DOWNLOADS: 'Asset downloads', + // Import process names + AM_IMPORT_FIELDS: 'Import fields', + AM_IMPORT_ASSET_TYPES: 'Import asset types', + AM_IMPORT_FOLDERS: 'Import folders', + AM_IMPORT_ASSETS: 'Import assets', } as const; /** - * Status messages for each process (exporting, fetching, failed). + * Status messages for each process (exporting, fetching, importing, failed). */ export const PROCESS_STATUS = { [PROCESS_NAMES.AM_SPACE_METADATA]: { @@ -47,4 +79,20 @@ export const PROCESS_STATUS = { DOWNLOADING: 'Downloading asset files...', FAILED: 'Failed to download assets.', }, + [PROCESS_NAMES.AM_IMPORT_FIELDS]: { + IMPORTING: 'Importing shared fields...', + FAILED: 'Failed to import fields.', + }, + [PROCESS_NAMES.AM_IMPORT_ASSET_TYPES]: { + IMPORTING: 'Importing shared asset types...', + FAILED: 'Failed to import asset types.', + }, + [PROCESS_NAMES.AM_IMPORT_FOLDERS]: { + IMPORTING: 'Importing folders...', + FAILED: 'Failed to import folders.', + }, + [PROCESS_NAMES.AM_IMPORT_ASSETS]: { + IMPORTING: 'Importing assets...', + FAILED: 'Failed to import assets.', + }, } as const; diff --git a/packages/contentstack-asset-management/src/export/assets.ts b/packages/contentstack-asset-management/src/export/assets.ts index 140b5664..28016691 100644 --- a/packages/contentstack-asset-management/src/export/assets.ts +++ b/packages/contentstack-asset-management/src/export/assets.ts @@ -21,8 +21,8 @@ export default class ExportAssets extends AssetManagementExportAdapter { log.debug(`Fetching folders and assets for space ${workspace.space_uid}`, this.exportContext.context); const [folders, assetsData] = await Promise.all([ - this.getWorkspaceFolders(workspace.space_uid), - this.getWorkspaceAssets(workspace.space_uid), + this.getWorkspaceFolders(workspace.space_uid, workspace.uid), + this.getWorkspaceAssets(workspace.space_uid, workspace.uid), ]); await writeFile(pResolve(assetsDir, 'folders.json'), JSON.stringify(folders, null, 2)); diff --git a/packages/contentstack-asset-management/src/export/base.ts b/packages/contentstack-asset-management/src/export/base.ts index 7521eae5..6fff78b4 100644 --- a/packages/contentstack-asset-management/src/export/base.ts +++ b/packages/contentstack-asset-management/src/export/base.ts @@ -5,8 +5,7 @@ import { FsUtility, log, CLIProgressManager, configHandler } from '@contentstack import type { AssetManagementAPIConfig } from '../types/asset-management-api'; import type { ExportContext } from '../types/export-types'; import { AssetManagementAdapter } from '../utils/asset-management-api-adapter'; -import { AM_MAIN_PROCESS_NAME } from '../constants/index'; -import { BATCH_SIZE, CHUNK_FILE_SIZE_MB } from '../constants/index'; +import { AM_MAIN_PROCESS_NAME, FALLBACK_AM_CHUNK_FILE_SIZE_MB } from '../constants/index'; export type { ExportContext }; @@ -83,19 +82,17 @@ export class AssetManagementExportAdapter extends AssetManagementAdapter { await writeFile(pResolve(dir, indexFileName), '{}'); return; } + const chunkMb = this.exportContext.chunkFileSizeMb ?? FALLBACK_AM_CHUNK_FILE_SIZE_MB; const fs = new FsUtility({ basePath: dir, indexFileName, - chunkFileSize: CHUNK_FILE_SIZE_MB, + chunkFileSize: chunkMb, moduleName, fileExt: 'json', metaPickKeys, keepMetadata: true, }); - for (let i = 0; i < items.length; i += BATCH_SIZE) { - const batch = items.slice(i, i + BATCH_SIZE); - fs.writeIntoFile(batch as Record[], { mapKeyVal: true }); - } + fs.writeIntoFile(items as Record[], { mapKeyVal: true }); fs.completeFile(true); } } diff --git a/packages/contentstack-asset-management/src/export/spaces.ts b/packages/contentstack-asset-management/src/export/spaces.ts index cf3ff2c3..5bb6c747 100644 --- a/packages/contentstack-asset-management/src/export/spaces.ts +++ b/packages/contentstack-asset-management/src/export/spaces.ts @@ -35,8 +35,10 @@ export class ExportSpaces { branchName, assetManagementUrl, org_uid, + apiKey, context, securedAssets, + chunkFileSizeMb, } = this.options; if (!linkedWorkspaces.length) { @@ -54,7 +56,9 @@ export class ExportSpaces { const totalSteps = 2 + linkedWorkspaces.length * 4; const progress = this.createProgress(); progress.addProcess(AM_MAIN_PROCESS_NAME, totalSteps); - progress.startProcess(AM_MAIN_PROCESS_NAME).updateStatus(PROCESS_STATUS[PROCESS_NAMES.AM_FIELDS].FETCHING, AM_MAIN_PROCESS_NAME); + progress + .startProcess(AM_MAIN_PROCESS_NAME) + .updateStatus(PROCESS_STATUS[PROCESS_NAMES.AM_FIELDS].FETCHING, AM_MAIN_PROCESS_NAME); const apiConfig: AssetManagementAPIConfig = { baseURL: assetManagementUrl, @@ -65,6 +69,7 @@ export class ExportSpaces { spacesRootPath, context, securedAssets, + chunkFileSizeMb, }; const sharedFieldsDir = pResolve(spacesRootPath, 'fields'); diff --git a/packages/contentstack-asset-management/src/import/asset-types.ts b/packages/contentstack-asset-management/src/import/asset-types.ts new file mode 100644 index 00000000..e5cc4f15 --- /dev/null +++ b/packages/contentstack-asset-management/src/import/asset-types.ts @@ -0,0 +1,135 @@ +import { existsSync } from 'node:fs'; +import { join } from 'node:path'; +import omit from 'lodash/omit'; +import isEqual from 'lodash/isEqual'; +import { log } from '@contentstack/cli-utilities'; + +import type { AssetManagementAPIConfig, ImportContext } from '../types/asset-management-api'; +import { AssetManagementImportAdapter } from './base'; +import { FALLBACK_ASSET_TYPES_IMPORT_INVALID_KEYS, PROCESS_NAMES, PROCESS_STATUS } from '../constants/index'; +import { runInBatches } from '../utils/concurrent-batch'; +import { forEachChunkedJsonStore } from '../utils/chunked-json-reader'; + +type AssetTypeToCreate = { uid: string; payload: Record }; + +/** + * Reads shared asset types from `spaces/asset_types/asset-types.json` and POSTs + * each to the target org-level AM endpoint (`POST /api/asset_types`). + * + * Strategy: Fetch → Diff → Create only missing, warn on conflict + * 1. Fetch asset types that already exist in the target org. + * 2. Skip entries where is_system=true (platform-owned, cannot be created via API). + * 3. If uid already exists and definition differs → warn and skip. + * 4. If uid already exists and definition matches → silently skip. + * 5. Strip read-only/computed keys from the POST body before creating new asset types. + */ +export default class ImportAssetTypes extends AssetManagementImportAdapter { + constructor(apiConfig: AssetManagementAPIConfig, importContext: ImportContext) { + super(apiConfig, importContext); + } + + async start(): Promise { + await this.init(); + + const stripKeys = this.importContext.assetTypesImportInvalidKeys ?? [...FALLBACK_ASSET_TYPES_IMPORT_INVALID_KEYS]; + const dir = this.getAssetTypesDir(); + const indexName = this.importContext.assetTypesFileName ?? 'asset-types.json'; + const indexPath = join(dir, indexName); + + if (!existsSync(indexPath)) { + log.debug('No shared asset types to import (index missing)', this.importContext.context); + return; + } + + const existingByUid = await this.loadExistingAssetTypesMap(); + + this.updateStatus(PROCESS_STATUS[PROCESS_NAMES.AM_IMPORT_ASSET_TYPES].IMPORTING, PROCESS_NAMES.AM_IMPORT_ASSET_TYPES); + + await forEachChunkedJsonStore>( + dir, + indexName, + { + context: this.importContext.context, + chunkReadLogLabel: 'asset-types', + onOpenError: (e) => + log.debug(`Could not open chunked asset-types index: ${e}`, this.importContext.context), + onEmptyIndexer: () => + log.debug('No shared asset types to import (empty indexer)', this.importContext.context), + }, + async (records) => { + const toCreate = this.buildAssetTypesToCreate(records, existingByUid, stripKeys); + await this.importAssetTypesCreates(toCreate); + }, + ); + } + + /** Org-level asset types keyed by uid for diff; empty map if list API fails. */ + private async loadExistingAssetTypesMap(): Promise>> { + const existingByUid = new Map>(); + try { + const existing = await this.getWorkspaceAssetTypes(''); + for (const at of existing.asset_types ?? []) { + existingByUid.set(at.uid, at as Record); + } + log.debug(`Target org has ${existingByUid.size} existing asset type(s)`, this.importContext.context); + } catch (e) { + log.debug(`Could not fetch existing asset types, will attempt to create all: ${e}`, this.importContext.context); + } + return existingByUid; + } + + private buildAssetTypesToCreate( + items: Record[], + existingByUid: Map>, + stripKeys: string[], + ): AssetTypeToCreate[] { + const toCreate: AssetTypeToCreate[] = []; + + for (const assetType of items) { + const uid = assetType.uid as string; + + if (assetType.is_system) { + log.debug(`Skipping system asset type: ${uid}`, this.importContext.context); + continue; + } + + const existing = existingByUid.get(uid); + if (existing) { + const exportedClean = omit(assetType, stripKeys); + const existingClean = omit(existing, stripKeys); + if (!isEqual(exportedClean, existingClean)) { + log.warn( + `Asset type "${uid}" already exists in the target org with a different definition. Skipping — to apply the exported definition, delete the asset type from the target org first.`, + this.importContext.context, + ); + } else { + log.debug(`Asset type "${uid}" already exists with matching definition, skipping`, this.importContext.context); + } + this.tick(true, `asset-type: ${uid} (skipped, already exists)`, null, PROCESS_NAMES.AM_IMPORT_ASSET_TYPES); + continue; + } + + toCreate.push({ uid, payload: omit(assetType, stripKeys) as Record }); + } + + return toCreate; + } + + private async importAssetTypesCreates(toCreate: AssetTypeToCreate[]): Promise { + await runInBatches(toCreate, this.apiConcurrency, async ({ uid, payload }) => { + try { + await this.createAssetType(payload as any); + this.tick(true, `asset-type: ${uid}`, null, PROCESS_NAMES.AM_IMPORT_ASSET_TYPES); + log.debug(`Imported asset type: ${uid}`, this.importContext.context); + } catch (e) { + this.tick( + false, + `asset-type: ${uid}`, + (e as Error)?.message ?? PROCESS_STATUS[PROCESS_NAMES.AM_IMPORT_ASSET_TYPES].FAILED, + PROCESS_NAMES.AM_IMPORT_ASSET_TYPES, + ); + log.debug(`Failed to import asset type ${uid}: ${e}`, this.importContext.context); + } + }); + } +} diff --git a/packages/contentstack-asset-management/src/import/assets.ts b/packages/contentstack-asset-management/src/import/assets.ts new file mode 100644 index 00000000..14c5baed --- /dev/null +++ b/packages/contentstack-asset-management/src/import/assets.ts @@ -0,0 +1,280 @@ +import { resolve as pResolve, join } from 'node:path'; +import { existsSync, readFileSync } from 'node:fs'; +import { FsUtility, log } from '@contentstack/cli-utilities'; + +import type { AssetManagementAPIConfig, ImportContext } from '../types/asset-management-api'; +import { AssetManagementImportAdapter } from './base'; +import { getArrayFromResponse } from '../utils/export-helpers'; +import { runInBatches } from '../utils/concurrent-batch'; +import { forEachChunkRecordsFromFs } from '../utils/chunked-json-reader'; +import { PROCESS_NAMES, PROCESS_STATUS } from '../constants/index'; + +type FolderRecord = { + uid: string; + title: string; + description?: string; + parent_uid?: string; +}; + +type AssetRecord = { + uid: string; + url: string; + filename?: string; + file_name?: string; + parent_uid?: string; + title?: string; + description?: string; +}; + +type UploadJob = { + asset: AssetRecord; + filePath: string; + mappedParentUid: string | undefined; + oldUid: string; +}; + +/** + * Imports folders and assets for a single AM space. + * - Reads `spaces/{spaceUid}/assets/folders.json` → creates folders, builds folderUidMap + * - Reads chunked `assets.json` → uploads each file from `files/{oldUid}/{filename}` + * - Builds UID and URL mapper entries for entries.ts consumption + * Mirrors ExportAssets. + */ +export default class ImportAssets extends AssetManagementImportAdapter { + constructor(apiConfig: AssetManagementAPIConfig, importContext: ImportContext) { + super(apiConfig, importContext); + } + + private resolveAssetsChunkedLocation(spaceDir: string): { assetsDir: string; indexName: string } | null { + const assetsDir = pResolve(spaceDir, 'assets'); + const indexName = this.importContext.assetsFileName ?? 'assets.json'; + if (!existsSync(join(assetsDir, indexName))) { + return null; + } + return { assetsDir, indexName }; + } + + /** + * Build identity uid/url mappers from export JSON only (reuse path — no upload). + * Keys and values are equal so lookupAssets contract is satisfied without remapping. + */ + async buildIdentityMappersFromExport( + spaceDir: string, + ): Promise<{ uidMap: Record; urlMap: Record }> { + const uidMap: Record = {}; + const urlMap: Record = {}; + + const loc = this.resolveAssetsChunkedLocation(spaceDir); + if (!loc) { + log.debug( + `No assets.json index in ${pResolve(spaceDir, 'assets')}, identity mappers empty`, + this.importContext.context, + ); + return { uidMap, urlMap }; + } + + const fs = new FsUtility({ basePath: loc.assetsDir, indexFileName: loc.indexName }); + let totalRows = 0; + + await forEachChunkRecordsFromFs( + fs, + { context: this.importContext.context, chunkReadLogLabel: 'assets' }, + async (records) => { + totalRows += records.length; + for (const asset of records) { + if (asset.uid) { + uidMap[asset.uid] = asset.uid; + } + if (asset.url) { + urlMap[asset.url] = asset.url; + } + } + }, + ); + + log.debug( + `Built identity mappers for ${totalRows} exported asset row(s) (reuse path, chunked read)`, + this.importContext.context, + ); + + return { uidMap, urlMap }; + } + + async start( + newSpaceUid: string, + spaceDir: string, + ): Promise<{ uidMap: Record; urlMap: Record }> { + const assetsDir = pResolve(spaceDir, 'assets'); + const uidMap: Record = {}; + const urlMap: Record = {}; + + // ----------------------------------------------------------------------- + // 1. Import folders + // ----------------------------------------------------------------------- + const folderUidMap: Record = {}; + const foldersFileName = this.importContext.foldersFileName ?? 'folders.json'; + const foldersFilePath = join(assetsDir, foldersFileName); + + if (existsSync(foldersFilePath)) { + let foldersData: unknown; + try { + foldersData = JSON.parse(readFileSync(foldersFilePath, 'utf8')); + } catch (e) { + log.debug(`Could not read ${foldersFileName}: ${e}`, this.importContext.context); + } + + if (foldersData) { + const folders = getArrayFromResponse(foldersData, 'folders') as FolderRecord[]; + this.updateStatus(PROCESS_STATUS[PROCESS_NAMES.AM_IMPORT_FOLDERS].IMPORTING, PROCESS_NAMES.AM_IMPORT_FOLDERS); + log.debug(`Importing ${folders.length} folder(s) for space ${newSpaceUid}`, this.importContext.context); + await this.importFolders(newSpaceUid, folders, folderUidMap); + } + } + + // ----------------------------------------------------------------------- + // 2. Import assets (chunked on disk — process one chunk file at a time) + // ----------------------------------------------------------------------- + const loc = this.resolveAssetsChunkedLocation(spaceDir); + if (!loc) { + log.debug(`No assets.json index found in ${assetsDir}, skipping asset upload`, this.importContext.context); + return { uidMap, urlMap }; + } + + this.updateStatus(PROCESS_STATUS[PROCESS_NAMES.AM_IMPORT_ASSETS].IMPORTING, PROCESS_NAMES.AM_IMPORT_ASSETS); + log.debug( + `Uploading assets for space ${newSpaceUid} from chunked export (incremental chunks)`, + this.importContext.context, + ); + + const assetFs = new FsUtility({ basePath: loc.assetsDir, indexFileName: loc.indexName }); + let exportRowCount = 0; + + await forEachChunkRecordsFromFs( + assetFs, + { context: this.importContext.context, chunkReadLogLabel: 'assets' }, + async (assetChunk) => { + exportRowCount += assetChunk.length; + const uploadJobs: UploadJob[] = []; + + for (const asset of assetChunk) { + const oldUid = asset.uid; + const filename = asset.filename ?? asset.file_name ?? 'asset'; + const filePath = pResolve(assetsDir, 'files', oldUid, filename); + + if (!existsSync(filePath)) { + log.debug(`Asset file not found: ${filePath}, skipping`, this.importContext.context); + this.tick(false, `asset: ${oldUid}`, 'File not found on disk', PROCESS_NAMES.AM_IMPORT_ASSETS); + continue; + } + + const assetParent = asset.parent_uid && asset.parent_uid !== 'root' ? asset.parent_uid : undefined; + const mappedParentUid = assetParent ? folderUidMap[assetParent] ?? undefined : undefined; + + uploadJobs.push({ asset, filePath, mappedParentUid, oldUid }); + } + + await runInBatches( + uploadJobs, + this.uploadAssetsBatchConcurrency, + async ({ asset, filePath, mappedParentUid, oldUid }) => { + const filename = asset.filename ?? asset.file_name ?? 'asset'; + try { + const { asset: created } = await this.uploadAsset(newSpaceUid, filePath, { + title: asset.title ?? filename, + description: asset.description, + parent_uid: mappedParentUid, + }); + + uidMap[oldUid] = created.uid; + + if (asset.url && created.url) { + urlMap[asset.url] = created.url; + } + + this.tick(true, `asset: ${oldUid}`, null, PROCESS_NAMES.AM_IMPORT_ASSETS); + log.debug(`Uploaded asset ${oldUid} → ${created.uid}`, this.importContext.context); + } catch (e) { + this.tick( + false, + `asset: ${oldUid}`, + (e as Error)?.message ?? PROCESS_STATUS[PROCESS_NAMES.AM_IMPORT_ASSETS].FAILED, + PROCESS_NAMES.AM_IMPORT_ASSETS, + ); + log.debug(`Failed to upload asset ${oldUid}: ${e}`, this.importContext.context); + } + }, + ); + }, + ); + + log.debug( + `Finished asset uploads for space ${newSpaceUid} (${exportRowCount} row(s) read from export chunks)`, + this.importContext.context, + ); + + return { uidMap, urlMap }; + } + + /** + * Creates folders respecting hierarchy: parents before children. + * Uses multiple passes to handle arbitrary depth without requiring sorted input. + */ + private async importFolders( + newSpaceUid: string, + folders: FolderRecord[], + folderUidMap: Record, + ): Promise { + let remaining = [...folders]; + let prevLength = -1; + + while (remaining.length > 0 && remaining.length !== prevLength) { + prevLength = remaining.length; + const ready: FolderRecord[] = []; + const nextPass: FolderRecord[] = []; + + for (const folder of remaining) { + const { parent_uid: parentUid } = folder; + const isRootParent = !parentUid || parentUid === 'root'; + const parentMapped = isRootParent || folderUidMap[parentUid] !== undefined; + + if (!parentMapped) { + nextPass.push(folder); + } else { + ready.push(folder); + } + } + + await runInBatches(ready, this.importFoldersBatchConcurrency, async (folder) => { + const { parent_uid: parentUid } = folder; + const isRootParent = !parentUid || parentUid === 'root'; + try { + const { folder: created } = await this.createFolder(newSpaceUid, { + title: folder.title, + description: folder.description, + parent_uid: isRootParent ? undefined : folderUidMap[parentUid!], + }); + folderUidMap[folder.uid] = created.uid; + this.tick(true, `folder: ${folder.uid}`, null, PROCESS_NAMES.AM_IMPORT_FOLDERS); + log.debug(`Created folder ${folder.uid} → ${created.uid}`, this.importContext.context); + } catch (e) { + this.tick( + false, + `folder: ${folder.uid}`, + (e as Error)?.message ?? PROCESS_STATUS[PROCESS_NAMES.AM_IMPORT_FOLDERS].FAILED, + PROCESS_NAMES.AM_IMPORT_FOLDERS, + ); + log.debug(`Failed to create folder ${folder.uid}: ${e}`, this.importContext.context); + } + }); + + remaining = nextPass; + } + + if (remaining.length > 0) { + log.debug( + `${remaining.length} folder(s) could not be imported (unresolved parent UIDs)`, + this.importContext.context, + ); + } + } +} diff --git a/packages/contentstack-asset-management/src/import/base.ts b/packages/contentstack-asset-management/src/import/base.ts new file mode 100644 index 00000000..ef1d4c0f --- /dev/null +++ b/packages/contentstack-asset-management/src/import/base.ts @@ -0,0 +1,86 @@ +import { resolve as pResolve } from 'node:path'; +import { CLIProgressManager, configHandler } from '@contentstack/cli-utilities'; + +import type { AssetManagementAPIConfig, ImportContext } from '../types/asset-management-api'; +import { AssetManagementAdapter } from '../utils/asset-management-api-adapter'; +import { AM_MAIN_PROCESS_NAME, FALLBACK_AM_API_CONCURRENCY } from '../constants/index'; + +export type { ImportContext }; + +/** + * Base class for all AM 2.0 import modules. Mirrors AssetManagementExportAdapter + * but carries ImportContext (spacesRootPath, apiKey, host, etc.) instead of ExportContext. + */ +export class AssetManagementImportAdapter extends AssetManagementAdapter { + protected readonly apiConfig: AssetManagementAPIConfig; + protected readonly importContext: ImportContext; + protected progressManager: CLIProgressManager | null = null; + protected parentProgressManager: CLIProgressManager | null = null; + protected readonly processName: string = AM_MAIN_PROCESS_NAME; + + constructor(apiConfig: AssetManagementAPIConfig, importContext: ImportContext) { + super(apiConfig); + this.apiConfig = apiConfig; + this.importContext = importContext; + } + + public setParentProgressManager(parent: CLIProgressManager): void { + this.parentProgressManager = parent; + } + + protected get progressOrParent(): CLIProgressManager | null { + return this.parentProgressManager ?? this.progressManager; + } + + protected createNestedProgress(moduleName: string): CLIProgressManager { + if (this.parentProgressManager) { + this.progressManager = this.parentProgressManager; + return this.parentProgressManager; + } + const logConfig = configHandler.get('log') || {}; + const showConsoleLogs = logConfig.showConsoleLogs ?? false; + this.progressManager = CLIProgressManager.createNested(moduleName, showConsoleLogs); + return this.progressManager; + } + + protected tick(success: boolean, itemName: string, error: string | null, processName?: string): void { + this.progressOrParent?.tick?.(success, itemName, error, processName ?? this.processName); + } + + protected updateStatus(message: string, processName?: string): void { + this.progressOrParent?.updateStatus?.(message, processName ?? this.processName); + } + + protected completeProcess(processName: string, success: boolean): void { + if (!this.parentProgressManager) { + this.progressManager?.completeProcess?.(processName, success); + } + } + + protected get spacesRootPath(): string { + return this.importContext.spacesRootPath; + } + + /** Parallel AM API limit for import batches. */ + protected get apiConcurrency(): number { + return this.importContext.apiConcurrency ?? FALLBACK_AM_API_CONCURRENCY; + } + + /** Upload batch size; falls back to {@link apiConcurrency}. */ + protected get uploadAssetsBatchConcurrency(): number { + return this.importContext.uploadAssetsConcurrency ?? this.apiConcurrency; + } + + /** Folder creation batch size; falls back to {@link apiConcurrency}. */ + protected get importFoldersBatchConcurrency(): number { + return this.importContext.importFoldersConcurrency ?? this.apiConcurrency; + } + + protected getAssetTypesDir(): string { + return pResolve(this.importContext.spacesRootPath, this.importContext.assetTypesDir ?? 'asset_types'); + } + + protected getFieldsDir(): string { + return pResolve(this.importContext.spacesRootPath, this.importContext.fieldsDir ?? 'fields'); + } +} diff --git a/packages/contentstack-asset-management/src/import/fields.ts b/packages/contentstack-asset-management/src/import/fields.ts new file mode 100644 index 00000000..1e431b14 --- /dev/null +++ b/packages/contentstack-asset-management/src/import/fields.ts @@ -0,0 +1,133 @@ +import { existsSync } from 'node:fs'; +import { join } from 'node:path'; +import omit from 'lodash/omit'; +import isEqual from 'lodash/isEqual'; +import { log } from '@contentstack/cli-utilities'; + +import type { AssetManagementAPIConfig, ImportContext } from '../types/asset-management-api'; +import { AssetManagementImportAdapter } from './base'; +import { FALLBACK_FIELDS_IMPORT_INVALID_KEYS, PROCESS_NAMES, PROCESS_STATUS } from '../constants/index'; +import { runInBatches } from '../utils/concurrent-batch'; +import { forEachChunkedJsonStore } from '../utils/chunked-json-reader'; + +type FieldToCreate = { uid: string; payload: Record }; + +/** + * Reads shared fields from `spaces/fields/fields.json` and POSTs each to the + * target org-level AM fields endpoint (`POST /api/fields`). + * + * Strategy: Fetch → Diff → Create only missing, warn on conflict + * 1. Fetch fields that already exist in the target org. + * 2. Skip entries where is_system=true (platform-owned, cannot be created via API). + * 3. If uid already exists and definition differs → warn and skip. + * 4. If uid already exists and definition matches → silently skip. + * 5. Strip read-only/computed keys from the POST body before creating new fields. + */ +export default class ImportFields extends AssetManagementImportAdapter { + constructor(apiConfig: AssetManagementAPIConfig, importContext: ImportContext) { + super(apiConfig, importContext); + } + + async start(): Promise { + await this.init(); + + const stripKeys = this.importContext.fieldsImportInvalidKeys ?? [...FALLBACK_FIELDS_IMPORT_INVALID_KEYS]; + const dir = this.getFieldsDir(); + const indexName = this.importContext.fieldsFileName ?? 'fields.json'; + const indexPath = join(dir, indexName); + + if (!existsSync(indexPath)) { + log.debug('No shared fields to import (index missing)', this.importContext.context); + return; + } + + const existingByUid = await this.loadExistingFieldsMap(); + + this.updateStatus(PROCESS_STATUS[PROCESS_NAMES.AM_IMPORT_FIELDS].IMPORTING, PROCESS_NAMES.AM_IMPORT_FIELDS); + + await forEachChunkedJsonStore>( + dir, + indexName, + { + context: this.importContext.context, + chunkReadLogLabel: 'fields', + onOpenError: (e) => log.debug(`Could not open chunked fields index: ${e}`, this.importContext.context), + onEmptyIndexer: () => log.debug('No shared fields to import (empty indexer)', this.importContext.context), + }, + async (records) => { + const toCreate = this.buildFieldsToCreate(records, existingByUid, stripKeys); + await this.importFieldsCreates(toCreate); + }, + ); + } + + /** Org-level fields keyed by uid for diff; empty map if list API fails. */ + private async loadExistingFieldsMap(): Promise>> { + const existingByUid = new Map>(); + try { + const existing = await this.getWorkspaceFields(''); + for (const f of existing.fields ?? []) { + existingByUid.set(f.uid, f as Record); + } + log.debug(`Target org has ${existingByUid.size} existing field(s)`, this.importContext.context); + } catch (e) { + log.debug(`Could not fetch existing fields, will attempt to create all: ${e}`, this.importContext.context); + } + return existingByUid; + } + + private buildFieldsToCreate( + items: Record[], + existingByUid: Map>, + stripKeys: string[], + ): FieldToCreate[] { + const toCreate: FieldToCreate[] = []; + + for (const field of items) { + const uid = field.uid as string; + + if (field.is_system) { + log.debug(`Skipping system field: ${uid}`, this.importContext.context); + continue; + } + + const existing = existingByUid.get(uid); + if (existing) { + const exportedClean = omit(field, stripKeys); + const existingClean = omit(existing, stripKeys); + if (!isEqual(exportedClean, existingClean)) { + log.warn( + `Field "${uid}" already exists in the target org with a different definition. Skipping — to apply the exported definition, delete the field from the target org first.`, + this.importContext.context, + ); + } else { + log.debug(`Field "${uid}" already exists with matching definition, skipping`, this.importContext.context); + } + this.tick(true, `field: ${uid} (skipped, already exists)`, null, PROCESS_NAMES.AM_IMPORT_FIELDS); + continue; + } + + toCreate.push({ uid, payload: omit(field, stripKeys) as Record }); + } + + return toCreate; + } + + private async importFieldsCreates(toCreate: FieldToCreate[]): Promise { + await runInBatches(toCreate, this.apiConcurrency, async ({ uid, payload }) => { + try { + await this.createField(payload as any); + this.tick(true, `field: ${uid}`, null, PROCESS_NAMES.AM_IMPORT_FIELDS); + log.debug(`Imported field: ${uid}`, this.importContext.context); + } catch (e) { + this.tick( + false, + `field: ${uid}`, + (e as Error)?.message ?? PROCESS_STATUS[PROCESS_NAMES.AM_IMPORT_FIELDS].FAILED, + PROCESS_NAMES.AM_IMPORT_FIELDS, + ); + log.debug(`Failed to import field ${uid}: ${e}`, this.importContext.context); + } + }); + } +} diff --git a/packages/contentstack-asset-management/src/import/index.ts b/packages/contentstack-asset-management/src/import/index.ts new file mode 100644 index 00000000..61d8a457 --- /dev/null +++ b/packages/contentstack-asset-management/src/import/index.ts @@ -0,0 +1,7 @@ +export { ImportSpaces } from './spaces'; +export { default as ImportWorkspace } from './workspaces'; +export { default as ImportAssets } from './assets'; +export { default as ImportFields } from './fields'; +export { default as ImportAssetTypes } from './asset-types'; +export { AssetManagementImportAdapter } from './base'; +export type { ImportContext } from './base'; diff --git a/packages/contentstack-asset-management/src/import/spaces.ts b/packages/contentstack-asset-management/src/import/spaces.ts new file mode 100644 index 00000000..c2642baa --- /dev/null +++ b/packages/contentstack-asset-management/src/import/spaces.ts @@ -0,0 +1,222 @@ +import { resolve as pResolve, join } from 'node:path'; +import { mkdirSync, readdirSync, statSync } from 'node:fs'; +import { writeFile } from 'node:fs/promises'; +import { log, CLIProgressManager, configHandler } from '@contentstack/cli-utilities'; + +import type { + AssetManagementAPIConfig, + AssetManagementImportOptions, + ImportContext, + ImportResult, + SpaceMapping, +} from '../types/asset-management-api'; +import { AM_MAIN_PROCESS_NAME } from '../constants/index'; +import { AssetManagementAdapter } from '../utils/asset-management-api-adapter'; +import ImportAssetTypes from './asset-types'; +import ImportFields from './fields'; +import ImportWorkspace from './workspaces'; + +/** + * Top-level orchestrator for AM 2.0 import. + * Mirrors ExportSpaces: creates shared fields + asset types, then imports each space. + * Returns combined uidMap, urlMap, and spaceMappings for the bridge module. + */ +export class ImportSpaces { + private readonly options: AssetManagementImportOptions; + private parentProgressManager: CLIProgressManager | null = null; + private progressManager: CLIProgressManager | null = null; + + constructor(options: AssetManagementImportOptions) { + this.options = options; + } + + public setParentProgressManager(parent: CLIProgressManager): void { + this.parentProgressManager = parent; + } + + async start(): Promise { + const { + contentDir, + assetManagementUrl, + org_uid, + apiKey, + host, + sourceApiKey, + context, + apiConcurrency, + uploadAssetsConcurrency, + importFoldersConcurrency, + spacesDirName, + fieldsDir, + assetTypesDir, + fieldsFileName, + assetTypesFileName, + foldersFileName, + assetsFileName, + fieldsImportInvalidKeys, + assetTypesImportInvalidKeys, + mapperRootDir, + mapperAssetsModuleDir, + mapperUidFileName, + mapperUrlFileName, + mapperSpaceUidFileName, + } = this.options; + + const spacesRootPath = pResolve(contentDir, spacesDirName ?? 'spaces'); + + const importContext: ImportContext = { + spacesRootPath, + sourceApiKey, + apiKey, + host, + org_uid, + context, + apiConcurrency, + uploadAssetsConcurrency, + importFoldersConcurrency, + spacesDirName, + fieldsDir, + assetTypesDir, + fieldsFileName, + assetTypesFileName, + foldersFileName, + assetsFileName, + fieldsImportInvalidKeys, + assetTypesImportInvalidKeys, + mapperRootDir, + mapperAssetsModuleDir, + mapperUidFileName, + mapperUrlFileName, + mapperSpaceUidFileName, + }; + + const apiConfig: AssetManagementAPIConfig = { + baseURL: assetManagementUrl, + headers: { organization_uid: org_uid }, + context, + }; + + // Discover space directories + let spaceDirs: string[] = []; + try { + spaceDirs = readdirSync(spacesRootPath).filter((entry) => { + try { + return statSync(join(spacesRootPath, entry)).isDirectory() && entry.startsWith('am'); + } catch { + return false; + } + }); + } catch (e) { + log.debug(`Could not read spaces root path ${spacesRootPath}: ${e}`, context); + } + + const totalSteps = 2 + spaceDirs.length * 2; + const progress = this.createProgress(); + progress.addProcess(AM_MAIN_PROCESS_NAME, totalSteps); + progress.startProcess(AM_MAIN_PROCESS_NAME); + + const allUidMap: Record = {}; + const allUrlMap: Record = {}; + const allSpaceUidMap: Record = {}; + const spaceMappings: SpaceMapping[] = []; + let hasFailures = false; + + // Space UIDs already present in the target org — reuse when export dir name matches a uid here. + const existingSpaceUids = new Set(); + try { + const adapterForList = new AssetManagementAdapter(apiConfig); + await adapterForList.init(); + const { spaces } = await adapterForList.listSpaces(); + for (const s of spaces) { + if (s.uid) existingSpaceUids.add(s.uid); + } + log.debug(`Found ${existingSpaceUids.size} existing space uid(s) in target org`, context); + } catch (e) { + log.debug(`Could not fetch existing spaces — reuse-by-uid disabled: ${e}`, context); + } + + try { + // 1. Import shared fields + progress.updateStatus(`Importing shared fields...`, AM_MAIN_PROCESS_NAME); + const fieldsImporter = new ImportFields(apiConfig, importContext); + fieldsImporter.setParentProgressManager(progress); + await fieldsImporter.start(); + + // 2. Import shared asset types + progress.updateStatus('Importing shared asset types...', AM_MAIN_PROCESS_NAME); + const assetTypesImporter = new ImportAssetTypes(apiConfig, importContext); + assetTypesImporter.setParentProgressManager(progress); + await assetTypesImporter.start(); + + // 3. Import each space — continue on failure so partially-imported data is never lost + for (const spaceUid of spaceDirs) { + const spaceDir = join(spacesRootPath, spaceUid); + progress.updateStatus(`Importing space: ${spaceUid}...`, AM_MAIN_PROCESS_NAME); + log.debug(`Importing space: ${spaceUid}`, context); + + try { + const workspaceImporter = new ImportWorkspace(apiConfig, importContext); + workspaceImporter.setParentProgressManager(progress); + const result = await workspaceImporter.start(spaceUid, spaceDir, existingSpaceUids); + + // Newly created spaces get a new uid — add so later iterations in this run see it. + existingSpaceUids.add(result.newSpaceUid); + + Object.assign(allUidMap, result.uidMap); + Object.assign(allUrlMap, result.urlMap); + allSpaceUidMap[result.oldSpaceUid] = result.newSpaceUid; + spaceMappings.push({ + oldSpaceUid: result.oldSpaceUid, + newSpaceUid: result.newSpaceUid, + workspaceUid: result.workspaceUid, + isDefault: result.isDefault, + }); + + log.debug(`Imported space ${spaceUid} → ${result.newSpaceUid}`, context); + } catch (err) { + hasFailures = true; + progress.tick( + false, + `space: ${spaceUid}`, + (err as Error)?.message ?? 'Failed to import space', + AM_MAIN_PROCESS_NAME, + ); + log.debug(`Failed to import space ${spaceUid}: ${err}`, context); + } + } + + if (this.options.backupDir) { + const mapperRoot = this.options.mapperRootDir ?? 'mapper'; + const mapperAssetsMod = this.options.mapperAssetsModuleDir ?? 'assets'; + const mapperDir = join(this.options.backupDir, mapperRoot, mapperAssetsMod); + mkdirSync(mapperDir, { recursive: true }); + const uidFile = this.options.mapperUidFileName ?? 'uid-mapping.json'; + const urlFile = this.options.mapperUrlFileName ?? 'url-mapping.json'; + const spaceUidFile = this.options.mapperSpaceUidFileName ?? 'space-uid-mapping.json'; + await writeFile(join(mapperDir, uidFile), JSON.stringify(allUidMap), 'utf8'); + await writeFile(join(mapperDir, urlFile), JSON.stringify(allUrlMap), 'utf8'); + await writeFile(join(mapperDir, spaceUidFile), JSON.stringify(allSpaceUidMap), 'utf8'); + log.debug('Wrote AM 2.0 mapper files (uid, url, space-uid)', context); + } + + progress.completeProcess(AM_MAIN_PROCESS_NAME, !hasFailures); + log.debug('Asset Management 2.0 import completed', context); + } catch (err) { + progress.completeProcess(AM_MAIN_PROCESS_NAME, false); + throw err; + } + + return { uidMap: allUidMap, urlMap: allUrlMap, spaceMappings, spaceUidMap: allSpaceUidMap }; + } + + private createProgress(): CLIProgressManager { + if (this.parentProgressManager) { + this.progressManager = this.parentProgressManager; + return this.parentProgressManager; + } + const logConfig = configHandler.get('log') || {}; + const showConsoleLogs = logConfig.showConsoleLogs ?? false; + this.progressManager = CLIProgressManager.createNested(AM_MAIN_PROCESS_NAME, showConsoleLogs); + return this.progressManager; + } +} diff --git a/packages/contentstack-asset-management/src/import/workspaces.ts b/packages/contentstack-asset-management/src/import/workspaces.ts new file mode 100644 index 00000000..5d13450d --- /dev/null +++ b/packages/contentstack-asset-management/src/import/workspaces.ts @@ -0,0 +1,82 @@ +import { join } from 'node:path'; +import { readFileSync } from 'node:fs'; +import { log } from '@contentstack/cli-utilities'; + +import type { AssetManagementAPIConfig, ImportContext, SpaceMapping } from '../types/asset-management-api'; +import { AssetManagementImportAdapter } from './base'; +import ImportAssets from './assets'; +import { PROCESS_NAMES } from '../constants/index'; + +type WorkspaceResult = SpaceMapping & { + uidMap: Record; + urlMap: Record; +}; + +/** + * Handles import for a single AM 2.0 space directory. + * Reads `metadata.json`, creates the space in the target org when its uid is not + * already present, or reuses the existing space and emits identity mappers only. + * Returns the SpaceMapping plus UID/URL maps for the mapper files. + */ +export default class ImportWorkspace extends AssetManagementImportAdapter { + constructor(apiConfig: AssetManagementAPIConfig, importContext: ImportContext) { + super(apiConfig, importContext); + } + + async start( + oldSpaceUid: string, + spaceDir: string, + existingSpaceUids: Set = new Set(), + ): Promise { + await this.init(); + + // Read exported metadata + const metadataPath = join(spaceDir, 'metadata.json'); + let metadata: Record = {}; + try { + metadata = JSON.parse(readFileSync(metadataPath, 'utf8')) as Record; + } catch (e) { + log.debug(`Could not read metadata.json for space ${oldSpaceUid}: ${e}`, this.importContext.context); + } + + const exportedTitle = (metadata.title as string) ?? oldSpaceUid; + const description = metadata.description as string | undefined; + const isDefault = (metadata.is_default as boolean) ?? false; + const workspaceUid = 'main'; + + const assetsImporter = new ImportAssets(this.apiConfig, this.importContext); + if (this.progressOrParent) assetsImporter.setParentProgressManager(this.progressOrParent); + + // Reuse: target org already has a space with the same uid as the export directory. + if (existingSpaceUids.has(oldSpaceUid)) { + log.info( + `Reusing existing AM space "${oldSpaceUid}" (uid matches export directory); skipping create and upload.`, + this.importContext.context, + ); + const newSpaceUid = oldSpaceUid; + const { uidMap, urlMap } = await assetsImporter.buildIdentityMappersFromExport(spaceDir); + this.tick(true, `space: ${oldSpaceUid} → ${newSpaceUid} (reused)`, null, PROCESS_NAMES.AM_SPACE_METADATA); + return { + oldSpaceUid, + newSpaceUid, + workspaceUid, + isDefault, + uidMap, + urlMap, + }; + } + + // Create new space with exact exported title + log.debug(`Creating space "${exportedTitle}" (old uid: ${oldSpaceUid})`, this.importContext.context); + + const { space } = await this.createSpace({ title: exportedTitle, description }); + const newSpaceUid = space.uid; + + log.debug(`Created space ${newSpaceUid} (old: ${oldSpaceUid})`, this.importContext.context); + this.tick(true, `space: ${oldSpaceUid} → ${newSpaceUid}`, null, PROCESS_NAMES.AM_SPACE_METADATA); + + const { uidMap, urlMap } = await assetsImporter.start(newSpaceUid, spaceDir); + + return { oldSpaceUid, newSpaceUid, workspaceUid, isDefault, uidMap, urlMap }; + } +} diff --git a/packages/contentstack-asset-management/src/index.ts b/packages/contentstack-asset-management/src/index.ts index f0ff59bd..c66c638d 100644 --- a/packages/contentstack-asset-management/src/index.ts +++ b/packages/contentstack-asset-management/src/index.ts @@ -2,3 +2,4 @@ export * from './constants/index'; export * from './types'; export * from './utils'; export * from './export'; +export * from './import'; diff --git a/packages/contentstack-asset-management/src/types/asset-management-api.ts b/packages/contentstack-asset-management/src/types/asset-management-api.ts index 733ecada..65ecfb3d 100644 --- a/packages/contentstack-asset-management/src/types/asset-management-api.ts +++ b/packages/contentstack-asset-management/src/types/asset-management-api.ts @@ -36,6 +36,9 @@ export type Space = { /** Response shape of GET /api/spaces/{space_uid}. */ export type SpaceResponse = { space: Space }; +/** Response shape of GET /api/spaces (list all spaces in the org). */ +export type SpacesListResponse = { spaces: Space[]; count?: number }; + /** * Field structure from GET /api/fields (org-level). */ @@ -118,10 +121,11 @@ export type AssetManagementAPIConfig = { */ export interface IAssetManagementAdapter { init(): Promise; + listSpaces(): Promise; getSpace(spaceUid: string): Promise; getWorkspaceFields(spaceUid: string): Promise; - getWorkspaceAssets(spaceUid: string): Promise; - getWorkspaceFolders(spaceUid: string): Promise; + getWorkspaceAssets(spaceUid: string, workspaceUid?: string): Promise; + getWorkspaceFolders(spaceUid: string, workspaceUid?: string): Promise; getWorkspaceAssetTypes(spaceUid: string): Promise; } @@ -137,4 +141,170 @@ export type AssetManagementExportOptions = { context?: Record; /** When true, the AM package will add authtoken to asset download URLs. */ securedAssets?: boolean; + /** + * API key of the stack being exported. + * Saved to `spaces/export-metadata.json` so that during import the URL mapper + * can reconstruct old CMA proxy URLs (format: /v3/assets/{apiKey}/{amUid}/...). + */ + apiKey?: string; + /** + * FsUtility `chunkFileSize` in MB for AM export chunked writes. + */ + chunkFileSizeMb?: number; +}; + +// --------------------------------------------------------------------------- +// Import types +// --------------------------------------------------------------------------- + +/** + * Context passed down to every import adapter class. + * Mirrors ExportContext but carries the import-specific fields needed for + * URL mapper reconstruction and API calls. + */ +export type ImportContext = { + /** Absolute path to the root `spaces/` directory inside the backup/content dir. */ + spacesRootPath: string; + /** Source stack API key — used to reconstruct old CMA proxy URLs. */ + sourceApiKey?: string; + /** Target stack API key — used to build new CMA proxy URLs. */ + apiKey: string; + /** Target CMA host (may include /v3), e.g. "https://api.contentstack.io/v3". */ + host: string; + /** Target org UID — required as `x-organization-uid` header when creating spaces. */ + org_uid: string; + /** Optional logging context (same shape as ExportConfig.context). */ + context?: Record; + /** + * Max parallel AM API calls for import (fields, asset types, and default for folders/uploads). + * Set from `AssetManagementImportOptions.apiConcurrency`. + */ + apiConcurrency?: number; + /** Overrides parallel limit for asset uploads when set (import `modules['asset-management'].uploadAssetsConcurrency`). */ + uploadAssetsConcurrency?: number; + /** Overrides parallel limit for folder creation batches when set (import `modules['asset-management'].importFoldersConcurrency`). */ + importFoldersConcurrency?: number; + /** Relative dir under content dir for AM export root (e.g. `spaces`). */ + spacesDirName?: string; + fieldsDir?: string; + assetTypesDir?: string; + fieldsFileName?: string; + assetTypesFileName?: string; + foldersFileName?: string; + assetsFileName?: string; + fieldsImportInvalidKeys?: string[]; + assetTypesImportInvalidKeys?: string[]; + /** `{backupDir}/{mapperRootDir}/{mapperAssetsModuleDir}/` for AM mapper JSON. */ + mapperRootDir?: string; + mapperAssetsModuleDir?: string; + mapperUidFileName?: string; + mapperUrlFileName?: string; + mapperSpaceUidFileName?: string; +}; + +/** + * Options accepted by the top-level `ImportSpaces` class. + */ +export type AssetManagementImportOptions = { + /** Absolute path to the root content / backup directory. */ + contentDir: string; + /** AM 2.0 base URL (e.g. "https://am.contentstack.io"). */ + assetManagementUrl: string; + /** Target organisation UID. */ + org_uid: string; + /** Target stack API key. */ + apiKey: string; + /** Target CMA host. */ + host: string; + /** Source stack API key — used for old CMA proxy URL reconstruction. */ + sourceApiKey?: string; + /** Optional logging context. */ + context?: Record; + /** + * When set, mapper files are written under `{backupDir}/mapper/assets/` after import. + */ + backupDir?: string; + /** Parallel AM API limit; defaults to package constant when omitted. */ + apiConcurrency?: number; + uploadAssetsConcurrency?: number; + importFoldersConcurrency?: number; + spacesDirName?: string; + fieldsDir?: string; + assetTypesDir?: string; + fieldsFileName?: string; + assetTypesFileName?: string; + foldersFileName?: string; + assetsFileName?: string; + fieldsImportInvalidKeys?: string[]; + assetTypesImportInvalidKeys?: string[]; + mapperRootDir?: string; + mapperAssetsModuleDir?: string; + mapperUidFileName?: string; + mapperUrlFileName?: string; + mapperSpaceUidFileName?: string; +}; + +/** + * Maps an old source-org space UID to the newly created target-org space UID. + */ +export type SpaceMapping = { + oldSpaceUid: string; + newSpaceUid: string; + /** Workspace identifier inside the space (typically "main"). */ + workspaceUid: string; + isDefault: boolean; +}; + +/** + * The value returned by `ImportSpaces.start()`. + * When `backupDir` is set on options, the AM package also writes these maps under + * `mapper/assets/` for `entries.ts` to resolve asset references. + */ +export type ImportResult = { + uidMap: Record; + urlMap: Record; + spaceMappings: SpaceMapping[]; + /** old space UID → new space UID, written to mapper/assets/space-uid-mapping.json */ + spaceUidMap: Record; +}; + +// --------------------------------------------------------------------------- +// Import payload types (confirmed from Postman collection) +// --------------------------------------------------------------------------- + +export type CreateSpacePayload = { + title: string; + description?: string; +}; + +export type CreateFolderPayload = { + title: string; + description?: string; + parent_uid?: string; +}; + +export type CreateAssetMetadata = { + title?: string; + description?: string; + parent_uid?: string; +}; + +export type CreateFieldPayload = { + uid: string; + title: string; + display_type?: string; + child?: unknown[]; + is_mandatory?: boolean; + is_multiple?: boolean; + [key: string]: unknown; +}; + +export type CreateAssetTypePayload = { + uid: string; + title: string; + description?: string; + content_type?: string; + file_extension?: string | string[]; + fields?: string[]; + [key: string]: unknown; }; diff --git a/packages/contentstack-asset-management/src/types/export-types.ts b/packages/contentstack-asset-management/src/types/export-types.ts index 25b8dcac..7cefa319 100644 --- a/packages/contentstack-asset-management/src/types/export-types.ts +++ b/packages/contentstack-asset-management/src/types/export-types.ts @@ -2,6 +2,7 @@ export type ExportContext = { spacesRootPath: string; context?: Record; securedAssets?: boolean; + chunkFileSizeMb?: number; }; /** diff --git a/packages/contentstack-asset-management/src/utils/asset-management-api-adapter.ts b/packages/contentstack-asset-management/src/utils/asset-management-api-adapter.ts index b159cc33..b26b2664 100644 --- a/packages/contentstack-asset-management/src/utils/asset-management-api-adapter.ts +++ b/packages/contentstack-asset-management/src/utils/asset-management-api-adapter.ts @@ -1,11 +1,20 @@ +import { readFileSync } from 'node:fs'; +import { basename } from 'node:path'; import { HttpClient, log, authenticationHandler } from '@contentstack/cli-utilities'; import type { AssetManagementAPIConfig, AssetTypesResponse, + CreateAssetMetadata, + CreateAssetTypePayload, + CreateFieldPayload, + CreateFolderPayload, + CreateSpacePayload, FieldsResponse, IAssetManagementAdapter, + Space, SpaceResponse, + SpacesListResponse, } from '../types/asset-management-api'; export class AssetManagementAdapter implements IAssetManagementAdapter { @@ -89,6 +98,13 @@ export class AssetManagementAdapter implements IAssetManagementAdapter { } } + async listSpaces(): Promise { + log.debug('Fetching all spaces in org', this.config.context); + const result = await this.getSpaceLevel('', '/api/spaces', {}); + log.debug(`Fetched ${result?.count ?? result?.spaces?.length ?? '?'} space(s)`, this.config.context); + return result; + } + async getSpace(spaceUid: string): Promise { log.debug(`Fetching space: ${spaceUid}`, this.config.context); const path = `/api/spaces/${spaceUid}`; @@ -114,27 +130,30 @@ export class AssetManagementAdapter implements IAssetManagementAdapter { spaceUid: string, path: string, logLabel: string, + queryParams: Record = {}, ): Promise { log.debug(`Fetching ${logLabel} for space: ${spaceUid}`, this.config.context); - const result = await this.getSpaceLevel(spaceUid, path, {}); + const result = await this.getSpaceLevel(spaceUid, path, queryParams); const count = (result as { count?: number })?.count ?? (Array.isArray(result) ? result.length : '?'); log.debug(`Fetched ${logLabel} (count: ${count})`, this.config.context); return result; } - async getWorkspaceAssets(spaceUid: string): Promise { + async getWorkspaceAssets(spaceUid: string, workspaceUid?: string): Promise { return this.getWorkspaceCollection( spaceUid, `/api/spaces/${encodeURIComponent(spaceUid)}/assets`, 'assets', + workspaceUid ? { workspace: workspaceUid } : {}, ); } - async getWorkspaceFolders(spaceUid: string): Promise { + async getWorkspaceFolders(spaceUid: string, workspaceUid?: string): Promise { return this.getWorkspaceCollection( spaceUid, `/api/spaces/${encodeURIComponent(spaceUid)}/folders`, 'folders', + workspaceUid ? { workspace: workspaceUid } : {}, ); } @@ -146,4 +165,118 @@ export class AssetManagementAdapter implements IAssetManagementAdapter { log.debug(`Fetched asset types (count: ${result?.count ?? '?'})`, this.config.context); return result; } + + // --------------------------------------------------------------------------- + // POST helpers + // --------------------------------------------------------------------------- + + /** + * Build headers for outgoing POST requests. + */ + private async getPostHeaders(extraHeaders: Record = {}): Promise> { + await authenticationHandler.getAuthDetails(); + const token = authenticationHandler.accessToken; + const authHeader: Record = authenticationHandler.isOauthEnabled + ? { authorization: token } + : { access_token: token }; + return { + Accept: 'application/json', + 'x-cs-api-version': '4', + ...(this.config.headers ?? {}), + ...authHeader, + ...extraHeaders, + }; + } + + private async postJson(path: string, body: unknown, extraHeaders: Record = {}): Promise { + const baseUrl = this.config.baseURL?.replace(/\/$/, '') ?? ''; + const headers = await this.getPostHeaders({ 'Content-Type': 'application/json', ...extraHeaders }); + log.debug(`POST ${path}`, this.config.context); + const response = await fetch(`${baseUrl}${path}`, { + method: 'POST', + headers, + body: JSON.stringify(body), + }); + if (!response.ok) { + const text = await response.text().catch(() => ''); + throw new Error(`AM API POST error: status ${response.status}, path ${path}, body: ${text}`); + } + return response.json() as Promise; + } + + private async postMultipart(path: string, form: FormData, extraHeaders: Record = {}): Promise { + const baseUrl = this.config.baseURL?.replace(/\/$/, '') ?? ''; + const headers = await this.getPostHeaders(extraHeaders); + log.debug(`POST (multipart) ${path}`, this.config.context); + const response = await fetch(`${baseUrl}${path}`, { + method: 'POST', + headers, + body: form, + }); + if (!response.ok) { + const text = await response.text().catch(() => ''); + throw new Error(`AM API multipart POST error: status ${response.status}, path ${path}, body: ${text}`); + } + return response.json() as Promise; + } + + // --------------------------------------------------------------------------- + // Import API methods + // --------------------------------------------------------------------------- + + /** + * POST /api/spaces — creates a new space in the target org. + */ + async createSpace(payload: CreateSpacePayload): Promise<{ space: Space }> { + const orgUid = (this.config.headers as Record | undefined)?.organization_uid ?? ''; + return this.postJson<{ space: Space }>('/api/spaces', payload, { + 'x-organization-uid': orgUid, + }); + } + + /** + * POST /api/spaces/{spaceUid}/folders — creates a folder inside a space. + */ + async createFolder(spaceUid: string, payload: CreateFolderPayload): Promise<{ folder: { uid: string } }> { + return this.postJson<{ folder: { uid: string } }>(`/api/spaces/${encodeURIComponent(spaceUid)}/folders`, payload, { + space_key: spaceUid, + }); + } + + /** + * POST /api/spaces/{spaceUid}/assets — uploads an asset file as multipart form-data. + */ + async uploadAsset( + spaceUid: string, + filePath: string, + metadata: CreateAssetMetadata, + ): Promise<{ asset: { uid: string; url: string } }> { + const filename = basename(filePath); + const fileBuffer = readFileSync(filePath); + const blob = new Blob([fileBuffer]); + const form = new FormData(); + form.append('file', blob, filename); + if (metadata.title) form.append('title', metadata.title); + if (metadata.description) form.append('description', metadata.description); + if (metadata.parent_uid) form.append('parent_uid', metadata.parent_uid); + return this.postMultipart<{ asset: { uid: string; url: string } }>( + `/api/spaces/${encodeURIComponent(spaceUid)}/assets`, + form, + { space_key: spaceUid }, + ); + } + + /** + * POST /api/fields — creates a shared field. + */ + async createField(payload: CreateFieldPayload): Promise<{ field: { uid: string } }> { + return this.postJson<{ field: { uid: string } }>('/api/fields', payload); + } + + /** + * POST /api/asset_types — creates a shared asset type. + */ + async createAssetType(payload: CreateAssetTypePayload): Promise<{ asset_type: { uid: string } }> { + return this.postJson<{ asset_type: { uid: string } }>('/api/asset_types', payload); + } } diff --git a/packages/contentstack-asset-management/src/utils/chunked-json-reader.ts b/packages/contentstack-asset-management/src/utils/chunked-json-reader.ts new file mode 100644 index 00000000..838cfe65 --- /dev/null +++ b/packages/contentstack-asset-management/src/utils/chunked-json-reader.ts @@ -0,0 +1,66 @@ +import { FsUtility, log } from '@contentstack/cli-utilities'; + +export type ForEachChunkedJsonStoreOptions = { + context?: Record; + /** Shown in log.debug: `Error reading