Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 3 additions & 2 deletions .talismanrc
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
fileignoreconfig:

98236e2588d5cffcbdda3464179d91e9540fd8ddb4d4fb5251cf0f7379876f42
version: ""
- filename: pnpm-lock.yaml
checksum: 98236e2588d5cffcbdda3464179d91e9540fd8ddb4d4fb5251cf0f7379876f42
version: ""
17 changes: 10 additions & 7 deletions packages/contentstack-bulk-operations/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -41,11 +41,11 @@ Bulk operations for assets: publish/unpublish/cross-publish (CMS) and delete/mov

```
USAGE
$ csdx cm:stacks:bulk-assets [-a <value>] [-k <value>] [--operation publish|unpublish|delete|move] [--environments
<value>...] [--locales <value>...] [--source-env <value>] [--source-alias <value>] [--publish-mode bulk|single]
[--branch <value>] [-c <value>] [-y] [--retry-failed <value>] [--revert <value>] [--bulk-operation-file <value>]
[--folder-uid <value>] [-d <value>] [--dry-run] [--space-uid <value>] [--org-uid <value>] [--workspace <value>]
[--asset-uids-file <value>] [--locale <value>] [--target-folder-uid <value>]
$ csdx cm:stacks:bulk-assets [-a <value>] [-k <value>] [--operation publish|unpublish|delete|move] [--source-alias
<value>] [--publish-mode bulk|single] [--branch <value>] [-c <value>] [-y] [--retry-failed <value>] [--revert
<value>] [--bulk-operation-file <value>] [--backup-dir <value> | --source-env <value> | --folder-uid <value> |
--environments <value>... | --locales <value>...] [--dry-run] [--space-uid <value>] [--org-uid <value>] [--workspace
<value>] [--asset-uids-file <value>] [--locale <value>] [--target-folder-uid <value>]

FLAGS
-a, --alias=<value> Uses the name of a saved Management Token to authenticate the command. The command
Expand All @@ -54,13 +54,16 @@ FLAGS
-c, --config=<value> (optional) Specifies the path to a JSON configuration file that defines the options
for the command. Use this file instead of passing multiple CLI flags for a single
run.
-d, --data-dir=<value> Path to exported content folder containing asset publish details.
-k, --stack-api-key=<value> API key of the source stack. You must use either the --stack-api-key flag or the
--alias flag.
-y, --yes Skips interactive confirmation prompts and runs the command immediately using the
provided options. Useful for automation and scripts.
--asset-uids-file=<value> Path to UTF-8 JSON file: exactly `{ "uids": ["uid1", "uid2"] }` (non-empty string
array, no trimming; large lists: see docs for NODE_OPTIONS)
--backup-dir=<value> Path to the import backup directory. Each imported asset is published only to the
environments and locales it was published to in the source stack (read from the
backup publish details and asset UID mapping), with asset-scan gating applied.
Intended for the post-import publish flow.
--branch=<value> [default: main] The name of the branch where you want to perform the bulk publish
operation. If you don't mention the branch name, then by default the content from
main branch will be published.
Expand Down Expand Up @@ -114,7 +117,7 @@ EXAMPLES

$ csdx cm:stacks:bulk-assets --revert ./bulk-operation -a myAlias

$ csdx cm:stacks:bulk-assets --data-dir ./content --operation publish -k blt123
$ csdx cm:stacks:bulk-assets --backup-dir ./content --operation publish -k blt123

$ csdx cm:stacks:bulk-assets --operation delete --space-uid am123 --org-uid bltOrg --locale en-us --asset-uids-file ./assets.json

Expand Down
12 changes: 7 additions & 5 deletions packages/contentstack-bulk-operations/src/base-bulk-command.ts
Original file line number Diff line number Diff line change
Expand Up @@ -27,9 +27,8 @@ import {
setupStackConfig,
setupBatchQueueListeners,
confirmOperation as confirmOperationUtil,
getUniqueEnvironments,
getUniqueLocales,
batchItems,
hasPublishTargets,
handleCrossPublishOperation,
logOperationInfo,
validateBatch,
Expand Down Expand Up @@ -519,9 +518,12 @@ export abstract class BaseBulkCommand extends Command {
* Execute operation in BULK mode - processes items in batches
*/
private async executeBulkMode(items: any[], startTime: number): Promise<BulkOperationResult> {
const environments = getUniqueEnvironments(items);
const locales = getUniqueLocales(items);
const batches = batchItems(items, environments, locales);
const untargeted = items.filter((item) => !hasPublishTargets(item)).length;
if (untargeted > 0) {
this.logger.warn(`Skipping ${untargeted} item(s) with no publish target (missing publish_details).`);
}

const batches = batchItems(items);

batches.forEach((batch) => validateBatch(batch));
this.logger.debug(`Created ${batches.length} batches for processing`, this.loggerContext);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,14 +3,15 @@ import * as path from 'path';

import { flags, handleAndLogError, log, FlagInput } from '@contentstack/cli-utilities';

import { AssetPublishData, BulkOperationResult, ResourceType, OperationType, CsAssetsFlags } from '../../../interfaces';
import { BulkOperationResult, ResourceType, OperationType, CsAssetsFlags } from '../../../interfaces';
import { BaseBulkCommand } from '../../../base-bulk-command';
import {
$t,
messages,
fetchAssets,
scanDataDirStats,
BATCH_CONSTANTS,
scanBackupDirStats,
assetPublishTargets,
TargetBatcher,
categorizeByScanStatus,
fillMissingCsAssetsFlags,
promptForOperation,
Expand All @@ -20,7 +21,7 @@ import {
OperationFlagMatrixError,
RETRY_REVERT_CONTEXT,
} from '../../../utils';
import type { DataDirScanStats } from '../../../utils';
import type { BackupDirScanStats } from '../../../utils';
import { AssetService } from '../../../services';

type RegionWithOptionalCsAssetsUrl = { csAssetsUrl?: string };
Expand All @@ -34,7 +35,7 @@ const ALL_OPERATION_CHOICES = [

/**
* Bulk operations command for assets
* Supports publish, unpublish, cross publish, and data-dir publish operations (CMS),
* Supports publish, unpublish, cross publish, and backup-dir publish operations (CMS),
* plus delete and move operations (CS Assets).
*
* The two families use fully separate execution paths:
Expand Down Expand Up @@ -68,8 +69,8 @@ export default class BulkAssets extends BaseBulkCommand {
// Revert (unpublish) previously published assets using success log
'<%= config.bin %> <%= command.id %> --revert ./bulk-operation -a myAlias',

// Publish assets from exported content folder (e.g. after asset scanning clears)
'<%= config.bin %> <%= command.id %> --data-dir ./content --operation publish -k blt123',
// Publish imported assets to their original environments (e.g. after asset scanning clears)
'<%= config.bin %> <%= command.id %> --backup-dir ./content --operation publish -k blt123',

// CS Assets bulk delete (asset UIDs from a JSON file `{ "uids": [...] }`)
'<%= config.bin %> <%= command.id %> --operation delete --space-uid am123 --org-uid bltOrg --locale en-us --asset-uids-file ./assets.json',
Expand All @@ -88,9 +89,10 @@ export default class BulkAssets extends BaseBulkCommand {
'folder-uid': flags.string({
description: messages.FOLDER_UID,
}),
'data-dir': flags.string({
char: 'd',
description: messages.DATA_DIR_FLAG_DESC,
'backup-dir': flags.string({
description: messages.BACKUP_DIR_FLAG_DESC,
// Environments and locales are always derived per-asset from the backup.
exclusive: ['source-env', 'folder-uid', 'environments', 'locales'],
}),
'dry-run': flags.boolean({
description: messages.DRY_RUN_FLAG_DESC,
Expand Down Expand Up @@ -196,8 +198,8 @@ export default class BulkAssets extends BaseBulkCommand {
return;
}

if (this.bulkOperationConfig.dataDir) {
await this.runDataDirFlow();
if (this.bulkOperationConfig.backupDir) {
await this.runBackupDirFlow();
return;
}

Expand Down Expand Up @@ -249,22 +251,19 @@ export default class BulkAssets extends BaseBulkCommand {
}
}

private async runDataDirFlow(): Promise<void> {
const { dataDir, dryRun } = this.bulkOperationConfig;

// Capture original CLI locales/envs before pass 1 overwrites them on the config.
const cliLocales = [...(this.bulkOperationConfig.locales || [])];
const cliEnvs = [...(this.bulkOperationConfig.environments || [])];
private async runBackupDirFlow(): Promise<void> {
const { backupDir, dryRun } = this.bulkOperationConfig;

// Pass 1 — count-only scan: no AssetPublishData objects built, one chunk in memory at a time.
let stats: DataDirScanStats;
let stats: BackupDirScanStats;
try {
stats = await scanDataDirStats(dataDir!, cliEnvs, cliLocales, this.logger);
stats = await scanBackupDirStats(backupDir!, this.logger);
} catch (err: any) {
this.logger.error($t(messages.DATA_DIR_READ_ERROR, { path: dataDir!, error: err.message || String(err) }));
this.logger.error($t(messages.BACKUP_DIR_READ_ERROR, { path: backupDir!, error: err.message || String(err) }));
Comment thread
naman-contentstack marked this conversation as resolved.
return;
}

// Unions for the dashboard and confirmation prompt only — pass 2 derives each asset's own targets.
this.bulkOperationConfig.environments = stats.environments;
this.bulkOperationConfig.locales = stats.locales;

Expand Down Expand Up @@ -305,43 +304,31 @@ export default class BulkAssets extends BaseBulkCommand {
}

if (dryRun) {
log.info($t(messages.DATA_DIR_DRY_RUN));
log.info($t(messages.BACKUP_DIR_DRY_RUN));
return;
}

// Pass 2 — stream and publish: one chunk at a time, batches of ≤50 items enqueued directly.
// stats.assetUidMapper and stats.assetsIndex are reused from pass 1 — no second disk read.
const result = await this.streamAndPublish(
dataDir!,
cliLocales,
stats.totalItems,
stats.assetUidMapper,
stats.assetsIndex,
scanStatusMap
);
const result = await this.streamAndPublish(backupDir!, stats, scanStatusMap);
this.printOperationSummary(result);
}

/**
* Pass 2 of the data-dir flow.
* Reads chunk files one at a time, fills a working batch of ≤50 AssetPublishData items,
* and enqueues each batch directly into the queue manager without ever holding the full
* asset list in memory. Peak memory: one chunk file + one batch of ≤50 items.
* Pass 2 of the backup-dir flow.
* Reads chunk files one at a time, feeding each asset's own publish targets into a
* TargetBatcher that enqueues batches directly, without ever holding the full asset list in
* memory. Peak memory: one chunk file + one open bucket per distinct (locale, environment set).
*
* assetUidMapper and assetsIndex are passed in from pass 1 to avoid re-reading those files.
* scanStatusMap filters out non-clean assets before enqueueing.
*/
private async streamAndPublish(
dataDir: string,
cliLocales: string[],
totalItemCount: number,
assetUidMapper: Record<string, string>,
assetsIndex: Record<string, string>,
backupDir: string,
stats: BackupDirScanStats,
scanStatusMap: Map<string, string | undefined>
): Promise<BulkOperationResult> {
// Snapshot both arrays so in-flight mutations to bulkOperationConfig can't corrupt payloads.
const environments = [...this.bulkOperationConfig.environments!];
const locales = [...this.bulkOperationConfig.locales!];
const { assetUidMapper, assetsIndex, environmentsMap, totalBatches } = stats;
const operation = this.bulkOperationConfig.operation as OperationType;
const startTime = Date.now();

Expand All @@ -352,35 +339,26 @@ export default class BulkAssets extends BaseBulkCommand {
);
}

const useOverrideLocales = cliLocales.length > 0;
const BATCH_SIZE = BATCH_CONSTANTS.maxItems;
// totalItemCount comes from pass 1 using identical counting logic — used as upper bound for totalBatches.
// Scan status filtering may reduce the actual count; the invariant check below will log any mismatch.
const totalBatches = Math.ceil(totalItemCount / BATCH_SIZE);

let workingBatch: AssetPublishData[] = [];
let batchNumber = 0;
let totalSubmitted = 0;

this.batchResults.clear();

const flushBatch = (): void => {
if (workingBatch.length === 0) return;
batchNumber++;
// Batches are keyed on their publish target, so each asset reaches only its own
// environments and locales.
const batcher = new TargetBatcher((batch) => {
this.queueManager.enqueue(ResourceType.ASSET, operation, {
items: [...workingBatch],
environments,
locales,
batchNumber,
items: batch.items,
environments: batch.environments,
locales: batch.locales,
batchNumber: batch.batchNumber,
totalBatches,
operation,
});
totalSubmitted += workingBatch.length;
workingBatch = [];
};
totalSubmitted += batch.items.length;
});

for (const chunkFilename of Object.values(assetsIndex)) {
const chunkPath = path.join(dataDir, 'assets', chunkFilename);
const chunkPath = path.join(backupDir, 'assets', chunkFilename);
const chunkData: Record<string, any> = JSON.parse(fs.readFileSync(chunkPath, 'utf-8'));

for (const asset of Object.values(chunkData)) {
Expand All @@ -399,27 +377,32 @@ export default class BulkAssets extends BaseBulkCommand {
continue;
}

const assetLocales: string[] = useOverrideLocales
? cliLocales
: [...new Set<string>(asset.publish_details.map((pd: any) => pd.locale as string))];
const targets = assetPublishTargets(asset, environmentsMap);

for (const locale of assetLocales) {
workingBatch.push({ type: 'asset', uid: targetUid, locale, version: asset._version });
if (workingBatch.length >= BATCH_SIZE) {
flushBatch();
}
for (const [locale, environments] of targets) {
batcher.add({
type: 'asset',
uid: targetUid,
locale,
version: asset._version,
publish_details: environments.map((environment) => ({ environment, locale })),
});
}
}
// chunkData falls out of scope here — GC can reclaim it before the next chunk is read.
}

flushBatch();
batcher.end();

if (batcher.skippedCount > 0) {
this.logger.warn(`Skipped ${batcher.skippedCount} asset item(s) with no resolvable publish target.`);
}

// Invariant: pass 1 and pass 2 use identical counting logic (excluding scan status filtering).
// If batchNumber < totalBatches, scan status filtering reduced the published count — expected.
if (batchNumber !== totalBatches) {
// Invariant: pass 1 and pass 2 use identical target logic (excluding scan status filtering).
// If fewer batches were emitted, scan status filtering reduced the published count — expected.
if (batcher.emittedCount !== totalBatches) {
this.logger.debug(
`Batch count: predicted ${totalBatches}, actual ${batchNumber}. Difference is expected when assets are skipped due to scan status.`
`Batch count: predicted ${totalBatches}, actual ${batcher.emittedCount}. Difference is expected when assets are skipped due to scan status.`
);
}

Expand All @@ -443,21 +426,21 @@ export default class BulkAssets extends BaseBulkCommand {
const SEP = '─'.repeat(42);

log.info('');
log.info(` ${messages.DATA_DIR_ASSET_SCANNING_HEADER}`);
log.info(` ${messages.BACKUP_DIR_ASSET_SCANNING_HEADER}`);
log.info(' ' + SEP);
log.info(` ${messages.DATA_DIR_TOTAL.padEnd(38)} ${total}`);
log.info(` ${messages.BACKUP_DIR_TOTAL.padEnd(38)} ${total}`);
if (localSkipped !== undefined) {
log.warn(` ${messages.DATA_DIR_NO_PUBLISH_DETAILS.padEnd(38)} ${localSkipped}`);
log.warn(` ${messages.BACKUP_DIR_NO_PUBLISH_DETAILS.padEnd(38)} ${localSkipped}`);
}
if (unmapped !== undefined) {
log.warn(` ${messages.DATA_DIR_UNMAPPED.padEnd(38)} ${unmapped}`);
log.warn(` ${messages.BACKUP_DIR_UNMAPPED.padEnd(38)} ${unmapped}`);
}
log.info(' ' + SEP);
log.info(` ${messages.SCAN_STATUS_CLEAN.padEnd(38)} ${clean}`);
if (pending > 0) log.warn(` ${messages.SCAN_STATUS_PENDING.padEnd(38)} ${pending}`);
if (quarantined > 0) log.warn(` ${messages.SCAN_STATUS_QUARANTINED.padEnd(38)} ${quarantined}`);
log.info(' ' + SEP);
log.info(` ${messages.DATA_DIR_WILL_PUBLISH.padEnd(38)} ${clean}`);
log.info(` ${messages.BACKUP_DIR_WILL_PUBLISH.padEnd(38)} ${clean}`);
log.info('');
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,7 @@ export interface BulkOperationConfig {

// Asset-specific options
folderUid?: string;
dataDir?: string;
backupDir?: string;
dryRun?: boolean;

// Cross-publish
Expand Down Expand Up @@ -199,7 +199,7 @@ export interface CommandFlags {

// Asset-specific flags
'folder-uid'?: string;
'data-dir'?: string;
'backup-dir'?: string;
'dry-run'?: boolean;

/** CS Assets bulk delete/move */
Expand Down
21 changes: 11 additions & 10 deletions packages/contentstack-bulk-operations/src/messages/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -213,17 +213,18 @@ const bulkAssetsMsg = {
SYNCED_ASSETS: 'Synced {count} assets from {sourceEnv}',
ASSETS_READY_FOR_CROSS_PUBLISH: '{count} assets ready for cross-publish',

// Data-dir / scanning dashboard
DATA_DIR_ASSET_SCANNING_HEADER: 'Asset Scan Status',
DATA_DIR_TOTAL: 'Total assets found',
DATA_DIR_VALID: 'Clean (will publish)',
DATA_DIR_NO_PUBLISH_DETAILS: 'No publish details (skipped)',
DATA_DIR_UNMAPPED: 'Not imported / UID unmapped (skipped)',
DATA_DIR_WILL_PUBLISH: 'Will publish',
DATA_DIR_DRY_RUN: 'Dry run — no publish API calls will be made.',
DATA_DIR_FLAG_DESC: 'Path to exported content folder containing asset publish details.',
// Backup-dir / scanning dashboard
BACKUP_DIR_ASSET_SCANNING_HEADER: 'Asset Scan Status',
BACKUP_DIR_TOTAL: 'Total assets found',
BACKUP_DIR_VALID: 'Clean (will publish)',
BACKUP_DIR_NO_PUBLISH_DETAILS: 'No publish details (skipped)',
BACKUP_DIR_UNMAPPED: 'Not imported / UID unmapped (skipped)',
BACKUP_DIR_WILL_PUBLISH: 'Will publish',
BACKUP_DIR_DRY_RUN: 'Dry run — no publish API calls will be made.',
BACKUP_DIR_FLAG_DESC:
'Path to the import backup directory. Each imported asset is published only to the environments and locales it was published to in the source stack (read from the backup publish details and asset UID mapping), with asset-scan gating applied. Intended for the post-import publish flow.',
DRY_RUN_FLAG_DESC: 'Preview the publish plan without making any API calls.',
DATA_DIR_READ_ERROR: 'Failed to read data directory at {path}: {error}',
BACKUP_DIR_READ_ERROR: 'Failed to read backup directory at {path}: {error}',
SCAN_STATUS_CLEAN: 'Clean (will publish)',
SCAN_STATUS_PENDING: 'Still scanning (skipped)',
SCAN_STATUS_QUARANTINED: 'Quarantined (skipped)',
Expand Down
Loading
Loading