feat: new deploy diff

.changeset/tired-donkeys-leave.md

@@ -0,0 +1,6 @@
+---
+'@openfn/project': minor
+'@openfn/cli': minor
+---
+
+Adds new form of json diffing for cli

integration-tests/cli/test/deploy.v2.test.ts

@@ -205,8 +205,14 @@ test.serial('pull, change and re-deploy twice', async (t) => {
   );
   t.falsy(stderr);
   const logs = extractLogs(stdout);
-  assertLog(t, logs, /Updated project/);
-  assertLog(t, logs, /Workflows modified/);
+  assertLog(
+    t,
+    logs,
+    /This will make the following changes to the remote project:/
+  );
+  assertLog(t, logs, /My Workflow: changed/);
+  assertLog(t, logs, /My Job:/gm);
+  assertLog(t, logs, /- body: <changed>/gm);
 
   proj = server.state.projects[projectId];
   t.regex(proj.workflows['my-workflow-1'].jobs['my-job'].body, /v\: 2/);
@@ -265,7 +271,9 @@ test.serial('deploy then pull, change one workflow, deploy', async (t) => {
 
   // another-workflow should appear in the modified list
   const anotherLog = logs.find(
-    (log) => log.level === 'always' && /another-workflow/.test(`${log.message}`)
+    (log) =>
+      log.level === 'always' &&
+      /Another Workflow: changed/.test(`${log.message}`)
   );
   t.truthy(anotherLog);
 

packages/cli/package.json

@@ -35,6 +35,7 @@
   "devDependencies": {
     "@openfn/language-collections": "^0.8.3",
     "@openfn/language-common": "3.2.3",
+    "@types/json-diff": "^1.0.3",
     "@types/lodash-es": "~4.17.12",
     "@types/mock-fs": "^4.13.4",
     "@types/node": "^18.19.130",
@@ -62,6 +63,7 @@
     "dotenv": "^17.3.1",
     "dotenv-expand": "^12.0.3",
     "figures": "^5.0.0",
+    "json-diff": "^1.0.6",
     "rimraf": "^6.1.3",
     "treeify": "^1.1.0",
     "undici": "6.24.1",

packages/cli/src/projects/deploy.ts

@@ -1,6 +1,5 @@
 import yargs from 'yargs';
 import Project, { versionsEqual, Workspace } from '@openfn/project';
-import c from 'chalk';
 import { writeFile } from 'node:fs/promises';
 import path from 'node:path';
 
@@ -17,6 +16,7 @@ import {
   AuthOptions,
 } from './util';
 import { build, ensure } from '../util/command-builders';
+import { printRichDiff } from './diff';
 
 import type { Provisioner } from '@openfn/lexicon/lightning';
 import type { Logger } from '../util/logger';
@@ -41,6 +41,7 @@ export type DeployOptions = Pick<
   new?: boolean;
   name?: string;
   alias?: string;
+  jsonDiff?: boolean;
 };
 
 const options = [
@@ -51,6 +52,7 @@ const options = [
   o2.new,
   o2.name,
   o2.alias,
+  o2.jsonDiff,
 
   // general options
   o.apiKey,
@@ -121,6 +123,12 @@ export const hasRemoteDiverged = (
   return diverged;
 };
 
+export type SyncResult = {
+  merged: Project;
+  remoteProject: Project;
+  locallyChangedWorkflows: string[];
+};
+
 // This function is responsible for syncing changes in the user's local project
 // with the remote app version
 // It returns a merged state object
@@ -131,7 +139,7 @@ const syncProjects = async (
   localProject: Project,
   trackedProject: Project, // the project we want to update
   logger: Logger
-): Promise<Project | null> => {
+): Promise<SyncResult | null> => {
   // First step, fetch the latest version and write
   // this may throw!
   let remoteProject: Project;
@@ -169,15 +177,10 @@ const syncProjects = async (
   );
 
   // TODO: what if remote diff and the version checked disagree for some reason?
-  let diffs = [];
-  if (locallyChangedWorkflows.length) {
-    diffs = reportDiff(
-      localProject,
-      remoteProject,
-      locallyChangedWorkflows,
-      logger
-    );
-  }
+  const diffs = locallyChangedWorkflows.length
+    ? remoteProject.diff(localProject, locallyChangedWorkflows)
+    : [];
+
   if (!diffs.length) {
     logger.success('Nothing to deploy');
     return null;
@@ -238,7 +241,7 @@ const syncProjects = async (
     onlyUpdated: true,
   });
 
-  return merged;
+  return { merged, remoteProject, locallyChangedWorkflows };
 };
 
 export async function handler(options: DeployOptions, logger: Logger) {
@@ -306,21 +309,25 @@ export async function handler(options: DeployOptions, logger: Logger) {
     `Loaded checked-out project ${printProjectName(localProject)}`
   );
 
-  let merged;
+  let merged: Project;
+  let remoteProject: Project | undefined;
+  let locallyChangedWorkflows: string[] = [];
+
   if (options.new) {
     merged = localProject;
   } else {
-    merged = await syncProjects(
+    const syncResult = await syncProjects(
       options,
       config,
       ws,
       localProject,
       tracker,
       logger
     );
-    if (!merged) {
+    if (!syncResult) {
       return;
     }
+    ({ merged, remoteProject, locallyChangedWorkflows } = syncResult);
   }
 
   const state = merged.serialize('state', {
@@ -335,7 +342,21 @@ export async function handler(options: DeployOptions, logger: Logger) {
   logger.debug(JSON.stringify(state, null, 2));
   logger.debug();
 
-  // TODO: I want to report diff HERE, after the merged state and stuff has been built
+  if (remoteProject) {
+    if (options.jsonDiff) {
+      const remoteState = remoteProject.serialize('state', {
+        format: 'json',
+      }) as object;
+      await printJsonDiff(remoteState, state as object, logger);
+    } else {
+      printRichDiff(
+        localProject,
+        remoteProject,
+        locallyChangedWorkflows,
+        logger
+      );
+    }
+  }
 
   if (options.dryRun) {
     logger.always('dryRun option set: skipping upload step');
@@ -404,48 +425,16 @@ export async function handler(options: DeployOptions, logger: Logger) {
   }
 }
 
-export const reportDiff = (
-  local: Project,
-  remote: Project,
-  locallyChangedWorkflows: string[],
+export const printJsonDiff = async (
+  remoteState: object,
+  mergedState: object,
   logger: Logger
 ) => {
-  const diffs = remote.diff(local, locallyChangedWorkflows);
-  if (diffs.length === 0) {
-    logger.info('No workflow changes detected');
-    return diffs;
-  }
-
-  const added = diffs.filter((d) => d.type === 'added');
-  const changed = diffs.filter((d) => d.type === 'changed');
-  const removed = diffs.filter((d) => d.type === 'removed');
-
-  if (added.length > 0) {
-    logger.break();
-    logger.always(c.green('Workflows added:'));
-    for (const diff of added) {
-      logger.always(c.green(`  - ${diff.id}`));
-    }
-    logger.break();
-  }
-
-  if (changed.length > 0) {
+  const { default: jsondiff } = await import('json-diff');
+  const diff = jsondiff.diffString(remoteState, mergedState);
+  if (diff) {
     logger.break();
-    logger.always(c.yellow('Workflows modified:'));
-    for (const diff of changed) {
-      logger.always(c.yellow(`  - ${diff.id}`));
-    }
+    logger.always(diff);
     logger.break();
   }
-
-  if (removed.length > 0) {
-    logger.break();
-    logger.always(c.red('Workflows removed:'));
-    for (const diff of removed) {
-      logger.always(c.red(`  - ${diff.id}`));
-    }
-    logger.break();
-  }
-
-  return diffs;
 };

packages/cli/src/projects/diff.ts

@@ -0,0 +1,110 @@
+import c from 'chalk';
+import Project, { generateStepDiff, generateEdgeDiff } from '@openfn/project';
+import type { StepChange, EdgeChange } from '@openfn/project';
+import type { Logger } from '../util/logger';
+
+export { generateStepDiff, generateEdgeDiff };
+export type { StepChange, EdgeChange };
+
+const printEdgeDiff = (edges: EdgeChange[], logger: Logger) => {
+  for (const edge of edges) {
+    if (edge.type === 'added') {
+      logger.always(c.green(`    ${edge.id}: added`));
+    } else if (edge.type === 'removed') {
+      logger.always(c.red(`    ${edge.id}: removed`));
+    } else if (edge.type === 'changed' && edge.changes) {
+      logger.always(c.yellow(`    ${edge.id}:`));
+      const { condition, label, enabled } = edge.changes;
+      if (condition)
+        logger.always(
+          c.yellow(
+            `      - condition: ${condition.from ?? 'none'} -> ${
+              condition.to ?? 'none'
+            }`
+          )
+        );
+      if (label)
+        logger.always(
+          c.yellow(
+            `      - label: "${label.from ?? ''}" -> "${label.to ?? ''}"`
+          )
+        );
+      if (enabled)
+        logger.always(
+          c.yellow(`      - enabled: ${enabled.from} -> ${enabled.to}`)
+        );
+    }
+  }
+};
+
+const printStepDiff = (steps: StepChange[], logger: Logger) => {
+  for (const step of steps) {
+    if (step.type === 'added') {
+      logger.always(c.green(`    ${step.name}: added`));
+    } else if (step.type === 'removed') {
+      logger.always(c.red(`    ${step.name}: removed`));
+    } else if (step.type === 'changed' && step.changes) {
+      logger.always(c.yellow(`    ${step.name}:`));
+      const { name, adaptor, body } = step.changes;
+      if (name)
+        logger.always(c.yellow(`      - name: "${name.from}" -> "${name.to}"`));
+      if (adaptor)
+        logger.always(
+          c.yellow(`      - adaptor: ${adaptor.from} -> ${adaptor.to}`)
+        );
+      if (body) logger.always(c.yellow(`      - body: ${body}`));
+    }
+  }
+};
+
+export const printRichDiff = (
+  local: Project,
+  remote: Project,
+  locallyChangedWorkflows: string[],
+  logger: Logger
+) => {
+  const diffs = remote.diff(local, locallyChangedWorkflows);
+  if (diffs.length === 0) {
+    logger.info('No workflow changes detected');
+    return diffs;
+  }
+
+  const removed = diffs.filter((d) => d.type === 'removed');
+  const changed = diffs.filter((d) => d.type === 'changed');
+  const added = diffs.filter((d) => d.type === 'added');
+
+  logger.always('This will make the following changes to the remote project:');
+
+  if (removed.length > 0) {
+    logger.break();
+    for (const diff of removed) {
+      const wf = remote.getWorkflow(diff.id);
+      const label = wf?.name || diff.id;
+      logger.always(c.red(`${label}: deleted`));
+    }
+  }
+
+  if (changed.length > 0) {
+    logger.break();
+    for (const diff of changed) {
+      const localWf = local.getWorkflow(diff.id);
+      const remoteWf = remote.getWorkflow(diff.id);
+      const label = localWf?.name || diff.id;
+      logger.always(c.yellow(`${label}: changed`));
+      printStepDiff(generateStepDiff(localWf, remoteWf), logger);
+      printEdgeDiff(generateEdgeDiff(localWf, remoteWf), logger);
+    }
+  }
+
+  if (added.length > 0) {
+    logger.break();
+    for (const diff of added) {
+      const wf = local.getWorkflow(diff.id);
+      const label = wf?.name || diff.id;
+      logger.always(c.green(`${label}: added`));
+    }
+  }
+
+  logger.break();
+  return diffs;
+};

packages/cli/src/projects/options.ts

@@ -119,4 +119,13 @@ export const name: CLIOption = {
   },
 };
 
+export const jsonDiff: CLIOption = {
+  name: 'json-diff',
+  yargs: {
+    boolean: true,
+    description:
+      'Show a full JSON diff of the project state instead of the default rich text summary',
+  },
+};
+
 export { newProject as new };