feat: add support for on-events syntax in nango.yaml (#3021)

We want to be able to support custom scripts for more than just
post-connection event. This commit is adding support for the following
syntax in nango.yaml:
```
my-integration:
    on-events:
        post-connection-creation:
            - script1
            - script2
        pre-connection-deletion:
            - script1
    syncs:
        ...
```
Events currently supported are `post-connection-creation` and
`pre-connection-deletion`
The change is backward compatible and still support the old
`post-connection-scripts` even though I don't think it is currently
being used.
I will add in next PR the logic to trigger the script when connection is
deleted

## Issue ticket number and link

https://linear.app/nango/issue/NAN-1962/connect-lifecycle-scripts

## Checklist before requesting a review (skip if just adding/editing
APIs & templates)
- [ ] I added tests, otherwise the reason is: 
- [ ] I added observability, otherwise the reason is:
- [ ] I added analytics, otherwise the reason is:

packages/cli/lib/cli.ts

@@ -34,7 +34,7 @@ export function generate({ fullPath, debug = false }: { fullPath: string; debug?
     const syncTemplateContents = fs.readFileSync(path.resolve(__dirname, './templates/sync.ejs'), 'utf8');
     const actionTemplateContents = fs.readFileSync(path.resolve(__dirname, './templates/action.ejs'), 'utf8');
     const githubExampleTemplateContents = fs.readFileSync(path.resolve(__dirname, './templates/github.sync.ejs'), 'utf8');
-    const postConnectionTemplateContents = fs.readFileSync(path.resolve(__dirname, './templates/post-connection.ejs'), 'utf8');
+    const onEventTemplateContents = fs.readFileSync(path.resolve(__dirname, './templates/on-event.ejs'), 'utf8');
 
     const parsed = loadYamlAndGenerate({ fullPath, debug });
     if (!parsed) {
@@ -44,12 +44,12 @@ export function generate({ fullPath, debug = false }: { fullPath: string; debug?
     const allSyncNames: Record<string, boolean> = {};
 
     for (const integration of parsed.integrations) {
-        const { syncs, actions, postConnectionScripts, providerConfigKey } = integration;
+        const { syncs, actions, onEventScripts, providerConfigKey } = integration;
 
-        if (postConnectionScripts) {
-            const type = 'post-connection-script';
-            for (const name of postConnectionScripts) {
-                const rendered = ejs.render(postConnectionTemplateContents, {
+        if (onEventScripts) {
+            const type = 'on-event';
+            for (const name of Object.values(onEventScripts).flat()) {
+                const rendered = ejs.render(onEventTemplateContents, {
                     interfaceFileName: TYPES_FILE_NAME.replace('.ts', '')
                 });
                 const stripped = rendered.replace(/^\s+/, '');

packages/cli/lib/nango.yaml.schema.v2.json

@@ -18,6 +18,30 @@
                             "_": "post-connection-scripts must be an array of strings."
                         }
                     },
+                    "on-events": {
+                        "type": "object",
+                        "additionalProperties": false,
+                        "properties": {
+                            "post-connection-creation": {
+                                "type": "array",
+                                "items": {
+                                    "type": "string"
+                                },
+                                "errorMessage": {
+                                    "_": "post-connection-creation must be an array of strings."
+                                }
+                            },
+                            "pre-connection-deletion": {
+                                "type": "array",
+                                "items": {
+                                    "type": "string"
+                                },
+                                "errorMessage": {
+                                    "_": "post-connection-creation must be an array of strings."
+                                }
+                            }
+                        }
+                    },
                     "syncs": {
                         "type": "object",
                         "patternProperties": {

packages/cli/lib/services/__snapshots__/config.service.unit.test.ts.snap

@@ -21,7 +21,10 @@ exports[`load > should parse a nango.yaml file that is version 1 as expected 1`]
           "version": "",
         },
       ],
-      "postConnectionScripts": [],
+      "onEventScripts": {
+        "post-connection-creation": [],
+        "pre-connection-deletion": [],
+      },
       "providerConfigKey": "demo-github-integration",
       "syncs": [
         {
@@ -216,7 +219,10 @@ exports[`load > should parse a nango.yaml file that is version 2 as expected 1`]
           "version": "",
         },
       ],
-      "postConnectionScripts": [],
+      "onEventScripts": {
+        "post-connection-creation": [],
+        "pre-connection-deletion": [],
+      },
       "providerConfigKey": "demo-github-integration",
       "syncs": [
         {

packages/cli/lib/services/compile.service.ts

@@ -304,12 +304,14 @@ export function listFilesToCompile({
             const syncPath = `${integration.providerConfigKey}/syncs`;
             const actionPath = `${integration.providerConfigKey}/actions`;
             const postConnectionPath = `${integration.providerConfigKey}/post-connection-scripts`;
+            const onEventsPath = `${integration.providerConfigKey}/on-events`;
 
             const syncFiles = globFiles(fullPath, syncPath, '*.ts');
             const actionFiles = globFiles(fullPath, actionPath, '*.ts');
             const postFiles = globFiles(fullPath, postConnectionPath, '*.ts');
+            const onEventsFiles = globFiles(fullPath, onEventsPath, '*.ts');
 
-            files = [...files, ...syncFiles, ...actionFiles, ...postFiles];
+            files = [...files, ...syncFiles, ...actionFiles, ...postFiles, ...onEventsFiles];
 
             if (debug) {
                 if (syncFiles.length > 0) {
@@ -319,7 +321,10 @@ export function listFilesToCompile({
                     printDebug(`Found nested action files in ${actionPath}`);
                 }
                 if (postFiles.length > 0) {
-                    printDebug(`Found nested post connection script files in ${postConnectionPath}`);
+                    printDebug(`Found nested post-connection-scripts files in ${postConnectionPath}`);
+                }
+                if (onEventsFiles.length > 0) {
+                    printDebug(`Found nested on-events files in ${onEventsPath}`);
                 }
             }
         });

packages/cli/lib/services/deploy.service.ts

@@ -12,7 +12,8 @@ import type {
     NangoConfigMetadata,
     PostDeploy,
     PostDeployInternal,
-    PostDeployConfirmation
+    PostDeployConfirmation,
+    OnEventType
 } from '@nangohq/types';
 import { compileSingleFile, compileAllFiles, resolveTsFileLocation, getFileToCompile } from './compile.service.js';
 
@@ -359,24 +360,36 @@ class DeployService {
         version?: string | undefined;
         optionalSyncName?: string | undefined;
         optionalActionName?: string | undefined;
-    }): { flowConfigs: IncomingFlowConfig[]; onEventScriptsByProvider: OnEventScriptsByProvider[]; jsonSchema: JSONSchema7 } | null {
+    }): { flowConfigs: IncomingFlowConfig[]; onEventScriptsByProvider: OnEventScriptsByProvider[] | undefined; jsonSchema: JSONSchema7 } | null {
         const postData: IncomingFlowConfig[] = [];
-        const onEventScriptsByProvider: OnEventScriptsByProvider[] = [];
+        const onEventScriptsByProvider: OnEventScriptsByProvider[] | undefined = optionalActionName || optionalSyncName ? undefined : []; // only load on-event scripts if we're not deploying a single sync or action
 
         for (const integration of parsed.integrations) {
-            const { providerConfigKey, postConnectionScripts } = integration;
+            const { providerConfigKey, onEventScripts, postConnectionScripts } = integration;
 
-            if (postConnectionScripts && postConnectionScripts.length > 0) {
+            if (onEventScriptsByProvider) {
                 const scripts: OnEventScriptsByProvider['scripts'] = [];
-                for (const postConnectionScript of postConnectionScripts) {
-                    const files = loadScriptFiles({ scriptName: postConnectionScript, providerConfigKey, fullPath, type: 'post-connection-scripts' });
-                    if (!files) {
-                        return null;
+                for (const event of Object.keys(onEventScripts) as OnEventType[]) {
+                    for (const scriptName of onEventScripts[event]) {
+                        const files = loadScriptFiles({ scriptName: scriptName, providerConfigKey, fullPath, type: 'on-events' });
+                        if (!files) {
+                            console.log(chalk.red(`No script files found for "${scriptName}"`));
+                            return null;
+                        }
+                        scripts.push({ name: scriptName, fileBody: files, event });
                     }
+                }
 
-                    scripts.push({ name: postConnectionScript, fileBody: files });
+                // for backward compatibility we also load post-connection-creation scripts
+                for (const scriptName of postConnectionScripts || []) {
+                    const files = loadScriptFiles({ scriptName: scriptName, providerConfigKey, fullPath, type: 'post-connection-scripts' });
+                    if (files) {
+                        scripts.push({ name: scriptName, fileBody: files, event: 'post-connection-creation' });
+                    }
+                }
+                if (scripts.length > 0) {
+                    onEventScriptsByProvider.push({ providerConfigKey, scripts });
                 }
-                onEventScriptsByProvider.push({ providerConfigKey, scripts });
             }
 
             if (!optionalActionName) {
@@ -475,7 +488,7 @@ class DeployService {
                 for (const script of scripts) {
                     const { name } = script;
 
-                    printDebug(`Post connection script found for ${providerConfigKey} with name ${name}`);
+                    printDebug(`on-events script found for ${providerConfigKey} with name ${name}`);
                 }
             }
         }
@@ -485,7 +498,7 @@ class DeployService {
             return null;
         }
 
-        return { flowConfigs: postData, onEventScriptsByProvider: onEventScriptsByProvider, jsonSchema };
+        return { flowConfigs: postData, onEventScriptsByProvider, jsonSchema };
     }
 }
 

packages/cli/lib/services/dryrun.service.ts

@@ -110,7 +110,7 @@ export class DryRunService {
         }
 
         let providerConfigKey: string | undefined;
-        let isPostConnectionScript = false;
+        let isOnEventScript = false;
 
         // Find the appropriate script to run
         let scriptInfo: ParsedNangoSync | ParsedNangoAction | undefined;
@@ -132,23 +132,23 @@ export class DryRunService {
                 providerConfigKey = integration.providerConfigKey;
             }
 
-            // If nothing that could still be a post connection script
+            // If nothing that could still be a on-event script
             if (!scriptInfo) {
-                for (const script of integration.postConnectionScripts) {
+                for (const script of Object.values(integration.onEventScripts).flat()) {
                     if (script !== syncName) {
                         continue;
                     }
-                    if (isPostConnectionScript) {
+                    if (isOnEventScript) {
                         console.log(chalk.red(`Multiple integrations contain a post connection script named "${syncName}". Please use "--integration-id"`));
                         return;
                     }
-                    isPostConnectionScript = true;
+                    isOnEventScript = true;
                     providerConfigKey = integration.providerConfigKey;
                 }
             }
         }
 
-        if ((!scriptInfo && !isPostConnectionScript) || !providerConfigKey) {
+        if ((!scriptInfo && !isOnEventScript) || !providerConfigKey) {
             console.log(
                 chalk.red(
                     `No script matched "${syncName}"${options.optionalProviderConfigKey ? ` for integration "${options.optionalProviderConfigKey}"` : ''}`
@@ -206,8 +206,8 @@ export class DryRunService {
         let type: ScriptFileType = 'syncs';
         if (scriptInfo?.type === 'action') {
             type = 'actions';
-        } else if (isPostConnectionScript) {
-            type = 'post-connection-scripts';
+        } else if (isOnEventScript) {
+            type = 'on-events';
         }
 
         const result = await compileAllFiles({ fullPath: process.cwd(), debug, scriptName: syncName, providerConfigKey, type });

packages/cli/lib/services/verification.service.ts

@@ -110,17 +110,18 @@ class VerificationService {
         const parser = parsing.value;
         const syncNames = parser.parsed!.integrations.map((provider) => provider.syncs.map((sync) => sync.name)).flat();
         const actionNames = parser.parsed!.integrations.map((provider) => provider.actions.map((action) => action.name)).flat();
-        const flows = [...syncNames, ...actionNames].filter((name) => name);
+        const onEventsScriptNames = parser.parsed!.integrations.map((provider) => Object.values(provider.onEventScripts).flat()).flat();
+        const flows = [...syncNames, ...actionNames, ...onEventsScriptNames].filter((name) => name);
 
         const tsFiles = listFilesToCompile({ fullPath, parsed: parser.parsed! });
 
         const tsFileNames = tsFiles.filter((file) => !file.inputPath.includes('models.ts')).map((file) => file.baseName);
 
-        const missingSyncsAndActions = flows.filter((syncOrActionName) => !tsFileNames.includes(syncOrActionName));
+        const missingFiles = flows.filter((scriptName) => !tsFileNames.includes(scriptName));
 
-        if (missingSyncsAndActions.length > 0) {
-            console.log(chalk.red(`The following syncs are missing a corresponding .ts file: ${missingSyncsAndActions.join(', ')}`));
-            throw new Error('Syncs missing .ts files');
+        if (missingFiles.length > 0) {
+            console.log(chalk.red(`The following scripts are missing a corresponding .ts file: ${missingFiles.join(', ')}`));
+            throw new Error('Script missing .ts files');
         }
 
         return true;

packages/cli/lib/templates/on-event.ejs

@@ -0,0 +1,5 @@
+import type { NangoAction } from '../../<%= interfaceFileName %>';
+
+export default async function onEvent(nango: NangoAction): Promise<void> {
+    // logic goes here
+}

packages/database/lib/migrations/2024111809211759_update_on_event_enum.cjs

@@ -0,0 +1,25 @@
+exports.config = { transaction: false };
+
+/**
+ * @param {import('knex').Knex} knex
+ */
+exports.up = async function (knex) {
+    await knex.schema.raw(`
+        ALTER TYPE script_trigger_event RENAME VALUE 'ON_CONNECTION_CREATED' TO 'POST_CONNECTION_CREATION';
+    `);
+    await knex.schema.raw(`
+        ALTER TYPE script_trigger_event RENAME VALUE 'ON_CONNECTION_DELETED' TO 'PRE_CONNECTION_DELETION';
+    `);
+};
+
+/**
+ * @param {import('knex').Knex} knex
+ */
+exports.down = async function (knex) {
+    await knex.schema.raw(`
+        ALTER TYPE script_trigger_event RENAME VALUE 'POST_CONNECTION_CREATION' TO 'ON_CONNECTION_CREATED';
+    `);
+    await knex.schema.raw(`
+        ALTER TYPE script_trigger_event RENAME VALUE 'PRE_CONNECTION_DELETION' TO 'ON_CONNECTION_DELETED';
+    `);
+};

packages/nango-yaml/lib/errors.ts

@@ -122,3 +122,13 @@ export class ParserErrorTypeSyntax extends ParserError {
         });
     }
 }
+
+export class ParserErrorBothPostConnectionScriptsAndOnEventsPresent extends ParserError {
+    constructor(options: { path: string[] }) {
+        super({
+            code: 'both_post_connection_scripts_and_on_events_present',
+            message: `Both post-connection-scripts and on-events are present. Only one of them can be used at a time.`,
+            path: options.path
+        });
+    }
+}

packages/nango-yaml/lib/helpers.ts

@@ -181,7 +181,7 @@ export function shouldQuote(name: string) {
 export function getProviderConfigurationFromPath({ filePath, parsed }: { filePath: string; parsed: NangoYamlParsed }): NangoYamlParsedIntegration | null {
     const pathSegments = filePath.split(path.sep);
     const scriptType = pathSegments.length > 1 ? pathSegments[pathSegments.length - 2] : null;
-    const isNested = scriptType === 'syncs' || scriptType === 'actions' || scriptType === 'post-connection-scripts';
+    const isNested = scriptType && ['syncs', 'actions', 'post-connection-scripts', 'on-events'].includes(scriptType);
 
     const baseName = path.basename(filePath, '.ts');
     let providerConfiguration: NangoYamlParsedIntegration | null = null;

packages/nango-yaml/lib/parser.v1.ts

@@ -82,7 +82,7 @@ export class NangoYamlParserV1 extends NangoYamlParser {
                 providerConfigKey: integrationName,
                 syncs,
                 actions,
-                postConnectionScripts: []
+                onEventScripts: { 'post-connection-creation': [], 'pre-connection-deletion': [] }
             };
 
             output.push(parsedIntegration);

packages/nango-yaml/lib/parser.v1.unit.test.ts

@@ -39,7 +39,7 @@ describe('parse', () => {
                             version: ''
                         }
                     ],
-                    postConnectionScripts: [],
+                    onEventScripts: { 'post-connection-creation': [], 'pre-connection-deletion': [] },
                     actions: [
                         {
                             description: '',
@@ -93,7 +93,7 @@ describe('parse', () => {
                             version: ''
                         }
                     ],
-                    postConnectionScripts: [],
+                    onEventScripts: { 'post-connection-creation': [], 'pre-connection-deletion': [] },
                     actions: []
                 }
             ],

packages/nango-yaml/lib/parser.v2.ts

@@ -11,7 +11,7 @@ import type {
     ScriptTypeLiteral
 } from '@nangohq/types';
 import { NangoYamlParser } from './parser.js';
-import { ParserErrorEndpointsMismatch, ParserErrorInvalidRuns } from './errors.js';
+import { ParserErrorEndpointsMismatch, ParserErrorInvalidRuns, ParserErrorBothPostConnectionScriptsAndOnEventsPresent } from './errors.js';
 import { getInterval, parseEndpoint } from './helpers.js';
 
 export class NangoYamlParserV2 extends NangoYamlParser {
@@ -37,15 +37,25 @@ export class NangoYamlParserV2 extends NangoYamlParser {
             const syncs = integration['syncs'];
             const actions = integration['actions'];
             const postConnectionScripts: string[] = integration['post-connection-scripts'] || [];
+            const onEventScripts: Record<string, string[]> = integration['on-events'] || {};
 
             const parsedSyncs = this.parseSyncs({ syncs, integrationName });
             const parseActions = this.parseActions({ actions, integrationName });
 
+            if (postConnectionScripts.length > 0 && Object.values(onEventScripts).length > 0) {
+                this.errors.push(new ParserErrorBothPostConnectionScriptsAndOnEventsPresent({ path: [integrationName, 'on-events'] }));
+            }
+            const parsedOnEventScripts = {
+                'post-connection-creation': onEventScripts['post-connection-creation'] || [],
+                'pre-connection-deletion': onEventScripts['pre-connection-deletion'] || []
+            };
+
             const parsedIntegration: NangoYamlParsedIntegration = {
                 providerConfigKey: integrationName,
                 syncs: parsedSyncs,
                 actions: parseActions,
-                postConnectionScripts
+                onEventScripts: parsedOnEventScripts,
+                ...(postConnectionScripts.length > 0 ? { postConnectionScripts } : {})
             };
 
             output.push(parsedIntegration);