From 6ead5938314519a532c530b5287cd14e9e73d1d1 Mon Sep 17 00:00:00 2001 From: Thomas Bonnin <233326+TBonnin@users.noreply.github.com> Date: Wed, 20 Nov 2024 16:20:05 -0500 Subject: [PATCH] feat: add support for on-events syntax in nango.yaml (#3021) We want to be able to support custom scripts for more than just post-connection event. This commit is adding support for the following syntax in nango.yaml: ``` my-integration: on-events: post-connection-creation: - script1 - script2 pre-connection-deletion: - script1 syncs: ... ``` Events currently supported are `post-connection-creation` and `pre-connection-deletion` The change is backward compatible and still support the old `post-connection-scripts` even though I don't think it is currently being used. I will add in next PR the logic to trigger the script when connection is deleted ## Issue ticket number and link https://linear.app/nango/issue/NAN-1962/connect-lifecycle-scripts ## Checklist before requesting a review (skip if just adding/editing APIs & templates) - [ ] I added tests, otherwise the reason is: - [ ] I added observability, otherwise the reason is: - [ ] I added analytics, otherwise the reason is: --- packages/cli/lib/cli.ts | 12 ++-- packages/cli/lib/nango.yaml.schema.v2.json | 24 +++++++ .../config.service.unit.test.ts.snap | 10 ++- packages/cli/lib/services/compile.service.ts | 9 ++- packages/cli/lib/services/deploy.service.ts | 39 +++++++---- packages/cli/lib/services/dryrun.service.ts | 16 ++--- .../cli/lib/services/verification.service.ts | 11 +-- packages/cli/lib/templates/on-event.ejs | 5 ++ .../cli/lib/templates/post-connection.ejs | 5 -- .../2024111809211759_update_on_event_enum.cjs | 25 +++++++ packages/nango-yaml/lib/errors.ts | 10 +++ packages/nango-yaml/lib/helpers.ts | 2 +- packages/nango-yaml/lib/parser.v1.ts | 2 +- .../nango-yaml/lib/parser.v1.unit.test.ts | 4 +- packages/nango-yaml/lib/parser.v2.ts | 14 +++- .../nango-yaml/lib/parser.v2.unit.test.ts | 54 ++++++++++++-- .../postConfirmation.integration.test.ts | 29 -------- .../deploy/postDeploy.integration.test.ts | 44 ++++-------- .../lib/controllers/sync/deploy/validation.ts | 45 ++++++++---- .../hooks/connection/on/connection-created.ts | 2 +- packages/shared/lib/models/NangoConfig.ts | 1 - .../services/sync/config/config.service.ts | 1 - .../services/sync/config/deploy.service.ts | 4 +- .../services/sync/on-event-scripts.service.ts | 70 +++++++++++-------- packages/types/lib/deploy/api.ts | 6 +- packages/types/lib/deploy/incomingFlow.ts | 3 +- packages/types/lib/nangoYaml/index.ts | 14 +++- .../types/lib/scripts/post-connection/db.ts | 1 + 28 files changed, 293 insertions(+), 169 deletions(-) create mode 100644 packages/cli/lib/templates/on-event.ejs delete mode 100644 packages/cli/lib/templates/post-connection.ejs create mode 100644 packages/database/lib/migrations/2024111809211759_update_on_event_enum.cjs diff --git a/packages/cli/lib/cli.ts b/packages/cli/lib/cli.ts index 89a8ab5b66..85e085408b 100644 --- a/packages/cli/lib/cli.ts +++ b/packages/cli/lib/cli.ts @@ -34,7 +34,7 @@ export function generate({ fullPath, debug = false }: { fullPath: string; debug? const syncTemplateContents = fs.readFileSync(path.resolve(__dirname, './templates/sync.ejs'), 'utf8'); const actionTemplateContents = fs.readFileSync(path.resolve(__dirname, './templates/action.ejs'), 'utf8'); const githubExampleTemplateContents = fs.readFileSync(path.resolve(__dirname, './templates/github.sync.ejs'), 'utf8'); - const postConnectionTemplateContents = fs.readFileSync(path.resolve(__dirname, './templates/post-connection.ejs'), 'utf8'); + const onEventTemplateContents = fs.readFileSync(path.resolve(__dirname, './templates/on-event.ejs'), 'utf8'); const parsed = loadYamlAndGenerate({ fullPath, debug }); if (!parsed) { @@ -44,12 +44,12 @@ export function generate({ fullPath, debug = false }: { fullPath: string; debug? const allSyncNames: Record = {}; for (const integration of parsed.integrations) { - const { syncs, actions, postConnectionScripts, providerConfigKey } = integration; + const { syncs, actions, onEventScripts, providerConfigKey } = integration; - if (postConnectionScripts) { - const type = 'post-connection-script'; - for (const name of postConnectionScripts) { - const rendered = ejs.render(postConnectionTemplateContents, { + if (onEventScripts) { + const type = 'on-event'; + for (const name of Object.values(onEventScripts).flat()) { + const rendered = ejs.render(onEventTemplateContents, { interfaceFileName: TYPES_FILE_NAME.replace('.ts', '') }); const stripped = rendered.replace(/^\s+/, ''); diff --git a/packages/cli/lib/nango.yaml.schema.v2.json b/packages/cli/lib/nango.yaml.schema.v2.json index 1375f72b7e..d4d00d2098 100644 --- a/packages/cli/lib/nango.yaml.schema.v2.json +++ b/packages/cli/lib/nango.yaml.schema.v2.json @@ -18,6 +18,30 @@ "_": "post-connection-scripts must be an array of strings." } }, + "on-events": { + "type": "object", + "additionalProperties": false, + "properties": { + "post-connection-creation": { + "type": "array", + "items": { + "type": "string" + }, + "errorMessage": { + "_": "post-connection-creation must be an array of strings." + } + }, + "pre-connection-deletion": { + "type": "array", + "items": { + "type": "string" + }, + "errorMessage": { + "_": "post-connection-creation must be an array of strings." + } + } + } + }, "syncs": { "type": "object", "patternProperties": { diff --git a/packages/cli/lib/services/__snapshots__/config.service.unit.test.ts.snap b/packages/cli/lib/services/__snapshots__/config.service.unit.test.ts.snap index 6c33d3cd9b..c0ffe1a1ff 100644 --- a/packages/cli/lib/services/__snapshots__/config.service.unit.test.ts.snap +++ b/packages/cli/lib/services/__snapshots__/config.service.unit.test.ts.snap @@ -21,7 +21,10 @@ exports[`load > should parse a nango.yaml file that is version 1 as expected 1`] "version": "", }, ], - "postConnectionScripts": [], + "onEventScripts": { + "post-connection-creation": [], + "pre-connection-deletion": [], + }, "providerConfigKey": "demo-github-integration", "syncs": [ { @@ -216,7 +219,10 @@ exports[`load > should parse a nango.yaml file that is version 2 as expected 1`] "version": "", }, ], - "postConnectionScripts": [], + "onEventScripts": { + "post-connection-creation": [], + "pre-connection-deletion": [], + }, "providerConfigKey": "demo-github-integration", "syncs": [ { diff --git a/packages/cli/lib/services/compile.service.ts b/packages/cli/lib/services/compile.service.ts index 822133a9c5..0e5b3b2097 100644 --- a/packages/cli/lib/services/compile.service.ts +++ b/packages/cli/lib/services/compile.service.ts @@ -304,12 +304,14 @@ export function listFilesToCompile({ const syncPath = `${integration.providerConfigKey}/syncs`; const actionPath = `${integration.providerConfigKey}/actions`; const postConnectionPath = `${integration.providerConfigKey}/post-connection-scripts`; + const onEventsPath = `${integration.providerConfigKey}/on-events`; const syncFiles = globFiles(fullPath, syncPath, '*.ts'); const actionFiles = globFiles(fullPath, actionPath, '*.ts'); const postFiles = globFiles(fullPath, postConnectionPath, '*.ts'); + const onEventsFiles = globFiles(fullPath, onEventsPath, '*.ts'); - files = [...files, ...syncFiles, ...actionFiles, ...postFiles]; + files = [...files, ...syncFiles, ...actionFiles, ...postFiles, ...onEventsFiles]; if (debug) { if (syncFiles.length > 0) { @@ -319,7 +321,10 @@ export function listFilesToCompile({ printDebug(`Found nested action files in ${actionPath}`); } if (postFiles.length > 0) { - printDebug(`Found nested post connection script files in ${postConnectionPath}`); + printDebug(`Found nested post-connection-scripts files in ${postConnectionPath}`); + } + if (onEventsFiles.length > 0) { + printDebug(`Found nested on-events files in ${onEventsPath}`); } } }); diff --git a/packages/cli/lib/services/deploy.service.ts b/packages/cli/lib/services/deploy.service.ts index 94f1f062a3..41361b8119 100644 --- a/packages/cli/lib/services/deploy.service.ts +++ b/packages/cli/lib/services/deploy.service.ts @@ -12,7 +12,8 @@ import type { NangoConfigMetadata, PostDeploy, PostDeployInternal, - PostDeployConfirmation + PostDeployConfirmation, + OnEventType } from '@nangohq/types'; import { compileSingleFile, compileAllFiles, resolveTsFileLocation, getFileToCompile } from './compile.service.js'; @@ -359,24 +360,36 @@ class DeployService { version?: string | undefined; optionalSyncName?: string | undefined; optionalActionName?: string | undefined; - }): { flowConfigs: IncomingFlowConfig[]; onEventScriptsByProvider: OnEventScriptsByProvider[]; jsonSchema: JSONSchema7 } | null { + }): { flowConfigs: IncomingFlowConfig[]; onEventScriptsByProvider: OnEventScriptsByProvider[] | undefined; jsonSchema: JSONSchema7 } | null { const postData: IncomingFlowConfig[] = []; - const onEventScriptsByProvider: OnEventScriptsByProvider[] = []; + const onEventScriptsByProvider: OnEventScriptsByProvider[] | undefined = optionalActionName || optionalSyncName ? undefined : []; // only load on-event scripts if we're not deploying a single sync or action for (const integration of parsed.integrations) { - const { providerConfigKey, postConnectionScripts } = integration; + const { providerConfigKey, onEventScripts, postConnectionScripts } = integration; - if (postConnectionScripts && postConnectionScripts.length > 0) { + if (onEventScriptsByProvider) { const scripts: OnEventScriptsByProvider['scripts'] = []; - for (const postConnectionScript of postConnectionScripts) { - const files = loadScriptFiles({ scriptName: postConnectionScript, providerConfigKey, fullPath, type: 'post-connection-scripts' }); - if (!files) { - return null; + for (const event of Object.keys(onEventScripts) as OnEventType[]) { + for (const scriptName of onEventScripts[event]) { + const files = loadScriptFiles({ scriptName: scriptName, providerConfigKey, fullPath, type: 'on-events' }); + if (!files) { + console.log(chalk.red(`No script files found for "${scriptName}"`)); + return null; + } + scripts.push({ name: scriptName, fileBody: files, event }); } + } - scripts.push({ name: postConnectionScript, fileBody: files }); + // for backward compatibility we also load post-connection-creation scripts + for (const scriptName of postConnectionScripts || []) { + const files = loadScriptFiles({ scriptName: scriptName, providerConfigKey, fullPath, type: 'post-connection-scripts' }); + if (files) { + scripts.push({ name: scriptName, fileBody: files, event: 'post-connection-creation' }); + } + } + if (scripts.length > 0) { + onEventScriptsByProvider.push({ providerConfigKey, scripts }); } - onEventScriptsByProvider.push({ providerConfigKey, scripts }); } if (!optionalActionName) { @@ -475,7 +488,7 @@ class DeployService { for (const script of scripts) { const { name } = script; - printDebug(`Post connection script found for ${providerConfigKey} with name ${name}`); + printDebug(`on-events script found for ${providerConfigKey} with name ${name}`); } } } @@ -485,7 +498,7 @@ class DeployService { return null; } - return { flowConfigs: postData, onEventScriptsByProvider: onEventScriptsByProvider, jsonSchema }; + return { flowConfigs: postData, onEventScriptsByProvider, jsonSchema }; } } diff --git a/packages/cli/lib/services/dryrun.service.ts b/packages/cli/lib/services/dryrun.service.ts index 163d2bad6a..8f62c22fd0 100644 --- a/packages/cli/lib/services/dryrun.service.ts +++ b/packages/cli/lib/services/dryrun.service.ts @@ -110,7 +110,7 @@ export class DryRunService { } let providerConfigKey: string | undefined; - let isPostConnectionScript = false; + let isOnEventScript = false; // Find the appropriate script to run let scriptInfo: ParsedNangoSync | ParsedNangoAction | undefined; @@ -132,23 +132,23 @@ export class DryRunService { providerConfigKey = integration.providerConfigKey; } - // If nothing that could still be a post connection script + // If nothing that could still be a on-event script if (!scriptInfo) { - for (const script of integration.postConnectionScripts) { + for (const script of Object.values(integration.onEventScripts).flat()) { if (script !== syncName) { continue; } - if (isPostConnectionScript) { + if (isOnEventScript) { console.log(chalk.red(`Multiple integrations contain a post connection script named "${syncName}". Please use "--integration-id"`)); return; } - isPostConnectionScript = true; + isOnEventScript = true; providerConfigKey = integration.providerConfigKey; } } } - if ((!scriptInfo && !isPostConnectionScript) || !providerConfigKey) { + if ((!scriptInfo && !isOnEventScript) || !providerConfigKey) { console.log( chalk.red( `No script matched "${syncName}"${options.optionalProviderConfigKey ? ` for integration "${options.optionalProviderConfigKey}"` : ''}` @@ -206,8 +206,8 @@ export class DryRunService { let type: ScriptFileType = 'syncs'; if (scriptInfo?.type === 'action') { type = 'actions'; - } else if (isPostConnectionScript) { - type = 'post-connection-scripts'; + } else if (isOnEventScript) { + type = 'on-events'; } const result = await compileAllFiles({ fullPath: process.cwd(), debug, scriptName: syncName, providerConfigKey, type }); diff --git a/packages/cli/lib/services/verification.service.ts b/packages/cli/lib/services/verification.service.ts index 371bb698b4..927dda5f80 100644 --- a/packages/cli/lib/services/verification.service.ts +++ b/packages/cli/lib/services/verification.service.ts @@ -110,17 +110,18 @@ class VerificationService { const parser = parsing.value; const syncNames = parser.parsed!.integrations.map((provider) => provider.syncs.map((sync) => sync.name)).flat(); const actionNames = parser.parsed!.integrations.map((provider) => provider.actions.map((action) => action.name)).flat(); - const flows = [...syncNames, ...actionNames].filter((name) => name); + const onEventsScriptNames = parser.parsed!.integrations.map((provider) => Object.values(provider.onEventScripts).flat()).flat(); + const flows = [...syncNames, ...actionNames, ...onEventsScriptNames].filter((name) => name); const tsFiles = listFilesToCompile({ fullPath, parsed: parser.parsed! }); const tsFileNames = tsFiles.filter((file) => !file.inputPath.includes('models.ts')).map((file) => file.baseName); - const missingSyncsAndActions = flows.filter((syncOrActionName) => !tsFileNames.includes(syncOrActionName)); + const missingFiles = flows.filter((scriptName) => !tsFileNames.includes(scriptName)); - if (missingSyncsAndActions.length > 0) { - console.log(chalk.red(`The following syncs are missing a corresponding .ts file: ${missingSyncsAndActions.join(', ')}`)); - throw new Error('Syncs missing .ts files'); + if (missingFiles.length > 0) { + console.log(chalk.red(`The following scripts are missing a corresponding .ts file: ${missingFiles.join(', ')}`)); + throw new Error('Script missing .ts files'); } return true; diff --git a/packages/cli/lib/templates/on-event.ejs b/packages/cli/lib/templates/on-event.ejs new file mode 100644 index 0000000000..54cc5a7392 --- /dev/null +++ b/packages/cli/lib/templates/on-event.ejs @@ -0,0 +1,5 @@ +import type { NangoAction } from '../../<%= interfaceFileName %>'; + +export default async function onEvent(nango: NangoAction): Promise { + // logic goes here +} diff --git a/packages/cli/lib/templates/post-connection.ejs b/packages/cli/lib/templates/post-connection.ejs deleted file mode 100644 index bee4967568..0000000000 --- a/packages/cli/lib/templates/post-connection.ejs +++ /dev/null @@ -1,5 +0,0 @@ -import type { NangoAction } from '../../<%= interfaceFileName %>'; - -export default async function postConnection(nango: NangoAction): Promise { - // post connection logic goes here -} diff --git a/packages/database/lib/migrations/2024111809211759_update_on_event_enum.cjs b/packages/database/lib/migrations/2024111809211759_update_on_event_enum.cjs new file mode 100644 index 0000000000..3997fafe86 --- /dev/null +++ b/packages/database/lib/migrations/2024111809211759_update_on_event_enum.cjs @@ -0,0 +1,25 @@ +exports.config = { transaction: false }; + +/** + * @param {import('knex').Knex} knex + */ +exports.up = async function (knex) { + await knex.schema.raw(` + ALTER TYPE script_trigger_event RENAME VALUE 'ON_CONNECTION_CREATED' TO 'POST_CONNECTION_CREATION'; + `); + await knex.schema.raw(` + ALTER TYPE script_trigger_event RENAME VALUE 'ON_CONNECTION_DELETED' TO 'PRE_CONNECTION_DELETION'; + `); +}; + +/** + * @param {import('knex').Knex} knex + */ +exports.down = async function (knex) { + await knex.schema.raw(` + ALTER TYPE script_trigger_event RENAME VALUE 'POST_CONNECTION_CREATION' TO 'ON_CONNECTION_CREATED'; + `); + await knex.schema.raw(` + ALTER TYPE script_trigger_event RENAME VALUE 'PRE_CONNECTION_DELETION' TO 'ON_CONNECTION_DELETED'; + `); +}; diff --git a/packages/nango-yaml/lib/errors.ts b/packages/nango-yaml/lib/errors.ts index dc7f8e7ec5..22bc4a910f 100644 --- a/packages/nango-yaml/lib/errors.ts +++ b/packages/nango-yaml/lib/errors.ts @@ -122,3 +122,13 @@ export class ParserErrorTypeSyntax extends ParserError { }); } } + +export class ParserErrorBothPostConnectionScriptsAndOnEventsPresent extends ParserError { + constructor(options: { path: string[] }) { + super({ + code: 'both_post_connection_scripts_and_on_events_present', + message: `Both post-connection-scripts and on-events are present. Only one of them can be used at a time.`, + path: options.path + }); + } +} diff --git a/packages/nango-yaml/lib/helpers.ts b/packages/nango-yaml/lib/helpers.ts index 41009ea636..e90400f6fe 100644 --- a/packages/nango-yaml/lib/helpers.ts +++ b/packages/nango-yaml/lib/helpers.ts @@ -181,7 +181,7 @@ export function shouldQuote(name: string) { export function getProviderConfigurationFromPath({ filePath, parsed }: { filePath: string; parsed: NangoYamlParsed }): NangoYamlParsedIntegration | null { const pathSegments = filePath.split(path.sep); const scriptType = pathSegments.length > 1 ? pathSegments[pathSegments.length - 2] : null; - const isNested = scriptType === 'syncs' || scriptType === 'actions' || scriptType === 'post-connection-scripts'; + const isNested = scriptType && ['syncs', 'actions', 'post-connection-scripts', 'on-events'].includes(scriptType); const baseName = path.basename(filePath, '.ts'); let providerConfiguration: NangoYamlParsedIntegration | null = null; diff --git a/packages/nango-yaml/lib/parser.v1.ts b/packages/nango-yaml/lib/parser.v1.ts index 4d5c35643d..95537fd0d4 100644 --- a/packages/nango-yaml/lib/parser.v1.ts +++ b/packages/nango-yaml/lib/parser.v1.ts @@ -82,7 +82,7 @@ export class NangoYamlParserV1 extends NangoYamlParser { providerConfigKey: integrationName, syncs, actions, - postConnectionScripts: [] + onEventScripts: { 'post-connection-creation': [], 'pre-connection-deletion': [] } }; output.push(parsedIntegration); diff --git a/packages/nango-yaml/lib/parser.v1.unit.test.ts b/packages/nango-yaml/lib/parser.v1.unit.test.ts index e0a1c39ae9..c3b221c601 100644 --- a/packages/nango-yaml/lib/parser.v1.unit.test.ts +++ b/packages/nango-yaml/lib/parser.v1.unit.test.ts @@ -39,7 +39,7 @@ describe('parse', () => { version: '' } ], - postConnectionScripts: [], + onEventScripts: { 'post-connection-creation': [], 'pre-connection-deletion': [] }, actions: [ { description: '', @@ -93,7 +93,7 @@ describe('parse', () => { version: '' } ], - postConnectionScripts: [], + onEventScripts: { 'post-connection-creation': [], 'pre-connection-deletion': [] }, actions: [] } ], diff --git a/packages/nango-yaml/lib/parser.v2.ts b/packages/nango-yaml/lib/parser.v2.ts index 41a312e756..0d832474b2 100644 --- a/packages/nango-yaml/lib/parser.v2.ts +++ b/packages/nango-yaml/lib/parser.v2.ts @@ -11,7 +11,7 @@ import type { ScriptTypeLiteral } from '@nangohq/types'; import { NangoYamlParser } from './parser.js'; -import { ParserErrorEndpointsMismatch, ParserErrorInvalidRuns } from './errors.js'; +import { ParserErrorEndpointsMismatch, ParserErrorInvalidRuns, ParserErrorBothPostConnectionScriptsAndOnEventsPresent } from './errors.js'; import { getInterval, parseEndpoint } from './helpers.js'; export class NangoYamlParserV2 extends NangoYamlParser { @@ -37,15 +37,25 @@ export class NangoYamlParserV2 extends NangoYamlParser { const syncs = integration['syncs']; const actions = integration['actions']; const postConnectionScripts: string[] = integration['post-connection-scripts'] || []; + const onEventScripts: Record = integration['on-events'] || {}; const parsedSyncs = this.parseSyncs({ syncs, integrationName }); const parseActions = this.parseActions({ actions, integrationName }); + if (postConnectionScripts.length > 0 && Object.values(onEventScripts).length > 0) { + this.errors.push(new ParserErrorBothPostConnectionScriptsAndOnEventsPresent({ path: [integrationName, 'on-events'] })); + } + const parsedOnEventScripts = { + 'post-connection-creation': onEventScripts['post-connection-creation'] || [], + 'pre-connection-deletion': onEventScripts['pre-connection-deletion'] || [] + }; + const parsedIntegration: NangoYamlParsedIntegration = { providerConfigKey: integrationName, syncs: parsedSyncs, actions: parseActions, - postConnectionScripts + onEventScripts: parsedOnEventScripts, + ...(postConnectionScripts.length > 0 ? { postConnectionScripts } : {}) }; output.push(parsedIntegration); diff --git a/packages/nango-yaml/lib/parser.v2.unit.test.ts b/packages/nango-yaml/lib/parser.v2.unit.test.ts index d11be900b6..c933f33a2d 100644 --- a/packages/nango-yaml/lib/parser.v2.unit.test.ts +++ b/packages/nango-yaml/lib/parser.v2.unit.test.ts @@ -1,7 +1,13 @@ import { expect, describe, it } from 'vitest'; import { NangoYamlParserV2 } from './parser.v2.js'; import type { NangoYamlParsed, NangoYamlV2 } from '@nangohq/types'; -import { ParserErrorDuplicateEndpoint, ParserErrorMissingId, ParserErrorModelIsLiteral, ParserErrorModelNotFound } from './errors.js'; +import { + ParserErrorBothPostConnectionScriptsAndOnEventsPresent, + ParserErrorDuplicateEndpoint, + ParserErrorMissingId, + ParserErrorModelIsLiteral, + ParserErrorModelNotFound +} from './errors.js'; describe('parse', () => { it('should parse', () => { @@ -39,7 +45,7 @@ describe('parse', () => { version: '' } ], - postConnectionScripts: [], + onEventScripts: { 'post-connection-creation': [], 'pre-connection-deletion': [] }, actions: [ { description: '', @@ -87,7 +93,7 @@ describe('parse', () => { { providerConfigKey: 'provider', syncs: [], - postConnectionScripts: [], + onEventScripts: { 'post-connection-creation': [], 'pre-connection-deletion': [] }, actions: [ { description: '', @@ -145,7 +151,7 @@ describe('parse', () => { version: '' } ], - postConnectionScripts: [], + onEventScripts: { 'post-connection-creation': [], 'pre-connection-deletion': [] }, actions: [] } ], @@ -335,4 +341,44 @@ describe('parse', () => { ]); }); }); + it('should error if both post-connection-scripts and on-events are present', () => { + const v2: NangoYamlV2 = { + models: {}, + integrations: { + provider: { + 'post-connection-scripts': ['test'], + 'on-events': { 'post-connection-creation': ['test'] } + } + } + }; + const parser = new NangoYamlParserV2({ raw: v2, yaml: '' }); + parser.parse(); + expect(parser.errors).toStrictEqual([new ParserErrorBothPostConnectionScriptsAndOnEventsPresent({ path: ['provider', 'on-events'] })]); + expect(parser.warnings).toStrictEqual([]); + }); + it('should handle post-connection-scripts', () => { + const v2: NangoYamlV2 = { + models: {}, + integrations: { provider: { 'post-connection-scripts': ['test'] } } + }; + const parser = new NangoYamlParserV2({ raw: v2, yaml: '' }); + parser.parse(); + expect(parser.errors).toStrictEqual([]); + expect(parser.warnings).toStrictEqual([]); + expect(parser.parsed?.integrations[0]?.postConnectionScripts).toStrictEqual(['test']); + }); + it('should handle on-events', () => { + const v2: NangoYamlV2 = { + models: {}, + integrations: { provider: { 'on-events': { 'post-connection-creation': ['test1', 'test2'], 'pre-connection-deletion': ['test3', 'test4'] } } } + }; + const parser = new NangoYamlParserV2({ raw: v2, yaml: '' }); + parser.parse(); + expect(parser.errors).toStrictEqual([]); + expect(parser.warnings).toStrictEqual([]); + expect(parser.parsed?.integrations[0]?.onEventScripts).toStrictEqual({ + 'post-connection-creation': ['test1', 'test2'], + 'pre-connection-deletion': ['test3', 'test4'] + }); + }); }); diff --git a/packages/server/lib/controllers/sync/deploy/postConfirmation.integration.test.ts b/packages/server/lib/controllers/sync/deploy/postConfirmation.integration.test.ts index 4a25185f75..e0f1d00337 100644 --- a/packages/server/lib/controllers/sync/deploy/postConfirmation.integration.test.ts +++ b/packages/server/lib/controllers/sync/deploy/postConfirmation.integration.test.ts @@ -48,35 +48,6 @@ describe(`POST ${endpoint}`, () => { expect(res.res.status).toBe(400); }); - it('should validate onEventScriptsByProvider', async () => { - const { env } = await seeders.seedAccountEnvAndUser(); - const res = await api.fetch(endpoint, { - method: 'POST', - token: env.secret_key, - // @ts-expect-error on purpose - body: { - debug: false, - flowConfigs: [], - reconcile: false - } - }); - - isError(res.json); - expect(res.json).toStrictEqual({ - error: { - code: 'invalid_body', - errors: [ - { - code: 'custom', - message: 'Either onEventScriptsByProvider or postConnectionScriptsByProvider must be provided', - path: ['onEventScriptsByProvider or postConnectionScriptsByProvider'] - } - ] - } - }); - expect(res.res.status).toBe(400); - }); - it('should accept empty body', async () => { const { env } = await seeders.seedAccountEnvAndUser(); const res = await api.fetch(endpoint, { diff --git a/packages/server/lib/controllers/sync/deploy/postDeploy.integration.test.ts b/packages/server/lib/controllers/sync/deploy/postDeploy.integration.test.ts index edbf94f5e2..eb7dc590bd 100644 --- a/packages/server/lib/controllers/sync/deploy/postDeploy.integration.test.ts +++ b/packages/server/lib/controllers/sync/deploy/postDeploy.integration.test.ts @@ -54,36 +54,6 @@ describe(`POST ${endpoint}`, () => { expect(res.res.status).toBe(400); }); - it('should validate onEventScriptsByProvider', async () => { - const { env } = await seeders.seedAccountEnvAndUser(); - const res = await api.fetch(endpoint, { - method: 'POST', - token: env.secret_key, - // @ts-expect-error on purpose - body: { - debug: false, - flowConfigs: [], - nangoYamlBody: '', - reconcile: false - } - }); - - isError(res.json); - expect(res.json).toStrictEqual({ - error: { - code: 'invalid_body', - errors: [ - { - code: 'custom', - message: 'Either onEventScriptsByProvider or postConnectionScriptsByProvider must be provided', - path: ['onEventScriptsByProvider or postConnectionScriptsByProvider'] - } - ] - } - }); - expect(res.res.status).toBe(400); - }); - it('should accept empty body', async () => { const { env } = await seeders.seedAccountEnvAndUser(); const res = await api.fetch(endpoint, { @@ -151,7 +121,18 @@ describe(`POST ${endpoint}`, () => { } ], nangoYamlBody: ``, - onEventScriptsByProvider: [], + onEventScriptsByProvider: [ + { + providerConfigKey: 'unauthenticated', + scripts: [ + { + name: 'test', + fileBody: { js: 'js file', ts: 'ts file' }, + event: 'post-connection-creation' + } + ] + } + ], reconcile: false, singleDeployMode: false } @@ -170,7 +151,6 @@ describe(`POST ${endpoint}`, () => { expect(syncConfigs).toStrictEqual([ { actions: [], - postConnectionScripts: [], provider: 'unauthenticated', providerConfigKey: 'unauthenticated', syncs: [ diff --git a/packages/server/lib/controllers/sync/deploy/validation.ts b/packages/server/lib/controllers/sync/deploy/validation.ts index 39cc19cebb..b0d8c65c05 100644 --- a/packages/server/lib/controllers/sync/deploy/validation.ts +++ b/packages/server/lib/controllers/sync/deploy/validation.ts @@ -81,12 +81,37 @@ export const flowConfig = z .strict(); const flowConfigs = z.array(flowConfig); const onEventScriptsByProvider = z.array( + z + .object({ + providerConfigKey: providerConfigKeySchema, + scripts: z.array( + z + .object({ + name: z.string().min(1).max(255), + fileBody, + event: z.enum(['post-connection-creation', 'pre-connection-deletion']) + }) + .strict() + ) + }) + .strict() +); +// DEPRECATED +const postConnectionScriptsByProvider = z.array( z .object({ providerConfigKey: providerConfigKeySchema, scripts: z.array(z.object({ name: z.string().min(1).max(255), fileBody }).strict()) }) .strict() + .transform((data) => ({ + providerConfigKey: data.providerConfigKey, + scripts: data.scripts.map((script) => ({ + name: script.name, + fileBody: script.fileBody, + event: 'post-connection-creation' + })) + })) ); const commonValidation = z @@ -94,7 +119,7 @@ const commonValidation = z flowConfigs, onEventScriptsByProvider: onEventScriptsByProvider.optional(), // postConnectionScriptsByProvider is deprecated but still supported for backwards compatibility - postConnectionScriptsByProvider: onEventScriptsByProvider.optional(), + postConnectionScriptsByProvider: postConnectionScriptsByProvider.optional(), jsonSchema: jsonSchema.optional(), reconcile: z.boolean(), debug: z.boolean(), @@ -105,20 +130,10 @@ const commonValidation = z const addOnEventScriptsValidation = (schema: T) => // cannot transform commonValidation because it cannot be merge with another schema // https://github.com/colinhacks/zod/issues/2474 - schema - .refine( - (data) => { - return data.onEventScriptsByProvider || data.postConnectionScriptsByProvider; - }, - { - message: 'Either onEventScriptsByProvider or postConnectionScriptsByProvider must be provided', - path: ['onEventScriptsByProvider or postConnectionScriptsByProvider'] - } - ) - .transform((data) => ({ - ...data, - onEventScriptsByProvider: data.onEventScriptsByProvider || data.postConnectionScriptsByProvider || [] - })); + schema.transform((data) => ({ + ...data, + onEventScriptsByProvider: data.onEventScriptsByProvider || data.postConnectionScriptsByProvider + })); export const validation = addOnEventScriptsValidation(commonValidation); diff --git a/packages/server/lib/hooks/connection/on/connection-created.ts b/packages/server/lib/hooks/connection/on/connection-created.ts index 82cefdf31f..9ec9c7a26b 100644 --- a/packages/server/lib/hooks/connection/on/connection-created.ts +++ b/packages/server/lib/hooks/connection/on/connection-created.ts @@ -14,7 +14,7 @@ export async function onConnectionCreated(createdConnection: RecentlyCreatedConn return; } - const onConnectionCreatedScripts = await onEventScriptService.getByConfig(config_id); + const onConnectionCreatedScripts = await onEventScriptService.getByConfig(config_id, 'post-connection-creation'); if (!onConnectionCreatedScripts || onConnectionCreatedScripts.length === 0) { return; diff --git a/packages/shared/lib/models/NangoConfig.ts b/packages/shared/lib/models/NangoConfig.ts index 60f3c8df80..80f7439e96 100644 --- a/packages/shared/lib/models/NangoConfig.ts +++ b/packages/shared/lib/models/NangoConfig.ts @@ -139,5 +139,4 @@ export interface StandardNangoConfig { provider?: string; syncs: NangoSyncConfig[]; actions: NangoSyncConfig[]; - postConnectionScripts?: string[]; } diff --git a/packages/shared/lib/services/sync/config/config.service.ts b/packages/shared/lib/services/sync/config/config.service.ts index 53254c53d8..77b5db05a4 100644 --- a/packages/shared/lib/services/sync/config/config.service.ts +++ b/packages/shared/lib/services/sync/config/config.service.ts @@ -22,7 +22,6 @@ function convertSyncConfigToStandardConfig(syncConfigs: ExtendedSyncConfig[]): S if (!tmp[syncConfig.provider]) { tmp[syncConfig.provider] = { actions: [], - postConnectionScripts: [], providerConfigKey: syncConfig.unique_key, provider: syncConfig.provider, syncs: [] diff --git a/packages/shared/lib/services/sync/config/deploy.service.ts b/packages/shared/lib/services/sync/config/deploy.service.ts index 7a06f6d32f..988ed5fb8a 100644 --- a/packages/shared/lib/services/sync/config/deploy.service.ts +++ b/packages/shared/lib/services/sync/config/deploy.service.ts @@ -83,7 +83,7 @@ export async function deploy({ account: DBTeam; flows: CleanedIncomingFlowConfig[]; jsonSchema?: JSONSchema7 | undefined; - onEventScriptsByProvider: OnEventScriptsByProvider[]; + onEventScriptsByProvider?: OnEventScriptsByProvider[] | undefined; nangoYamlBody: string; logContextGetter: LogContextGetter; orchestrator: Orchestrator; @@ -179,7 +179,7 @@ export async function deploy({ await db.knex.from(ENDPOINT_TABLE).insert(endpoints); } - if (onEventScriptsByProvider.length > 0) { + if (onEventScriptsByProvider) { await onEventScriptService.update({ environment, account, onEventScriptsByProvider }); } diff --git a/packages/shared/lib/services/sync/on-event-scripts.service.ts b/packages/shared/lib/services/sync/on-event-scripts.service.ts index 8ff1c50137..1a98c03e99 100644 --- a/packages/shared/lib/services/sync/on-event-scripts.service.ts +++ b/packages/shared/lib/services/sync/on-event-scripts.service.ts @@ -1,12 +1,21 @@ import db from '@nangohq/database'; import remoteFileService from '../file/remote.service.js'; import { env } from '@nangohq/utils'; -import type { OnEventScriptsByProvider, OnEventScript, DBTeam, DBEnvironment } from '@nangohq/types'; +import type { OnEventScriptsByProvider, OnEventScript, DBTeam, DBEnvironment, OnEventType } from '@nangohq/types'; import { increment } from './config/config.service.js'; import configService from '../config.service.js'; const TABLE = 'on_event_scripts'; +function toDbEvent(eventType: OnEventType): OnEventScript['event'] { + switch (eventType) { + case 'post-connection-creation': + return 'POST_CONNECTION_CREATION'; + case 'pre-connection-deletion': + return 'PRE_CONNECTION_DELETION'; + } +} + export const onEventScriptService = { async update({ environment, @@ -19,39 +28,35 @@ export const onEventScriptService = { }): Promise { await db.knex.transaction(async (trx) => { const onEventInserts: Omit[] = []; + + // Deactivate all previous scripts for the environment + // This is done to ensure that we don't have any orphaned scripts when they are removed from nango.yaml + const previousScriptVersions = await trx + .from(TABLE) + .whereRaw(`config_id IN (SELECT id FROM _nango_configs WHERE environment_id = ?)`, [environment.id]) + .where({ + active: true + }) + .update({ + active: false + }) + .returning('*'); + for (const onEventScriptByProvider of onEventScriptsByProvider) { const { providerConfigKey, scripts } = onEventScriptByProvider; - for (const script of scripts) { - const { name, fileBody } = script; - const config = await configService.getProviderConfig(providerConfigKey, environment.id); - - if (!config || !config.id) { - continue; - } + const config = await configService.getProviderConfig(providerConfigKey, environment.id); + if (!config || !config.id) { + continue; + } - const previousScriptVersion = await trx - .from(TABLE) - .select('version') - .where({ - config_id: config.id, - name, - active: true - }) - .first(); + for (const script of scripts) { + const { name, fileBody, event: scriptEvent } = script; + const event = toDbEvent(scriptEvent); + const previousScriptVersion = previousScriptVersions.find((p) => p.config_id === config.id && p.name === name && p.event === event); const version = previousScriptVersion ? increment(previousScriptVersion.version) : '0.0.1'; - await trx - .from(TABLE) - .where({ - config_id: config.id, - name - }) - .update({ - active: false - }); - const file_location = await remoteFileService.upload( fileBody.js, `${env}/account/${account.id}/environment/${environment.id}/config/${config.id}/${name}-v${version}.js`, @@ -73,14 +78,17 @@ export const onEventScriptService = { name, file_location, version: version.toString(), - active: true + active: true, + event }); } } - await trx.insert(onEventInserts).into(TABLE); + if (onEventInserts.length > 0) { + await trx.insert(onEventInserts).into(TABLE); + } }); }, - getByConfig: async (configId: number): Promise => { - return db.knex.from(TABLE).where({ config_id: configId, active: true }); + getByConfig: async (configId: number, event: OnEventType): Promise => { + return db.knex.from(TABLE).where({ config_id: configId, active: true, event: toDbEvent(event) }); } }; diff --git a/packages/types/lib/deploy/api.ts b/packages/types/lib/deploy/api.ts index 4d562bca2f..618bb11af8 100644 --- a/packages/types/lib/deploy/api.ts +++ b/packages/types/lib/deploy/api.ts @@ -9,7 +9,7 @@ export type PostDeployConfirmation = Endpoint<{ Path: '/sync/deploy/confirmation'; Body: { flowConfigs: IncomingFlowConfig[]; - onEventScriptsByProvider: OnEventScriptsByProvider[]; + onEventScriptsByProvider?: OnEventScriptsByProvider[] | undefined; reconcile: boolean; debug: boolean; singleDeployMode?: boolean; @@ -23,7 +23,7 @@ export type PostDeploy = Endpoint<{ Path: '/sync/deploy'; Body: { flowConfigs: IncomingFlowConfig[]; - onEventScriptsByProvider: OnEventScriptsByProvider[]; + onEventScriptsByProvider?: OnEventScriptsByProvider[] | undefined; nangoYamlBody: string; reconcile: boolean; debug: boolean; @@ -41,7 +41,7 @@ export type PostDeployInternal = Endpoint<{ }; Body: { flowConfigs: IncomingFlowConfig[]; - onEventScriptsByProvider: OnEventScriptsByProvider[]; + onEventScriptsByProvider?: OnEventScriptsByProvider[] | undefined; nangoYamlBody: string; reconcile: boolean; debug: boolean; diff --git a/packages/types/lib/deploy/incomingFlow.ts b/packages/types/lib/deploy/incomingFlow.ts index 521bbf7d6b..80f3dbcba1 100644 --- a/packages/types/lib/deploy/incomingFlow.ts +++ b/packages/types/lib/deploy/incomingFlow.ts @@ -1,5 +1,5 @@ import type { Merge } from 'type-fest'; -import type { NangoModel, NangoSyncEndpointOld, NangoSyncEndpointV2, ScriptTypeLiteral, SyncTypeLiteral } from '../nangoYaml'; +import type { NangoModel, NangoSyncEndpointOld, NangoSyncEndpointV2, OnEventType, ScriptTypeLiteral, SyncTypeLiteral } from '../nangoYaml'; export interface IncomingScriptFiles { js: string; @@ -8,6 +8,7 @@ export interface IncomingScriptFiles { export interface IncomingOnEventScript { name: string; fileBody: IncomingScriptFiles; + event: OnEventType; } export interface OnEventScriptsByProvider { diff --git a/packages/types/lib/nangoYaml/index.ts b/packages/types/lib/nangoYaml/index.ts index fba3dd438a..ca14c0ecc2 100644 --- a/packages/types/lib/nangoYaml/index.ts +++ b/packages/types/lib/nangoYaml/index.ts @@ -1,6 +1,6 @@ export type HTTP_METHOD = 'GET' | 'POST' | 'PATCH' | 'PUT' | 'DELETE'; export type SyncTypeLiteral = 'incremental' | 'full'; -export type ScriptFileType = 'actions' | 'syncs' | 'post-connection-scripts'; +export type ScriptFileType = 'actions' | 'syncs' | 'on-events' | 'post-connection-scripts'; // post-connection-scripts is deprecated export type ScriptTypeLiteral = 'action' | 'sync'; // -------------- @@ -36,6 +36,10 @@ export interface NangoYamlV2Integration { provider?: string; syncs?: Record; actions?: Record; + 'on-events'?: Record; + /** + * @deprecated + */ 'post-connection-scripts'?: string[]; } export interface NangoYamlV2IntegrationSync { @@ -76,6 +80,8 @@ export type NangoYamlModelField = boolean | number | string | null | string[] | export type NangoYaml = NangoYamlV1 | NangoYamlV2; // -------------- Parsed +export type OnEventType = 'post-connection-creation' | 'pre-connection-deletion'; + export interface NangoYamlParsed { yamlVersion: 'v1' | 'v2'; integrations: NangoYamlParsedIntegration[]; @@ -85,7 +91,11 @@ export interface NangoYamlParsedIntegration { providerConfigKey: string; syncs: ParsedNangoSync[]; actions: ParsedNangoAction[]; - postConnectionScripts: string[]; + onEventScripts: Record; + /** + * @deprecated + */ + postConnectionScripts?: string[]; } export interface ParsedNangoSync { name: string; diff --git a/packages/types/lib/scripts/post-connection/db.ts b/packages/types/lib/scripts/post-connection/db.ts index d2e252e7e4..8b43896935 100644 --- a/packages/types/lib/scripts/post-connection/db.ts +++ b/packages/types/lib/scripts/post-connection/db.ts @@ -7,4 +7,5 @@ export interface OnEventScript extends Timestamps { file_location: string; version: string; active: boolean; + event: 'POST_CONNECTION_CREATION' | 'PRE_CONNECTION_DELETION'; }