From 731efc7a66e31dc9243ebaa729e5061bd61c4f45 Mon Sep 17 00:00:00 2001 From: Kostya Tsymbal Date: Wed, 31 Mar 2021 18:04:52 +0300 Subject: [PATCH 01/22] Add abbility to reverse from Confluent --- reverse_engineering/api.js | 2 +- reverse_engineering/config.json | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/reverse_engineering/api.js b/reverse_engineering/api.js index f385084..b3d9e97 100644 --- a/reverse_engineering/api.js +++ b/reverse_engineering/api.js @@ -111,7 +111,7 @@ const handleFileData = (filePath) => { if (extension === '.avro') { readAvroData(filePath, respond); - } else if (extension === '.avsc') { + } else if (extension === '.avsc' || extension==='.confluent-avro') { fs.readFile(filePath, 'utf-8', respond); } else { const error = new Error(`The file ${filePath} is not recognized as Avro Schema or Data.`) diff --git a/reverse_engineering/config.json b/reverse_engineering/config.json index 2aeacd9..d9ffad7 100644 --- a/reverse_engineering/config.json +++ b/reverse_engineering/config.json @@ -3,5 +3,6 @@ "name": "avro", "extensions": ["avsc", "avro"], "useCloudStorage": true, - "useMultipleFileReverse": true + "useMultipleFileReverse": true, + "useConfluent": true } \ No newline at end of file From 5dc6f483a9784756319f1a29c0a6472c506a877a Mon Sep 17 00:00:00 2001 From: Kostya Tsymbal Date: Fri, 9 Apr 2021 17:54:42 +0300 Subject: [PATCH 02/22] Add extension to reverse from Confluent --- reverse_engineering/config.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/reverse_engineering/config.json b/reverse_engineering/config.json index d9ffad7..f64df43 100644 --- a/reverse_engineering/config.json +++ b/reverse_engineering/config.json @@ -1,7 +1,7 @@ { "menu_label": "Avro schema or data file", "name": "avro", - "extensions": ["avsc", "avro"], + "extensions": ["avsc", "avro", "confluent-avro"], "useCloudStorage": true, "useMultipleFileReverse": true, "useConfluent": true From dd52616a21e29225c16e18ec7c63c57760a74aea Mon Sep 17 00:00:00 2001 From: Pascal Desmarets Date: Fri, 9 Apr 2021 19:01:14 +0200 Subject: [PATCH 03/22] Reverse-engineering from Confluent Schema Registry --- README.md | 4 ++-- package.json | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 1287063..e8f896b 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,8 @@ # Avro -Plugin to enable Avro as a target in Hackolade data modeling. +Plugin to enable Avro as a target in [Hackolade](https://hackolade.com) data modeling. Requires prior download of the Hackolade application from our [download page](https://hackolade.com/download.html) Hackolade exposes its core data modeling engine through a plugin architecture. Each plugin applies the Hackolade data modeling capabilities to a specific target technology, whether for data-at-rest (databases) or data-in-motion (communications.) Each plugin matches the specific aspects of the target in terms of terminology, storage model, data types, and communication protocol. -To enable data modeling capabilities for a target, you must first download and install the plugin, following these [instructions](https://hackolade.com/help/DownloadadditionalDBtargetplugin.htm "Plugin download instructions"). +To enable data modeling capabilities for a target, you must first download and install the plugin, following these [instructions](https://hackolade.com/help/DownloadadditionalDBtargetplugin.html "Plugin download instructions"). Plugins can be customized by following these [instructions](https://hackolade.com/help/Userdefinedcustomproperties.html "Plugin customization instructions"). diff --git a/package.json b/package.json index 3b94165..33805b2 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "Avro", - "version": "0.1.58", - "versionDate": "2021-02-13", + "version": "0.1.59", + "versionDate": "2021-04-09", "author": "hackolade", "engines": { "hackolade": "3.3.2", From 21a4ed3e70fbf28111ab5395fb686d17d1861970 Mon Sep 17 00:00:00 2001 From: Kostya Tsymbal Date: Wed, 14 Apr 2021 19:34:00 +0300 Subject: [PATCH 04/22] Confluent: add apply to instance --- forward_engineering/api.js | 130 +++++++++++++++++++++++--------- forward_engineering/config.json | 61 +++++++++++---- localization/en.json | 1 + package.json | 2 +- 4 files changed, 143 insertions(+), 51 deletions(-) diff --git a/forward_engineering/api.js b/forward_engineering/api.js index c6ba6cd..2a432ae 100644 --- a/forward_engineering/api.js +++ b/forward_engineering/api.js @@ -33,38 +33,95 @@ const LOGICAL_TYPES_MAP = { fixed: ['decimal', 'duration'] }; -module.exports = { - generateScript(data, logger, cb) { - logger.clear(); - try { - const name = getRecordName(data); - let avroSchema = { name }; - let jsonSchema = JSON.parse(data.jsonSchema); - const udt = getUserDefinedTypes(data); +const getCommonEntitiesData=(data)=>{ + const {modelDefinitions, externalDefinitions}=data; + const options= { + targetScriptOptions: { + keyword: "confluentSchemaRegistry", + }, + }; + + return {options, modelDefinitions, externalDefinitions} +} - jsonSchema.type = 'root'; - handleRecursiveSchema(jsonSchema, avroSchema, {}, udt); +const getEntityData=(container, entityId)=>{ + const containerData=_.first(_.get(container, 'containerData', [])); + const jsonSchema=container.jsonSchema[entityId]; + const jsonData=container.jsonData[entityId]; + const entityData=_.first(container.entityData[entityId]); + const internalDefinitions=container.internalDefinitions[entityId]; - if (data.containerData) { - avroSchema.namespace = data.containerData.name; - } - avroSchema.type = 'record'; - avroSchema = reorderAvroSchema(avroSchema); - avroSchema = JSON.stringify(avroSchema, null, 4); - const options = data.options; - const additionalOptions = _.get(options, 'additionalOptions', []); - const targetScriptType = _.get(options, 'targetScriptOptions.keyword'); - if (targetScriptType === 'schemaRegistry') { - avroSchema = JSON.stringify({ schema: JSON.stringify(JSON.parse(avroSchema))}, null, 4); - } + return {containerData, jsonSchema, jsonData, entityData, internalDefinitions} +} - const needMinify = (additionalOptions.find(option => option.id === 'minify') || {}).value; - if (needMinify) { - avroSchema = JSON.stringify(JSON.parse(avroSchema)); - } +const getScript = (data) => { + const name = getRecordName(data); + let avroSchema = { name }; + let jsonSchema = JSON.parse(data.jsonSchema); + const udt = getUserDefinedTypes(data); - nameIndex = 0; - return cb(null, avroSchema); + jsonSchema.type = 'root'; + handleRecursiveSchema(jsonSchema, avroSchema, {}, udt); + + if (data.containerData) { + avroSchema.namespace = data.containerData.name; + } + avroSchema.type = 'record'; + avroSchema = reorderAvroSchema(avroSchema); + avroSchema = JSON.stringify(avroSchema, null, 4); + const options = data.options; + const additionalOptions = _.get(options, 'additionalOptions', []); + const targetScriptType = _.get(options, 'targetScriptOptions.keyword'); + if (targetScriptType === 'schemaRegistry') { + avroSchema = JSON.stringify({ schema: JSON.stringify(JSON.parse(avroSchema)) }, null, 4); + } + + if (targetScriptType === 'confluentSchemaRegistry') { + avroSchema = `POST /subjects/${name}/versions\n${JSON.stringify({ schemaType: "AVRO", schema: JSON.stringify(JSON.parse(avroSchema)) }, null, 4)}` + } + + const needMinify = targetScriptType !== 'confluentSchemaRegistry' && (additionalOptions.find(option => option.id === 'minify') || {}).value; + if (needMinify) { + avroSchema = JSON.stringify(JSON.parse(avroSchema)); + } + + nameIndex = 0; + return avroSchema; +} + +module.exports = { + generateModelScript(data, logger, cb){ + logger.clear(); + try{ + const commonData=getCommonEntitiesData(data); + const containers=_.get(data, 'containers', []); + const script =containers.reduce((createdQueries, container)=>{ + const containerEntities=container.entities.map(entityId=>{ + return Object.assign({}, commonData, getEntityData(container, entityId)) + }) + + const containerQueries=containerEntities.map(entity=>{ + try{ + return getScript(entity) + }catch(e){ + logger.log('error', { message: err.message, stack: err.stack }, 'Avro Forward-Engineering Error'); + return ''; + } + }) + + return [...createdQueries, ...containerQueries]; + }, []) + cb(null, script.join('\n\n')); + }catch(err){ + logger.log('error', { message: err.message, stack: err.stack }, 'Avro model Forward-Engineering Error'); + cb({ message: err.message, stack: err.stack }); + } + }, + generateScript(data, logger, cb) { + logger.clear(); + try { + const script=getScript(data); + cb(null, script) } catch(err) { nameIndex = 0; logger.log('error', { message: err.message, stack: err.stack }, 'Avro Forward-Engineering Error'); @@ -73,14 +130,19 @@ module.exports = { }, validate(data, logger, cb) { try { - let avroSchema = JSON.parse(data.script); - if (Object.keys(avroSchema).length === 1 && avroSchema.schema) { + let targetScript=data.script; + if(data.targetScriptOptions.keyword === 'confluentSchemaRegistry'){ + targetScript=targetScript.split('\n').slice(1).join('\n') + } + let avroSchema = JSON.parse(targetScript); + + if(data.targetScriptOptions.keyword !== 'avroSchema'){ const messages = validationHelper.validate(avroSchema.schema); - cb(null, messages); - } else { - const messages = validationHelper.validate(data.script); - cb(null, messages); + return cb(null, messages); } + + const messages = validationHelper.validate(targetScript); + cb(null, messages); } catch (e) { logger.log('error', { error: e }, 'Avro Validation Error'); cb(null, [{ diff --git a/forward_engineering/config.json b/forward_engineering/config.json index 1de004a..c5586e6 100644 --- a/forward_engineering/config.json +++ b/forward_engineering/config.json @@ -1,18 +1,47 @@ { - "extension": "avsc", - "filterName": "Avro schema", - "namePrefix": "Avro Schema", - "resolvedDefinitions": false, - "validation": true, - "options": [ - { "name": "Avro Schema", "keyword": "avroSchema"}, - { "name": "Schema registry", "keyword": "schemaRegistry"} - ], - "additionalOptions": [ - { - "id": "minify", - "value": false, - "name": "Minify" - } - ] + "extension": "avsc", + "filterName": "Avro schema", + "namePrefix": "Avro Schema", + "resolvedDefinitions": false, + "validation": true, + "applyScriptToInstance": true, + "options": [ + { + "name": "Avro Schema", + "keyword": "avroSchema", + "disableApplyScriptToInstance": true + }, + { + "name": "Schema registry", + "keyword": "schemaRegistry", + "disableApplyScriptToInstance": false + }, + { + "name": "Confluent Schema", + "keyword": "confluentSchemaRegistry", + "isApplicationHandler": true + } + ], + "additionalOptions": [ + { + "id": "minify", + "value": false, + "name": "Minify", + "targetFEKeywords": ["avroSchema", "schemaRegistry"] + } + ], + "level": { + "entity": true, + "model": { + "validation": false, + "options": [ + { + "name": "Confluent Schema", + "keyword": "confluentSchemaRegistry", + "mode": "json", + "isApplicationHandler": true + } + ] + } + } } diff --git a/localization/en.json b/localization/en.json index 5eb4d9e..ff93c43 100644 --- a/localization/en.json +++ b/localization/en.json @@ -137,6 +137,7 @@ "TYPE": {}, "CENTRAL_PANE___TAB_MODEL_DEFINITIONS": "Model Definitions", "CENTRAL_PANE___FE_SCRIPT": "Avro Schema", + "CENTRAL_PANE___MODEL_FE_SCRIPT": "Confluent", "CONTEXT_MENU___ADD_MODEL_REFERENCE": "Model Definition", "CONTEXT_MENU___GO_TO_DEFINITION": "Go to Model Definition", "DOCUMENTATION___DB_DEFINITIONS": "Model Definitions", diff --git a/package.json b/package.json index 3b94165..1c0857c 100644 --- a/package.json +++ b/package.json @@ -4,7 +4,7 @@ "versionDate": "2021-02-13", "author": "hackolade", "engines": { - "hackolade": "3.3.2", + "hackolade": "4.3.16", "hackoladePlugin": "1.0.0" }, "contributes": { From e62e0214846c75006bcecfc5ae78645ea002e4ab Mon Sep 17 00:00:00 2001 From: Kostya Tsymbal Date: Thu, 15 Apr 2021 17:09:05 +0300 Subject: [PATCH 05/22] Formatted code --- forward_engineering/api.js | 58 +++++++++++++++++++------------------- 1 file changed, 29 insertions(+), 29 deletions(-) diff --git a/forward_engineering/api.js b/forward_engineering/api.js index 2a432ae..a63dc46 100644 --- a/forward_engineering/api.js +++ b/forward_engineering/api.js @@ -33,25 +33,25 @@ const LOGICAL_TYPES_MAP = { fixed: ['decimal', 'duration'] }; -const getCommonEntitiesData=(data)=>{ - const {modelDefinitions, externalDefinitions}=data; - const options= { +const getCommonEntitiesData = (data) => { + const { modelDefinitions, externalDefinitions } = data; + const options = { targetScriptOptions: { - keyword: "confluentSchemaRegistry", + keyword: "confluentSchemaRegistry", }, - }; + }; - return {options, modelDefinitions, externalDefinitions} + return { options, modelDefinitions, externalDefinitions } } -const getEntityData=(container, entityId)=>{ - const containerData=_.first(_.get(container, 'containerData', [])); - const jsonSchema=container.jsonSchema[entityId]; - const jsonData=container.jsonData[entityId]; - const entityData=_.first(container.entityData[entityId]); - const internalDefinitions=container.internalDefinitions[entityId]; +const getEntityData = (container, entityId) => { + const containerData = _.first(_.get(container, 'containerData', [])); + const jsonSchema = container.jsonSchema[entityId]; + const jsonData = container.jsonData[entityId]; + const entityData = _.first(container.entityData[entityId]); + const internalDefinitions = container.internalDefinitions[entityId]; - return {containerData, jsonSchema, jsonData, entityData, internalDefinitions} + return { containerData, jsonSchema, jsonData, entityData, internalDefinitions } } const getScript = (data) => { @@ -90,20 +90,20 @@ const getScript = (data) => { } module.exports = { - generateModelScript(data, logger, cb){ + generateModelScript(data, logger, cb) { logger.clear(); - try{ - const commonData=getCommonEntitiesData(data); - const containers=_.get(data, 'containers', []); - const script =containers.reduce((createdQueries, container)=>{ - const containerEntities=container.entities.map(entityId=>{ + try { + const commonData = getCommonEntitiesData(data); + const containers = _.get(data, 'containers', []); + const script = containers.reduce((createdQueries, container) => { + const containerEntities = container.entities.map(entityId => { return Object.assign({}, commonData, getEntityData(container, entityId)) }) - const containerQueries=containerEntities.map(entity=>{ - try{ + const containerQueries = containerEntities.map(entity => { + try { return getScript(entity) - }catch(e){ + } catch (e) { logger.log('error', { message: err.message, stack: err.stack }, 'Avro Forward-Engineering Error'); return ''; } @@ -112,7 +112,7 @@ module.exports = { return [...createdQueries, ...containerQueries]; }, []) cb(null, script.join('\n\n')); - }catch(err){ + } catch (err) { logger.log('error', { message: err.message, stack: err.stack }, 'Avro model Forward-Engineering Error'); cb({ message: err.message, stack: err.stack }); } @@ -120,9 +120,9 @@ module.exports = { generateScript(data, logger, cb) { logger.clear(); try { - const script=getScript(data); + const script = getScript(data); cb(null, script) - } catch(err) { + } catch (err) { nameIndex = 0; logger.log('error', { message: err.message, stack: err.stack }, 'Avro Forward-Engineering Error'); cb({ message: err.message, stack: err.stack }); @@ -130,13 +130,13 @@ module.exports = { }, validate(data, logger, cb) { try { - let targetScript=data.script; - if(data.targetScriptOptions.keyword === 'confluentSchemaRegistry'){ - targetScript=targetScript.split('\n').slice(1).join('\n') + let targetScript = data.script; + if (data.targetScriptOptions.keyword === 'confluentSchemaRegistry') { + targetScript = targetScript.split('\n').slice(1).join('\n') } let avroSchema = JSON.parse(targetScript); - if(data.targetScriptOptions.keyword !== 'avroSchema'){ + if (data.targetScriptOptions.keyword !== 'avroSchema') { const messages = validationHelper.validate(avroSchema.schema); return cb(null, messages); } From eee52babc9987b62a3f1aa4ac43cd493f8bb9fc5 Mon Sep 17 00:00:00 2001 From: Kostya Tsymbal Date: Thu, 15 Apr 2021 17:38:35 +0300 Subject: [PATCH 06/22] Moved functions --- forward_engineering/api.js | 112 ++++++++++++++++++------------------- 1 file changed, 56 insertions(+), 56 deletions(-) diff --git a/forward_engineering/api.js b/forward_engineering/api.js index a63dc46..c3956cb 100644 --- a/forward_engineering/api.js +++ b/forward_engineering/api.js @@ -33,62 +33,6 @@ const LOGICAL_TYPES_MAP = { fixed: ['decimal', 'duration'] }; -const getCommonEntitiesData = (data) => { - const { modelDefinitions, externalDefinitions } = data; - const options = { - targetScriptOptions: { - keyword: "confluentSchemaRegistry", - }, - }; - - return { options, modelDefinitions, externalDefinitions } -} - -const getEntityData = (container, entityId) => { - const containerData = _.first(_.get(container, 'containerData', [])); - const jsonSchema = container.jsonSchema[entityId]; - const jsonData = container.jsonData[entityId]; - const entityData = _.first(container.entityData[entityId]); - const internalDefinitions = container.internalDefinitions[entityId]; - - return { containerData, jsonSchema, jsonData, entityData, internalDefinitions } -} - -const getScript = (data) => { - const name = getRecordName(data); - let avroSchema = { name }; - let jsonSchema = JSON.parse(data.jsonSchema); - const udt = getUserDefinedTypes(data); - - jsonSchema.type = 'root'; - handleRecursiveSchema(jsonSchema, avroSchema, {}, udt); - - if (data.containerData) { - avroSchema.namespace = data.containerData.name; - } - avroSchema.type = 'record'; - avroSchema = reorderAvroSchema(avroSchema); - avroSchema = JSON.stringify(avroSchema, null, 4); - const options = data.options; - const additionalOptions = _.get(options, 'additionalOptions', []); - const targetScriptType = _.get(options, 'targetScriptOptions.keyword'); - if (targetScriptType === 'schemaRegistry') { - avroSchema = JSON.stringify({ schema: JSON.stringify(JSON.parse(avroSchema)) }, null, 4); - } - - if (targetScriptType === 'confluentSchemaRegistry') { - avroSchema = `POST /subjects/${name}/versions\n${JSON.stringify({ schemaType: "AVRO", schema: JSON.stringify(JSON.parse(avroSchema)) }, null, 4)}` - } - - const needMinify = targetScriptType !== 'confluentSchemaRegistry' && (additionalOptions.find(option => option.id === 'minify') || {}).value; - if (needMinify) { - avroSchema = JSON.stringify(JSON.parse(avroSchema)); - } - - nameIndex = 0; - return avroSchema; -} - module.exports = { generateModelScript(data, logger, cb) { logger.clear(); @@ -155,6 +99,62 @@ module.exports = { } }; +const getCommonEntitiesData = (data) => { + const { modelDefinitions, externalDefinitions } = data; + const options = { + targetScriptOptions: { + keyword: "confluentSchemaRegistry", + }, + }; + + return { options, modelDefinitions, externalDefinitions } +} + +const getEntityData = (container, entityId) => { + const containerData = _.first(_.get(container, 'containerData', [])); + const jsonSchema = container.jsonSchema[entityId]; + const jsonData = container.jsonData[entityId]; + const entityData = _.first(container.entityData[entityId]); + const internalDefinitions = container.internalDefinitions[entityId]; + + return { containerData, jsonSchema, jsonData, entityData, internalDefinitions } +} + +const getScript = (data) => { + const name = getRecordName(data); + let avroSchema = { name }; + let jsonSchema = JSON.parse(data.jsonSchema); + const udt = getUserDefinedTypes(data); + + jsonSchema.type = 'root'; + handleRecursiveSchema(jsonSchema, avroSchema, {}, udt); + + if (data.containerData) { + avroSchema.namespace = data.containerData.name; + } + avroSchema.type = 'record'; + avroSchema = reorderAvroSchema(avroSchema); + avroSchema = JSON.stringify(avroSchema, null, 4); + const options = data.options; + const additionalOptions = _.get(options, 'additionalOptions', []); + const targetScriptType = _.get(options, 'targetScriptOptions.keyword'); + if (targetScriptType === 'schemaRegistry') { + avroSchema = JSON.stringify({ schema: JSON.stringify(JSON.parse(avroSchema)) }, null, 4); + } + + if (targetScriptType === 'confluentSchemaRegistry') { + avroSchema = `POST /subjects/${name}/versions\n${JSON.stringify({ schemaType: "AVRO", schema: JSON.stringify(JSON.parse(avroSchema)) }, null, 4)}` + } + + const needMinify = targetScriptType !== 'confluentSchemaRegistry' && (additionalOptions.find(option => option.id === 'minify') || {}).value; + if (needMinify) { + avroSchema = JSON.stringify(JSON.parse(avroSchema)); + } + + nameIndex = 0; + return avroSchema; +} + const getUserDefinedTypes = ({ internalDefinitions, externalDefinitions, modelDefinitions }) => { let udt = convertSchemaToUserDefinedTypes(JSON.parse(externalDefinitions), {}); udt = convertSchemaToUserDefinedTypes(JSON.parse(modelDefinitions), udt); From d91f7de5ba36798427823736585692378098872b Mon Sep 17 00:00:00 2001 From: Kostya Tsymbal Date: Thu, 15 Apr 2021 17:44:23 +0300 Subject: [PATCH 07/22] Refactored --- forward_engineering/api.js | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/forward_engineering/api.js b/forward_engineering/api.js index c3956cb..9b287af 100644 --- a/forward_engineering/api.js +++ b/forward_engineering/api.js @@ -134,25 +134,25 @@ const getScript = (data) => { } avroSchema.type = 'record'; avroSchema = reorderAvroSchema(avroSchema); - avroSchema = JSON.stringify(avroSchema, null, 4); const options = data.options; const additionalOptions = _.get(options, 'additionalOptions', []); const targetScriptType = _.get(options, 'targetScriptOptions.keyword'); + nameIndex = 0; + if (targetScriptType === 'schemaRegistry') { - avroSchema = JSON.stringify({ schema: JSON.stringify(JSON.parse(avroSchema)) }, null, 4); + return JSON.stringify({ schema: JSON.stringify(avroSchema) }, null, 4); } if (targetScriptType === 'confluentSchemaRegistry') { - avroSchema = `POST /subjects/${name}/versions\n${JSON.stringify({ schemaType: "AVRO", schema: JSON.stringify(JSON.parse(avroSchema)) }, null, 4)}` + return `POST /subjects/${name}/versions\n${JSON.stringify({ schemaType: "AVRO", schema: JSON.stringify(avroSchema) }, null, 4)}` } const needMinify = targetScriptType !== 'confluentSchemaRegistry' && (additionalOptions.find(option => option.id === 'minify') || {}).value; if (needMinify) { - avroSchema = JSON.stringify(JSON.parse(avroSchema)); + return JSON.stringify(avroSchema); } - nameIndex = 0; - return avroSchema; + return JSON.stringify(avroSchema, null, 4); } const getUserDefinedTypes = ({ internalDefinitions, externalDefinitions, modelDefinitions }) => { From 06e7e6d94db5bb3aa8f6e6426421152e9b2d5d7b Mon Sep 17 00:00:00 2001 From: Kostya Tsymbal Date: Fri, 16 Apr 2021 13:06:52 +0300 Subject: [PATCH 08/22] Fixed localization --- forward_engineering/config.json | 4 ++-- localization/en.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/forward_engineering/config.json b/forward_engineering/config.json index c5586e6..aa02146 100644 --- a/forward_engineering/config.json +++ b/forward_engineering/config.json @@ -17,7 +17,7 @@ "disableApplyScriptToInstance": false }, { - "name": "Confluent Schema", + "name": "Confluent API Schema", "keyword": "confluentSchemaRegistry", "isApplicationHandler": true } @@ -36,7 +36,7 @@ "validation": false, "options": [ { - "name": "Confluent Schema", + "name": "Confluent API Schema", "keyword": "confluentSchemaRegistry", "mode": "json", "isApplicationHandler": true diff --git a/localization/en.json b/localization/en.json index ff93c43..56da4a1 100644 --- a/localization/en.json +++ b/localization/en.json @@ -137,7 +137,7 @@ "TYPE": {}, "CENTRAL_PANE___TAB_MODEL_DEFINITIONS": "Model Definitions", "CENTRAL_PANE___FE_SCRIPT": "Avro Schema", - "CENTRAL_PANE___MODEL_FE_SCRIPT": "Confluent", + "CENTRAL_PANE___MODEL_FE_SCRIPT": "Confluent API", "CONTEXT_MENU___ADD_MODEL_REFERENCE": "Model Definition", "CONTEXT_MENU___GO_TO_DEFINITION": "Go to Model Definition", "DOCUMENTATION___DB_DEFINITIONS": "Model Definitions", From ee3af05918fa939b1415bf242e0cb81af7a4a27e Mon Sep 17 00:00:00 2001 From: Vitalii Yarmus Date: Fri, 16 Apr 2021 15:51:52 +0300 Subject: [PATCH 09/22] Renamed doc property to description, removed description property from field and entity levels, added adapter to replace doc by description --- adapter/0.1.60.json | 60 +++++++++++++++++++ forward_engineering/api.js | 20 ++++++- .../entity_level/entityLevelConfig.json | 10 +--- .../field_level/fieldLevelConfig.json | 33 ++++------ reverse_engineering/api.js | 5 ++ 5 files changed, 94 insertions(+), 34 deletions(-) create mode 100644 adapter/0.1.60.json diff --git a/adapter/0.1.60.json b/adapter/0.1.60.json new file mode 100644 index 0000000..38a8bba --- /dev/null +++ b/adapter/0.1.60.json @@ -0,0 +1,60 @@ +/** + * Copyright © 2016-2021 by IntegrIT S.A. dba Hackolade. All rights reserved. + * + * The copyright to the computer software herein is the property of IntegrIT S.A. + * The software may be used and/or copied only with the written permission of + * IntegrIT S.A. or in accordance with the terms and conditions stipulated in + * the agreement/contract under which the software has been supplied. + * + * { + * "add": { + * "entity": [], + * "container": [], + * "model": [], + * "view": [], + * "field": { + * "": [] + * } + * }, + * "delete": { + * "entity": [], + * "container": [], + * "model": [], + * "view": [], + * "field": { + * "": [] + * } + * }, + * "modify": { + * "entity": [ + * { + * "from": { }, + * "to": { } + * } + * ], + * "container": [], + * "model": [], + * "view": [], + * "field": [] + * }, + * } + */ +{ + "add": {}, + "modify": { + "field": [ + "convertDocToDescription" + ], + "entity": [ + "convertDocToDescription" + ] + }, + "delete": { + "field": [ + "doc" + ], + "entity": [ + "doc" + ] + } +} \ No newline at end of file diff --git a/forward_engineering/api.js b/forward_engineering/api.js index c6ba6cd..1cae8dd 100644 --- a/forward_engineering/api.js +++ b/forward_engineering/api.js @@ -6,7 +6,7 @@ const _ = require('lodash'); const validationHelper = require('./validationHelper'); const mapJsonSchema = require('../reverse_engineering/helpers/mapJsonSchema'); -const ADDITIONAL_PROPS = ['doc', 'order', 'aliases', 'symbols', 'namespace', 'size', 'durationSize', 'default', 'precision', 'scale']; +const ADDITIONAL_PROPS = ['description', 'order', 'aliases', 'symbols', 'namespace', 'size', 'durationSize', 'default', 'precision', 'scale']; const ADDITIONAL_CHOICE_META_PROPS = ADDITIONAL_PROPS.concat('index'); const PRIMITIVE_FIELD_ATTRIBUTES = ['order', 'logicalType', 'precision', 'scale', 'aliases']; const DEFAULT_TYPE = 'string'; @@ -454,7 +454,7 @@ const handleType = (schema, avroSchema, udt) => { }; const handleMultiple = (avroSchema, schema, prop, udt) => { - const commonAttributes = ["aliases", "doc", "default"]; + const commonAttributes = ["aliases", "description", "default"]; avroSchema[prop] = schema[prop].map(type => { if (type && typeof type === 'object') { return type.type; @@ -493,6 +493,13 @@ const handleMultiple = (avroSchema, schema, prop, udt) => { const fieldProperties = commonAttributes.reduce((fieldProps, prop) => { if (schema[prop]) { + if(prop === 'description') { + return { + ...fieldProps, + doc: schema[prop], + } + } + return Object.assign({}, fieldProps, { [prop]: schema[prop] }); @@ -731,6 +738,13 @@ const handleOtherProps = (schema, prop, avroSchema, udt) => { if (!allowedProperties.includes(prop)) { return; } + + if(prop === 'description') { + avroSchema.doc = schema[prop]; + + return; + } + avroSchema[prop] = schema[prop]; if (prop === 'size' || prop === 'durationSize') { @@ -871,7 +885,7 @@ const getAllowedPropertyNames = (type, data, udt) => { return getAllowedPropertyNames(_.get(udt[type], 'type'), data, udt); } if(type === 'root') { - return ['aliases', 'doc']; + return ['aliases', 'description']; } if (!fieldLevelConfig.structure[type]) { return []; diff --git a/properties_pane/entity_level/entityLevelConfig.json b/properties_pane/entity_level/entityLevelConfig.json index fa2094e..c637f6b 100644 --- a/properties_pane/entity_level/entityLevelConfig.json +++ b/properties_pane/entity_level/entityLevelConfig.json @@ -127,7 +127,7 @@ making sure that you maintain a proper JSON format. }, { "propertyName": "Doc", - "propertyKeyword": "doc", + "propertyKeyword": "description", "propertyValidate": false, "propertyTooltip": "Popup for multi-line text entry", "propertyType": "details", @@ -147,14 +147,6 @@ making sure that you maintain a proper JSON format. "propertyType": "checkbox", "template": "boolean" }, - { - "propertyName": "Description", - "propertyKeyword": "description", - "propertyValidate": false, - "propertyTooltip": "Description", - "propertyType": "details", - "template": "textarea" - }, { "propertyName": "Comments", "propertyKeyword": "comments", diff --git a/properties_pane/field_level/fieldLevelConfig.json b/properties_pane/field_level/fieldLevelConfig.json index 749e969..95eb724 100644 --- a/properties_pane/field_level/fieldLevelConfig.json +++ b/properties_pane/field_level/fieldLevelConfig.json @@ -120,10 +120,9 @@ making sure that you maintain a proper JSON format. "string": [ "name", "schemaId", - "description", { "propertyName": "Doc", - "propertyKeyword": "doc", + "propertyKeyword": "description", "propertyValidate": false, "propertyTooltip": "Popup for multi-line text entry", "propertyType": "details", @@ -286,10 +285,9 @@ making sure that you maintain a proper JSON format. "bytes": [ "name", "schemaId", - "description", { "propertyName": "Doc", - "propertyKeyword": "doc", + "propertyKeyword": "description", "propertyValidate": false, "propertyTooltip": "Popup for multi-line text entry", "propertyType": "details", @@ -425,10 +423,9 @@ making sure that you maintain a proper JSON format. "number": [ "name", "schemaId", - "description", { "propertyName": "Doc", - "propertyKeyword": "doc", + "propertyKeyword": "description", "propertyValidate": false, "propertyTooltip": "Popup for multi-line text entry", "propertyType": "details", @@ -644,10 +641,9 @@ making sure that you maintain a proper JSON format. "boolean": [ "name", "schemaId", - "description", { "propertyName": "Doc", - "propertyKeyword": "doc", + "propertyKeyword": "description", "propertyValidate": false, "propertyTooltip": "Popup for multi-line text entry", "propertyType": "details", @@ -785,10 +781,9 @@ making sure that you maintain a proper JSON format. "null": [ "name", "schemaId", - "description", { "propertyName": "Doc", - "propertyKeyword": "doc", + "propertyKeyword": "description", "propertyValidate": false, "propertyTooltip": "Popup for multi-line text entry", "propertyType": "details", @@ -843,10 +838,9 @@ making sure that you maintain a proper JSON format. "name", "code", "schemaId", - "description", { "propertyName": "Doc", - "propertyKeyword": "doc", + "propertyKeyword": "description", "propertyValidate": false, "propertyTooltip": "Popup for multi-line text entry", "propertyType": "details", @@ -962,10 +956,9 @@ making sure that you maintain a proper JSON format. "name", "code", "schemaId", - "description", { "propertyName": "Doc", - "propertyKeyword": "doc", + "propertyKeyword": "description", "propertyValidate": false, "propertyTooltip": "Popup for multi-line text entry", "propertyType": "details", @@ -1069,10 +1062,9 @@ making sure that you maintain a proper JSON format. "name", "code", "schemaId", - "description", { "propertyName": "Doc", - "propertyKeyword": "doc", + "propertyKeyword": "description", "propertyValidate": false, "propertyTooltip": "Popup for multi-line text entry", "propertyType": "details", @@ -1191,10 +1183,9 @@ making sure that you maintain a proper JSON format. "name", "code", "schemaId", - "description", { "propertyName": "Doc", - "propertyKeyword": "doc", + "propertyKeyword": "description", "propertyValidate": false, "propertyTooltip": "Popup for multi-line text entry", "propertyType": "details", @@ -1360,10 +1351,9 @@ making sure that you maintain a proper JSON format. "name", "code", "schemaId", - "description", { "propertyName": "Doc", - "propertyKeyword": "doc", + "propertyKeyword": "description", "propertyValidate": false, "propertyTooltip": "Popup for multi-line text entry", "propertyType": "details", @@ -1541,10 +1531,9 @@ making sure that you maintain a proper JSON format. "choice": [ "name", "code", - "description", { "propertyName": "Doc", - "propertyKeyword": "doc", + "propertyKeyword": "description", "propertyValidate": false, "propertyTooltip": "Popup for multi-line text entry", "propertyType": "details", diff --git a/reverse_engineering/api.js b/reverse_engineering/api.js index b3d9e97..d661914 100644 --- a/reverse_engineering/api.js +++ b/reverse_engineering/api.js @@ -493,6 +493,11 @@ const handleOtherProps = (data, prop, schema) => { if (!ADDITIONAL_PROPS.includes(prop)) { return; } + if (prop === 'doc') { + schema.description = data[prop]; + + return; + } if (prop === 'logicalType' && (data.type === 'bytes' || data.type === 'fixed')) { schema['subtype'] = data.prop; } From 75d154a366f68b03198ce5f526b4ba47fb73c301 Mon Sep 17 00:00:00 2001 From: Kostya Tsymbal Date: Fri, 16 Apr 2021 17:07:48 +0300 Subject: [PATCH 10/22] Fixed Apply to instance connection modal --- reverse_engineering/config.json | 2 ++ 1 file changed, 2 insertions(+) diff --git a/reverse_engineering/config.json b/reverse_engineering/config.json index f64df43..c3352af 100644 --- a/reverse_engineering/config.json +++ b/reverse_engineering/config.json @@ -2,6 +2,8 @@ "menu_label": "Avro schema or data file", "name": "avro", "extensions": ["avsc", "avro", "confluent-avro"], + "connectionList": [ "name", "connectionType", "path" ], + "widestColumn": "path", "useCloudStorage": true, "useMultipleFileReverse": true, "useConfluent": true From c030b489ca63dfd8ef3306524f80aa621b0e428a Mon Sep 17 00:00:00 2001 From: Kostya Tsymbal Date: Fri, 16 Apr 2021 19:09:52 +0300 Subject: [PATCH 11/22] Disabled on model level --- forward_engineering/config.json | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/forward_engineering/config.json b/forward_engineering/config.json index aa02146..88dd9d6 100644 --- a/forward_engineering/config.json +++ b/forward_engineering/config.json @@ -31,17 +31,6 @@ } ], "level": { - "entity": true, - "model": { - "validation": false, - "options": [ - { - "name": "Confluent API Schema", - "keyword": "confluentSchemaRegistry", - "mode": "json", - "isApplicationHandler": true - } - ] - } + "entity": true } } From 42a8e585123f406c811a7d05dd5b462ad791a952 Mon Sep 17 00:00:00 2001 From: Kostya Tsymbal Date: Fri, 16 Apr 2021 19:20:51 +0300 Subject: [PATCH 12/22] Fixed FE from terminal --- forward_engineering/api.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/forward_engineering/api.js b/forward_engineering/api.js index 9b287af..31d55a1 100644 --- a/forward_engineering/api.js +++ b/forward_engineering/api.js @@ -80,7 +80,7 @@ module.exports = { } let avroSchema = JSON.parse(targetScript); - if (data.targetScriptOptions.keyword !== 'avroSchema') { + if (data.targetScriptOptions.keyword === 'confluentSchemaRegistry' || data.targetScriptOptions.keyword === 'schemaRegistry') { const messages = validationHelper.validate(avroSchema.schema); return cb(null, messages); } From be7dcce4c2b8b2d376d091396f1fe5ed1f5b12b1 Mon Sep 17 00:00:00 2001 From: Pascal Date: Fri, 16 Apr 2021 19:14:55 +0200 Subject: [PATCH 13/22] support for SSL and forward-engineering --- package.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/package.json b/package.json index 6be3b22..998ab9a 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "Avro", - "version": "0.1.59", - "versionDate": "2021-04-09", + "version": "0.1.60", + "versionDate": "2021-04-16", "author": "hackolade", "engines": { "hackolade": "4.3.16", From 604842ac7df206b3ba8ac29338d3c17951f56a0c Mon Sep 17 00:00:00 2001 From: Kostya Date: Tue, 20 Apr 2021 15:29:42 +0300 Subject: [PATCH 14/22] Revert "Disabled on model level" This reverts commit c030b489ca63dfd8ef3306524f80aa621b0e428a. --- forward_engineering/config.json | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/forward_engineering/config.json b/forward_engineering/config.json index 88dd9d6..aa02146 100644 --- a/forward_engineering/config.json +++ b/forward_engineering/config.json @@ -31,6 +31,17 @@ } ], "level": { - "entity": true + "entity": true, + "model": { + "validation": false, + "options": [ + { + "name": "Confluent API Schema", + "keyword": "confluentSchemaRegistry", + "mode": "json", + "isApplicationHandler": true + } + ] + } } } From 34047121ab67eeb74cee10bbde97299af9230c79 Mon Sep 17 00:00:00 2001 From: Kostya Date: Wed, 21 Apr 2021 00:20:53 +0300 Subject: [PATCH 15/22] Added model level --- forward_engineering/config.json | 1 + 1 file changed, 1 insertion(+) diff --git a/forward_engineering/config.json b/forward_engineering/config.json index aa02146..6c6a52f 100644 --- a/forward_engineering/config.json +++ b/forward_engineering/config.json @@ -34,6 +34,7 @@ "entity": true, "model": { "validation": false, + "isFEToFileEnabled":false, "options": [ { "name": "Confluent API Schema", From 8084b7b6e14cdfeeb80ff86d6e0a9ca50f98622a Mon Sep 17 00:00:00 2001 From: Kostya Date: Wed, 21 Apr 2021 15:08:32 +0300 Subject: [PATCH 16/22] Add minify --- forward_engineering/api.js | 7 +++++-- forward_engineering/config.json | 3 +-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/forward_engineering/api.js b/forward_engineering/api.js index 31d55a1..bdf12c4 100644 --- a/forward_engineering/api.js +++ b/forward_engineering/api.js @@ -105,6 +105,7 @@ const getCommonEntitiesData = (data) => { targetScriptOptions: { keyword: "confluentSchemaRegistry", }, + additionalOptions:data.options.additionalOptions }; return { options, modelDefinitions, externalDefinitions } @@ -143,11 +144,13 @@ const getScript = (data) => { return JSON.stringify({ schema: JSON.stringify(avroSchema) }, null, 4); } + const needMinify = (additionalOptions.find(option => option.id === 'minify') || {}).value; if (targetScriptType === 'confluentSchemaRegistry') { - return `POST /subjects/${name}/versions\n${JSON.stringify({ schemaType: "AVRO", schema: JSON.stringify(avroSchema) }, null, 4)}` + const schema = needMinify?JSON.stringify(avroSchema):avroSchema; + + return `POST /subjects/${name}/versions\n${JSON.stringify({ schema, schemaType: "AVRO" }, null, 4)}` } - const needMinify = targetScriptType !== 'confluentSchemaRegistry' && (additionalOptions.find(option => option.id === 'minify') || {}).value; if (needMinify) { return JSON.stringify(avroSchema); } diff --git a/forward_engineering/config.json b/forward_engineering/config.json index 6c6a52f..e1b8d39 100644 --- a/forward_engineering/config.json +++ b/forward_engineering/config.json @@ -26,8 +26,7 @@ { "id": "minify", "value": false, - "name": "Minify", - "targetFEKeywords": ["avroSchema", "schemaRegistry"] + "name": "Minify" } ], "level": { From d11aab61faf644582e07bc99fabdca036d93853e Mon Sep 17 00:00:00 2001 From: Kostya Date: Wed, 21 Apr 2021 15:26:11 +0300 Subject: [PATCH 17/22] Add minify --- forward_engineering/api.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/forward_engineering/api.js b/forward_engineering/api.js index bdf12c4..90e7362 100644 --- a/forward_engineering/api.js +++ b/forward_engineering/api.js @@ -105,7 +105,7 @@ const getCommonEntitiesData = (data) => { targetScriptOptions: { keyword: "confluentSchemaRegistry", }, - additionalOptions:data.options.additionalOptions + additionalOptions: data.options.additionalOptions }; return { options, modelDefinitions, externalDefinitions } From f358858fafb922f2356bd50e95c4bf1d23e0b718 Mon Sep 17 00:00:00 2001 From: Kostya Date: Wed, 21 Apr 2021 18:03:16 +0300 Subject: [PATCH 18/22] Enhanced FE config --- forward_engineering/config.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/forward_engineering/config.json b/forward_engineering/config.json index e1b8d39..8523dba 100644 --- a/forward_engineering/config.json +++ b/forward_engineering/config.json @@ -33,7 +33,7 @@ "entity": true, "model": { "validation": false, - "isFEToFileEnabled":false, + "file": false, "options": [ { "name": "Confluent API Schema", From 0a68bd51019d3854bd3daf5de38f0bebe1c2557f Mon Sep 17 00:00:00 2001 From: Pascal Date: Mon, 26 Apr 2021 21:46:35 +0200 Subject: [PATCH 19/22] Confluent: fetch versions --- package.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/package.json b/package.json index 998ab9a..83883c8 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "Avro", - "version": "0.1.60", - "versionDate": "2021-04-16", + "version": "0.1.61", + "versionDate": "2021-04-26", "author": "hackolade", "engines": { "hackolade": "4.3.16", From ab48fac4ed4366f2499fffb3511793e97eee5d78 Mon Sep 17 00:00:00 2001 From: Vitalii Yarmus Date: Fri, 30 Apr 2021 14:00:10 +0300 Subject: [PATCH 20/22] Doc property in multiple data type --- properties_pane/field_level/fieldLevelConfig.json | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/properties_pane/field_level/fieldLevelConfig.json b/properties_pane/field_level/fieldLevelConfig.json index 95eb724..fdafa8f 100644 --- a/properties_pane/field_level/fieldLevelConfig.json +++ b/properties_pane/field_level/fieldLevelConfig.json @@ -1585,6 +1585,14 @@ making sure that you maintain a proper JSON format. "exist": false }] }, + { + "propertyName": "Doc", + "propertyKeyword": "description", + "propertyValidate": false, + "propertyTooltip": "Popup for multi-line text entry", + "propertyType": "details", + "template": "textarea" + }, "dynamic" ] } From 9775c54d64ba188f9fbd7c111f2bdd8a53a6ef0f Mon Sep 17 00:00:00 2001 From: Pascal Date: Fri, 30 Apr 2021 13:31:10 +0200 Subject: [PATCH 21/22] Doc property in multiple data type --- package.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/package.json b/package.json index 83883c8..bd7eaaa 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "Avro", - "version": "0.1.61", - "versionDate": "2021-04-26", + "version": "0.1.62", + "versionDate": "2021-04-30", "author": "hackolade", "engines": { "hackolade": "4.3.16", From 6d161c05a0df3a5a150c576217938ebcae36613d Mon Sep 17 00:00:00 2001 From: Mikhail <44769346+mtseluiko@users.noreply.github.com> Date: Fri, 30 Apr 2021 14:33:35 +0300 Subject: [PATCH 22/22] JSON RE: remove choice if there is only one record at the root level (#22) Co-authored-by: Pascal Desmarets