Merge pull request #2 from hackolade/master

 Sync

README.md

@@ -1,8 +1,8 @@
 # Avro
-Plugin to enable Avro as a target in Hackolade data modeling.
+Plugin to enable Avro as a target in [Hackolade](https://hackolade.com) data modeling.  Requires prior download of the Hackolade application from our [download page](https://hackolade.com/download.html)
 
 Hackolade exposes its core data modeling engine through a plugin architecture.  Each plugin applies the Hackolade data modeling capabilities to a specific target technology, whether for data-at-rest (databases) or data-in-motion (communications.)  Each plugin matches the specific aspects of the target in terms of terminology, storage model, data types, and communication protocol.
 
-To enable data modeling capabilities for a target, you must first download and install the plugin, following these [instructions](https://hackolade.com/help/DownloadadditionalDBtargetplugin.htm "Plugin download instructions").
+To enable data modeling capabilities for a target, you must first download and install the plugin, following these [instructions](https://hackolade.com/help/DownloadadditionalDBtargetplugin.html "Plugin download instructions").
 
 Plugins can be customized by following these [instructions](https://hackolade.com/help/Userdefinedcustomproperties.html "Plugin customization instructions").

adapter/0.1.60.json

@@ -0,0 +1,60 @@
+/**
+ * Copyright © 2016-2021 by IntegrIT S.A. dba Hackolade.  All rights reserved.
+ *
+ * The copyright to the computer software herein is the property of IntegrIT S.A.
+ * The software may be used and/or copied only with the written permission of 
+ * IntegrIT S.A. or in accordance with the terms and conditions stipulated in 
+ * the agreement/contract under which the software has been supplied. 
+ * 
+ * {
+ * 		"add": {
+ * 			"entity": [<names of new property>],
+ * 			"container": [<names of new property>],
+ * 			"model": [<names of new property>],
+ * 			"view": [<names of new property>],
+ *			"field": {
+ *				"<type>": [<names of new property>]
+ *			}
+ * 		},
+ * 		"delete": {
+ * 			"entity": [<names of new property>],
+ * 			"container": [<names of new property>],
+ * 			"model": [<names of new property>],
+ * 			"view": [<names of new property>],
+ *			"field": {
+ *				"<type>": [<names of new property>]
+ *			}
+ * 		},
+ * 		"modify": {
+ *	 		"entity": [
+ *	 			{
+ *					"from": { <properties that identify record> },
+ *					"to": { <properties that need to be changed> }
+ *				}
+ *			],
+ *			"container": [],
+ *			"model": [],
+ *			"view": [],
+ *			"field": []
+ * 		},
+ * }
+ */
+{
+    "add": {},
+    "modify": {
+        "field": [
+            "convertDocToDescription"
+        ],
+        "entity": [
+            "convertDocToDescription"
+        ]
+    },
+    "delete": {
+        "field": [
+            "doc"
+        ],
+        "entity": [
+            "doc"
+        ]
+    }
+}

forward_engineering/api.js

@@ -6,7 +6,7 @@ const _ = require('lodash');
 const validationHelper = require('./validationHelper');
 const mapJsonSchema = require('../reverse_engineering/helpers/mapJsonSchema');
 
-const ADDITIONAL_PROPS = ['doc', 'order', 'aliases', 'symbols', 'namespace', 'size', 'durationSize', 'default', 'precision', 'scale'];
+const ADDITIONAL_PROPS = ['description', 'order', 'aliases', 'symbols', 'namespace', 'size', 'durationSize', 'default', 'precision', 'scale'];
 const ADDITIONAL_CHOICE_META_PROPS = ADDITIONAL_PROPS.concat('index');
 const PRIMITIVE_FIELD_ATTRIBUTES = ['order', 'logicalType', 'precision', 'scale', 'aliases'];
 const DEFAULT_TYPE = 'string';
@@ -34,53 +34,59 @@ const LOGICAL_TYPES_MAP = {
 };
 
 module.exports = {
-	generateScript(data, logger, cb) {
+	generateModelScript(data, logger, cb) {
 		logger.clear();
 		try {
-			const name = getRecordName(data);
-			let avroSchema = { name };
-			let jsonSchema = JSON.parse(data.jsonSchema);
-			const udt = getUserDefinedTypes(data);
-
-			jsonSchema.type = 'root';
-			handleRecursiveSchema(jsonSchema, avroSchema, {}, udt);
-
-			if (data.containerData) {
-				avroSchema.namespace = data.containerData.name;
-			}
-			avroSchema.type = 'record';
-			avroSchema = reorderAvroSchema(avroSchema);
-			avroSchema = JSON.stringify(avroSchema, null, 4);
-			const options = data.options;
-			const additionalOptions = _.get(options, 'additionalOptions', []);
-			const targetScriptType = _.get(options, 'targetScriptOptions.keyword');
-			if (targetScriptType === 'schemaRegistry') {
-				avroSchema = JSON.stringify({ schema: JSON.stringify(JSON.parse(avroSchema))}, null, 4);
-			}
+			const commonData = getCommonEntitiesData(data);
+			const containers = _.get(data, 'containers', []);
+			const script = containers.reduce((createdQueries, container) => {
+				const containerEntities = container.entities.map(entityId => {
+					return Object.assign({}, commonData, getEntityData(container, entityId))
+				})
 
-			const needMinify = (additionalOptions.find(option => option.id === 'minify') || {}).value;
-			if (needMinify) {
-				avroSchema = JSON.stringify(JSON.parse(avroSchema));
-			}
+				const containerQueries = containerEntities.map(entity => {
+					try {
+						return getScript(entity)
+					} catch (e) {
+						logger.log('error', { message: err.message, stack: err.stack }, 'Avro Forward-Engineering Error');
+						return '';
+					}
+				})
 
-			nameIndex = 0;
-			return cb(null, avroSchema);
-		} catch(err) {
+				return [...createdQueries, ...containerQueries];
+			}, [])
+			cb(null, script.join('\n\n'));
+		} catch (err) {
+			logger.log('error', { message: err.message, stack: err.stack }, 'Avro model Forward-Engineering Error');
+			cb({ message: err.message, stack: err.stack });
+		}
+	},
+	generateScript(data, logger, cb) {
+		logger.clear();
+		try {
+			const script = getScript(data);
+			cb(null, script)
+		} catch (err) {
 			nameIndex = 0;
 			logger.log('error', { message: err.message, stack: err.stack }, 'Avro Forward-Engineering Error');
 			cb({ message: err.message, stack: err.stack });
 		}
 	},
 	validate(data, logger, cb) {
 		try {
-			let avroSchema = JSON.parse(data.script);
-			if (Object.keys(avroSchema).length === 1 && avroSchema.schema) {
+			let targetScript = data.script;
+			if (data.targetScriptOptions.keyword === 'confluentSchemaRegistry') {
+				targetScript = targetScript.split('\n').slice(1).join('\n')
+			}
+			let avroSchema = JSON.parse(targetScript);
+
+			if (data.targetScriptOptions.keyword === 'confluentSchemaRegistry' || data.targetScriptOptions.keyword === 'schemaRegistry') {
 				const messages = validationHelper.validate(avroSchema.schema);
-				cb(null, messages);
-			} else {
-				const messages = validationHelper.validate(data.script);
-				cb(null, messages);
+				return cb(null, messages);
 			}
+
+			const messages = validationHelper.validate(targetScript);
+			cb(null, messages);
 		} catch (e) {
 			logger.log('error', { error: e }, 'Avro Validation Error');
 			cb(null, [{
@@ -93,6 +99,65 @@ module.exports = {
 	}
 };
 
+const getCommonEntitiesData = (data) => {
+	const { modelDefinitions, externalDefinitions } = data;
+	const options = {
+		targetScriptOptions: {
+			keyword: "confluentSchemaRegistry",
+		},
+		additionalOptions: data.options.additionalOptions
+	};
+
+	return { options, modelDefinitions, externalDefinitions }
+}
+
+const getEntityData = (container, entityId) => {
+	const containerData = _.first(_.get(container, 'containerData', []));
+	const jsonSchema = container.jsonSchema[entityId];
+	const jsonData = container.jsonData[entityId];
+	const entityData = _.first(container.entityData[entityId]);
+	const internalDefinitions = container.internalDefinitions[entityId];
+
+	return { containerData, jsonSchema, jsonData, entityData, internalDefinitions }
+}
+
+const getScript = (data) => {
+	const name = getRecordName(data);
+	let avroSchema = { name };
+	let jsonSchema = JSON.parse(data.jsonSchema);
+	const udt = getUserDefinedTypes(data);
+
+	jsonSchema.type = 'root';
+	handleRecursiveSchema(jsonSchema, avroSchema, {}, udt);
+
+	if (data.containerData) {
+		avroSchema.namespace = data.containerData.name;
+	}
+	avroSchema.type = 'record';
+	avroSchema = reorderAvroSchema(avroSchema);
+	const options = data.options;
+	const additionalOptions = _.get(options, 'additionalOptions', []);
+	const targetScriptType = _.get(options, 'targetScriptOptions.keyword');
+	nameIndex = 0;
+
+	if (targetScriptType === 'schemaRegistry') {
+		return JSON.stringify({ schema: JSON.stringify(avroSchema) }, null, 4);
+	}
+
+	const needMinify = (additionalOptions.find(option => option.id === 'minify') || {}).value;
+	if (targetScriptType === 'confluentSchemaRegistry') {
+		const schema = needMinify?JSON.stringify(avroSchema):avroSchema;
+
+		return `POST /subjects/${name}/versions\n${JSON.stringify({ schema, schemaType: "AVRO" }, null, 4)}`
+	}
+
+	if (needMinify) {
+		return JSON.stringify(avroSchema);
+	}
+
+	return JSON.stringify(avroSchema, null, 4);
+}
+
 const getUserDefinedTypes = ({ internalDefinitions, externalDefinitions, modelDefinitions }) => {
 	let udt = convertSchemaToUserDefinedTypes(JSON.parse(externalDefinitions), {});
 	 udt = convertSchemaToUserDefinedTypes(JSON.parse(modelDefinitions), udt);
@@ -454,7 +519,7 @@ const handleType = (schema, avroSchema, udt) => {
 };
 
 const handleMultiple = (avroSchema, schema, prop, udt) => {
-	const commonAttributes = ["aliases", "doc", "default"];
+	const commonAttributes = ["aliases", "description", "default"];
 	avroSchema[prop] = schema[prop].map(type => {
 		if (type && typeof type === 'object') {
 			return type.type;
@@ -493,6 +558,13 @@ const handleMultiple = (avroSchema, schema, prop, udt) => {
 
 	const fieldProperties = commonAttributes.reduce((fieldProps, prop) => {
 		if (schema[prop]) {
+			if(prop === 'description') {
+				return {
+					...fieldProps,
+					doc: schema[prop],
+				}
+			}
+
 			return Object.assign({}, fieldProps, {
 				[prop]: schema[prop]
 			});
@@ -731,6 +803,13 @@ const handleOtherProps = (schema, prop, avroSchema, udt) => {
 	if (!allowedProperties.includes(prop)) {
 		return;
 	}
+
+	if(prop === 'description') {
+		avroSchema.doc = schema[prop];
+
+		return;
+	}
+
 	avroSchema[prop] = schema[prop];
 
 	if (prop === 'size' || prop === 'durationSize') {
@@ -871,7 +950,7 @@ const getAllowedPropertyNames = (type, data, udt) => {
 		return getAllowedPropertyNames(_.get(udt[type], 'type'), data, udt);
 	}
 	if(type === 'root') {
-		return ['aliases', 'doc'];
+		return ['aliases', 'description'];
 	}
 	if (!fieldLevelConfig.structure[type]) {
 		return [];

forward_engineering/config.json

@@ -1,18 +1,47 @@
 {
-	"extension": "avsc",
-	"filterName": "Avro schema",
-	"namePrefix": "Avro Schema",
-	"resolvedDefinitions": false,
-	"validation": true,
-	"options": [
-		{ "name": "Avro Schema", "keyword": "avroSchema"},
-		{ "name": "Schema registry", "keyword": "schemaRegistry"}
-	],
-	"additionalOptions": [
-		{
-			"id": "minify",
-			"value": false,
-			"name": "Minify"
-		}
-	]
+  "extension": "avsc",
+  "filterName": "Avro schema",
+  "namePrefix": "Avro Schema",
+  "resolvedDefinitions": false,
+  "validation": true,
+  "applyScriptToInstance": true,
+  "options": [
+    {
+      "name": "Avro Schema",
+      "keyword": "avroSchema",
+      "disableApplyScriptToInstance": true
+    },
+    {
+      "name": "Schema registry",
+      "keyword": "schemaRegistry",
+      "disableApplyScriptToInstance": false
+    },
+    {
+      "name": "Confluent API Schema",
+      "keyword": "confluentSchemaRegistry",
+      "isApplicationHandler": true
+    }
+  ],
+  "additionalOptions": [
+    {
+      "id": "minify",
+      "value": false,
+      "name": "Minify"
+    }
+  ],
+  "level": {
+    "entity": true,
+    "model": {
+      "validation": false,
+      "file": false,
+      "options": [
+        {
+          "name": "Confluent API Schema",
+          "keyword": "confluentSchemaRegistry",
+          "mode": "json",
+          "isApplicationHandler": true
+        }
+      ]
+    }
+  }
 }

localization/en.json

@@ -137,6 +137,7 @@
 	"TYPE": {},
 	"CENTRAL_PANE___TAB_MODEL_DEFINITIONS": "Model Definitions",
 	"CENTRAL_PANE___FE_SCRIPT": "Avro Schema",
+	"CENTRAL_PANE___MODEL_FE_SCRIPT": "Confluent API",
 	"CONTEXT_MENU___ADD_MODEL_REFERENCE": "Model Definition",
 	"CONTEXT_MENU___GO_TO_DEFINITION": "Go to Model Definition",
 	"DOCUMENTATION___DB_DEFINITIONS": "Model Definitions",

package.json

@@ -1,10 +1,10 @@
 {
     "name": "Avro",
-    "version": "0.1.58",
-    "versionDate": "2021-02-13",
+    "version": "0.1.62",
+    "versionDate": "2021-04-30",
     "author": "hackolade",
     "engines": {
-        "hackolade": "3.3.2",
+        "hackolade": "4.3.16",
         "hackoladePlugin": "1.0.0"
     },
     "contributes": {

properties_pane/entity_level/entityLevelConfig.json

@@ -127,7 +127,7 @@ making sure that you maintain a proper JSON format.
 			},
 			{
 				"propertyName": "Doc",
-				"propertyKeyword": "doc",
+				"propertyKeyword": "description",
 				"propertyValidate": false,
 				"propertyTooltip": "Popup for multi-line text entry",
 				"propertyType": "details",
@@ -147,14 +147,6 @@ making sure that you maintain a proper JSON format.
 				"propertyType": "checkbox",
 				"template": "boolean"
 			},
-			{
-				"propertyName": "Description",
-				"propertyKeyword": "description",
-				"propertyValidate": false,
-				"propertyTooltip": "Description",
-				"propertyType": "details",
-				"template": "textarea"
-			},
 			{
 				"propertyName": "Comments",
 				"propertyKeyword": "comments",