Use non-null sample in JSON Data

hackolade · Sep 21, 2019 · 8182501 · 8182501
1 parent 9c32ec5
commit 8182501
Show file tree

Hide file tree

Showing 5 changed files with 101 additions and 57 deletions.
diff --git a/central_pane/style.json b/central_pane/style.json
@@ -27,44 +27,4 @@
 }
  */
 
-{
-	"field": {
-		"dtd": [{
-			"value": {
-				"border-style": "dotted",
-				"background": "#fff5e4"
-			},
-			"dependency": {
-				"key": "nullAllowed",
-				"value": true
-			}
-		}],
-		"erd": [ "keys", "type", [{
-			"value": "+",
-			"dependency": {
-				"key": "nullAllowed",
-				"value": true
-			},
-			"width": 20
-		}, {
-			"value": "*",
-			"dependency": {
-				"type": "and",
-				"values": [{
-					"key": "required",
-					"value": true
-				}, {
-					"type": "or",
-					"values": [{
-						"key": "nullAllowed",
-						"value": false
-					}, {
-						"key": "nullAllowed",
-						"exist": false
-					}]
-				}]
-			},
-			"width": 20
-		}], "indexes" ]
-	}
-}
+{}
diff --git a/package.json b/package.json
@@ -1,7 +1,7 @@
 {
     "name": "Avro",
-    "version": "0.1.26",
-    "versionDate": "2019-09-16",
+    "version": "0.1.27",
+    "versionDate": "2019-09-21",
     "author": "hackolade",
     "engines": {
         "hackolade": "3.3.2",

diff --git a/reverse_engineering/api.js b/reverse_engineering/api.js
@@ -5,7 +5,7 @@ const path = require('path');
 const _ = require('lodash');
 const avro = require('avsc');
 const snappy = require('snappyjs');
-const adaptJsonSchema = require('./helpers/adaptJsonSchema');
+const jsonSchemaAdapter = require('./helpers/adaptJsonSchema');
 const DEFAULT_FIELD_NAME = 'New_field';
 let stateExtension = null;
 
@@ -64,13 +64,14 @@ module.exports = {
 		logger.log('info', 'Adaptation of JSON Schema started...', 'Adapt JSON Schema');
 		try {
 			const jsonSchema = JSON.parse(data.jsonSchema);
-
-			const adaptedJsonSchema = adaptJsonSchema(jsonSchema);
+			const adaptedJsonSchema = jsonSchemaAdapter.adaptJsonSchema(jsonSchema);
+			const jsonSchemaName = jsonSchemaAdapter.adaptJsonSchemaName(data.jsonSchemaName);
 
 			logger.log('info', 'Adaptation of JSON Schema finished.', 'Adapt JSON Schema');
 
 			callback(null, {
-				jsonSchema: JSON.stringify(adaptedJsonSchema)
+				jsonSchema: JSON.stringify(adaptedJsonSchema),
+				jsonSchemaName
 			});
 		} catch(error) {
 			const formattedError = formatError(error);

diff --git a/reverse_engineering/helpers/adaptJsonSchema.js b/reverse_engineering/helpers/adaptJsonSchema.js
@@ -73,7 +73,7 @@ const adaptMultiple = field => {
 };
 
 const handleEmptyDefault = field => {
-	const typesWithoutDefault = ['bytes', 'fixed', 'record', 'array', 'map', 'null'];
+	const typesWithoutDefault = ['bytes', 'fixed', 'object', 'record', 'array', 'map', 'null'];
 	const hasDefault = !_.isUndefined(field.default) && field.default !== '';
 	const isMultiple = _.isArray(field.types);
 	if (isMultiple && field.types.every(type => typesWithoutDefault.includes(type))) {
@@ -157,14 +157,97 @@ const populateDefaultNullValuesForMultiple = field => {
 	return Object.assign({}, field, { default: null });
 };
 
-const adaptJsonSchema = jsonSchema => {
-	return mapJsonSchema(jsonSchema, jsonSchemaItem => {
-		return _.flow([
-			adaptType,
-			populateDefaultNullValuesForMultiple,
-			handleEmptyDefaultInProperties
-		])(jsonSchemaItem);
+const adaptTitle = jsonSchema => {
+	if (!jsonSchema.title) {
+		return jsonSchema;
+	}
+
+	return Object.assign({}, jsonSchema, {
+		title: convertToValidAvroName(jsonSchema.title)
 	});
 };
 
-module.exports = adaptJsonSchema;
+const adaptRequiredNames = jsonSchema => {
+	if (!_.isArray(jsonSchema.required)) {
+		return jsonSchema;
+	}
+
+	return Object.assign({}, jsonSchema, {
+		required: jsonSchema.required.map(convertToValidAvroName)
+	});
+};
+
+const adaptPropertiesNames = jsonSchema => {
+	if (!_.isPlainObject(jsonSchema)) {
+		return jsonSchema;
+	}
+
+	const propertiesKeys = [ 'properties', 'definitions', 'patternProperties' ];
+
+	const adaptedSchema = adaptRequiredNames(jsonSchema);
+
+	return propertiesKeys.reduce((schema, propertyKey) => {
+		const properties = schema[propertyKey];
+		if (_.isEmpty(properties)) {
+			return schema;
+		}
+
+		const adaptedProperties = Object.keys(properties).reduce((adaptedProperties, key) => {
+			if (key === '$ref') {
+				return Object.assign({}, adaptedProperties, {
+					[key]: convertReferenceName(properties[key])
+				})
+			}
+
+			const updatedKey = convertToValidAvroName(key);
+			const adaptedProperty = adaptPropertiesNames(properties[key]);
+
+			return Object.assign({}, adaptedProperties, {
+				[updatedKey]: adaptedProperty
+			});
+		}, {});
+
+		return Object.assign({}, schema, {
+			[propertyKey]: adaptedProperties
+		});
+	}, adaptedSchema);
+};
+
+const adaptNames = _.flow([
+	adaptTitle,
+	adaptPropertiesNames
+]);
+
+const convertReferenceName = ref => {
+	if (!_.isString(ref)) {
+		return ref;
+	}
+
+	const refNames = ref.split('/');
+	const referenceName = _.last(refNames);
+	const adaptedReferenceName = convertToValidAvroName(referenceName);
+
+	return refNames.slice(0, -1).concat(adaptedReferenceName).join('/');
+};
+
+const convertToValidAvroName = name => {
+	if (!_.isString(name)) {
+		return name;
+	}
+
+	return name.replace(/[^A-Za-z0-9_]/g, '_');
+};
+
+const adaptJsonSchema = jsonSchema => {
+	const adaptedJsonSchema = adaptNames(jsonSchema);
+
+	return mapJsonSchema(adaptedJsonSchema, _.flow([
+		adaptType,
+		populateDefaultNullValuesForMultiple,
+		handleEmptyDefaultInProperties
+	]));
+};
+
+const adaptJsonSchemaName = convertToValidAvroName;
+
+module.exports = { adaptJsonSchema, adaptJsonSchemaName };
diff --git a/types/null.json b/types/null.json
@@ -3,7 +3,7 @@
 	"name": "null",
 	"erdAbbreviation": "<null>",
 	"dtdAbbreviation": "{null}",
-	"useSample": false,
+	"useSample": true,
 	"defaultValues": {
 		"default": null,
 		"type": "null",