From 8182501dc1c8e1c35cfd494e163914c4695d51d7 Mon Sep 17 00:00:00 2001 From: Pascal Desmarets Date: Sat, 21 Sep 2019 10:15:35 +0200 Subject: [PATCH] Use non-null sample in JSON Data --- central_pane/style.json | 42 +------- package.json | 4 +- reverse_engineering/api.js | 9 +- .../helpers/adaptJsonSchema.js | 101 ++++++++++++++++-- types/null.json | 2 +- 5 files changed, 101 insertions(+), 57 deletions(-) diff --git a/central_pane/style.json b/central_pane/style.json index 8bc7e68..2e06044 100644 --- a/central_pane/style.json +++ b/central_pane/style.json @@ -27,44 +27,4 @@ } */ -{ - "field": { - "dtd": [{ - "value": { - "border-style": "dotted", - "background": "#fff5e4" - }, - "dependency": { - "key": "nullAllowed", - "value": true - } - }], - "erd": [ "keys", "type", [{ - "value": "+", - "dependency": { - "key": "nullAllowed", - "value": true - }, - "width": 20 - }, { - "value": "*", - "dependency": { - "type": "and", - "values": [{ - "key": "required", - "value": true - }, { - "type": "or", - "values": [{ - "key": "nullAllowed", - "value": false - }, { - "key": "nullAllowed", - "exist": false - }] - }] - }, - "width": 20 - }], "indexes" ] - } -} \ No newline at end of file +{} \ No newline at end of file diff --git a/package.json b/package.json index 9200188..1946497 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "Avro", - "version": "0.1.26", - "versionDate": "2019-09-16", + "version": "0.1.27", + "versionDate": "2019-09-21", "author": "hackolade", "engines": { "hackolade": "3.3.2", diff --git a/reverse_engineering/api.js b/reverse_engineering/api.js index e5fb2b0..fc58812 100644 --- a/reverse_engineering/api.js +++ b/reverse_engineering/api.js @@ -5,7 +5,7 @@ const path = require('path'); const _ = require('lodash'); const avro = require('avsc'); const snappy = require('snappyjs'); -const adaptJsonSchema = require('./helpers/adaptJsonSchema'); +const jsonSchemaAdapter = require('./helpers/adaptJsonSchema'); const DEFAULT_FIELD_NAME = 'New_field'; let stateExtension = null; @@ -64,13 +64,14 @@ module.exports = { logger.log('info', 'Adaptation of JSON Schema started...', 'Adapt JSON Schema'); try { const jsonSchema = JSON.parse(data.jsonSchema); - - const adaptedJsonSchema = adaptJsonSchema(jsonSchema); + const adaptedJsonSchema = jsonSchemaAdapter.adaptJsonSchema(jsonSchema); + const jsonSchemaName = jsonSchemaAdapter.adaptJsonSchemaName(data.jsonSchemaName); logger.log('info', 'Adaptation of JSON Schema finished.', 'Adapt JSON Schema'); callback(null, { - jsonSchema: JSON.stringify(adaptedJsonSchema) + jsonSchema: JSON.stringify(adaptedJsonSchema), + jsonSchemaName }); } catch(error) { const formattedError = formatError(error); diff --git a/reverse_engineering/helpers/adaptJsonSchema.js b/reverse_engineering/helpers/adaptJsonSchema.js index cf36f73..bc0c761 100644 --- a/reverse_engineering/helpers/adaptJsonSchema.js +++ b/reverse_engineering/helpers/adaptJsonSchema.js @@ -73,7 +73,7 @@ const adaptMultiple = field => { }; const handleEmptyDefault = field => { - const typesWithoutDefault = ['bytes', 'fixed', 'record', 'array', 'map', 'null']; + const typesWithoutDefault = ['bytes', 'fixed', 'object', 'record', 'array', 'map', 'null']; const hasDefault = !_.isUndefined(field.default) && field.default !== ''; const isMultiple = _.isArray(field.types); if (isMultiple && field.types.every(type => typesWithoutDefault.includes(type))) { @@ -157,14 +157,97 @@ const populateDefaultNullValuesForMultiple = field => { return Object.assign({}, field, { default: null }); }; -const adaptJsonSchema = jsonSchema => { - return mapJsonSchema(jsonSchema, jsonSchemaItem => { - return _.flow([ - adaptType, - populateDefaultNullValuesForMultiple, - handleEmptyDefaultInProperties - ])(jsonSchemaItem); +const adaptTitle = jsonSchema => { + if (!jsonSchema.title) { + return jsonSchema; + } + + return Object.assign({}, jsonSchema, { + title: convertToValidAvroName(jsonSchema.title) }); }; -module.exports = adaptJsonSchema; +const adaptRequiredNames = jsonSchema => { + if (!_.isArray(jsonSchema.required)) { + return jsonSchema; + } + + return Object.assign({}, jsonSchema, { + required: jsonSchema.required.map(convertToValidAvroName) + }); +}; + +const adaptPropertiesNames = jsonSchema => { + if (!_.isPlainObject(jsonSchema)) { + return jsonSchema; + } + + const propertiesKeys = [ 'properties', 'definitions', 'patternProperties' ]; + + const adaptedSchema = adaptRequiredNames(jsonSchema); + + return propertiesKeys.reduce((schema, propertyKey) => { + const properties = schema[propertyKey]; + if (_.isEmpty(properties)) { + return schema; + } + + const adaptedProperties = Object.keys(properties).reduce((adaptedProperties, key) => { + if (key === '$ref') { + return Object.assign({}, adaptedProperties, { + [key]: convertReferenceName(properties[key]) + }) + } + + const updatedKey = convertToValidAvroName(key); + const adaptedProperty = adaptPropertiesNames(properties[key]); + + return Object.assign({}, adaptedProperties, { + [updatedKey]: adaptedProperty + }); + }, {}); + + return Object.assign({}, schema, { + [propertyKey]: adaptedProperties + }); + }, adaptedSchema); +}; + +const adaptNames = _.flow([ + adaptTitle, + adaptPropertiesNames +]); + +const convertReferenceName = ref => { + if (!_.isString(ref)) { + return ref; + } + + const refNames = ref.split('/'); + const referenceName = _.last(refNames); + const adaptedReferenceName = convertToValidAvroName(referenceName); + + return refNames.slice(0, -1).concat(adaptedReferenceName).join('/'); +}; + +const convertToValidAvroName = name => { + if (!_.isString(name)) { + return name; + } + + return name.replace(/[^A-Za-z0-9_]/g, '_'); +}; + +const adaptJsonSchema = jsonSchema => { + const adaptedJsonSchema = adaptNames(jsonSchema); + + return mapJsonSchema(adaptedJsonSchema, _.flow([ + adaptType, + populateDefaultNullValuesForMultiple, + handleEmptyDefaultInProperties + ])); +}; + +const adaptJsonSchemaName = convertToValidAvroName; + +module.exports = { adaptJsonSchema, adaptJsonSchemaName }; diff --git a/types/null.json b/types/null.json index d0a664d..c43ee07 100644 --- a/types/null.json +++ b/types/null.json @@ -3,7 +3,7 @@ "name": "null", "erdAbbreviation": "", "dtdAbbreviation": "{null}", - "useSample": false, + "useSample": true, "defaultValues": { "default": null, "type": "null",