From 653c3102d11f1f2fa9b94d712a3549179270881a Mon Sep 17 00:00:00 2001 From: Alik Rakhmonov Date: Fri, 14 Jun 2024 15:12:35 +0300 Subject: [PATCH 01/10] feat: add vector type, update FE and RE --- .../ddlHelpers/columnDefinitionHelper.js | 77 ++++++++++++------- .../ddlProvider/ddlProvider.js | 2 + .../field_level/fieldLevelConfig.json | 48 ++++++++++++ types/object.json | 3 +- types/vector.json | 33 ++++++++ 5 files changed, 136 insertions(+), 27 deletions(-) create mode 100644 types/vector.json diff --git a/forward_engineering/ddlProvider/ddlHelpers/columnDefinitionHelper.js b/forward_engineering/ddlProvider/ddlHelpers/columnDefinitionHelper.js index 5ab5fea..7221885 100644 --- a/forward_engineering/ddlProvider/ddlHelpers/columnDefinitionHelper.js +++ b/forward_engineering/ddlProvider/ddlHelpers/columnDefinitionHelper.js @@ -124,6 +124,12 @@ module.exports = ({ _, wrap, assignTemplates, templates, commentIfDeactivated, w return ` INTERVAL DAY${_.isNumber(dayPrecision) ? `(${dayPrecision})` : ''} TO SECOND${_.isNumber(fractSecPrecision) ? `(${fractSecPrecision})` : ''}`; }; + const decorateVector = (type, dimensionNumber, dimensionFormat) => { + const dimension = dimensionNumber || '*'; + const format = _.toUpper(dimensionFormat || '*'); + return ` ${type}(${dimension}, ${format})`; + }; + const canHaveByte = type => ['CHAR', 'VARCHAR2'].includes(type); const canHaveLength = type => ['CHAR', 'VARCHAR2', 'NCHAR', 'NVARCHAR2', 'RAW', 'UROWID'].includes(type); const canHavePrecision = type => ['NUMBER', 'FLOAT'].includes(type); @@ -131,35 +137,54 @@ module.exports = ({ _, wrap, assignTemplates, templates, commentIfDeactivated, w const isIntervalYear = type => type === 'INTERVAL YEAR'; const isIntervalDay = type => type === 'INTERVAL DAY'; const isTimezone = type => type === 'TIMESTAMP'; + const isVector = type => type === 'VECTOR'; const decorateType = (type, columnDefinition) => { - switch (true) { - case columnDefinition.lengthSemantics && - canHaveByte(type) && - canHaveLength(type) && - _.isNumber(columnDefinition.length): - return addByteLength(type, columnDefinition.length, columnDefinition.lengthSemantics); - case canHaveLength(type) && _.isNumber(columnDefinition.length): - return addLength(type, columnDefinition.length); - case canHavePrecision(type) && canHaveScale(type): - return addScalePrecision(type, columnDefinition.precision, columnDefinition.scale); - case canHavePrecision(type) && _.isNumber(columnDefinition.precision): - return addPrecision(type, columnDefinition.precision); - case isTimezone(type): - return timestamp( - columnDefinition.fractSecPrecision, - columnDefinition.withTimeZone, - columnDefinition.localTimeZone, - ); - case isIntervalYear(type): - return intervalYear(columnDefinition.yearPrecision); - case isIntervalDay(type): - return intervalDay(columnDefinition.dayPrecision, columnDefinition.fractSecPrecision); - case !!(columnDefinition.isUDTRef && columnDefinition.schemaName): - return ` "${columnDefinition.schemaName}"."${type}"`; - default: - return ` ${type}`; + const { + lengthSemantics, + length, + precision, + scale, + fractSecPrecision, + withTimeZone, + localTimeZone, + yearPrecision, + dayPrecision, + isUDTRef, + schemaName, + dimension, + subtype, + } = columnDefinition; + + if (lengthSemantics && canHaveByte(type) && canHaveLength(type) && _.isNumber(length)) { + return addByteLength(type, length, lengthSemantics); + } + if (canHaveLength(type) && _.isNumber(length)) { + return addLength(type, length); + } + if (canHavePrecision(type) && canHaveScale(type)) { + return addScalePrecision(type, precision, scale); + } + if (canHavePrecision(type) && _.isNumber(precision)) { + return addPrecision(type, precision); + } + if (isTimezone(type)) { + return timestamp(fractSecPrecision, withTimeZone, localTimeZone); } + if (isIntervalYear(type)) { + return intervalYear(yearPrecision); + } + if (isIntervalDay(type)) { + return intervalDay(dayPrecision, fractSecPrecision); + } + if (isUDTRef && schemaName) { + return ` "${schemaName}"."${type}"`; + } + if (isVector(type) && (dimension || subtype)) { + return decorateVector(type, dimension, subtype); + } + + return ` ${type}`; }; /** diff --git a/forward_engineering/ddlProvider/ddlProvider.js b/forward_engineering/ddlProvider/ddlProvider.js index c9a1a69..311db94 100644 --- a/forward_engineering/ddlProvider/ddlProvider.js +++ b/forward_engineering/ddlProvider/ddlProvider.js @@ -195,6 +195,8 @@ module.exports = (baseProvider, options, app) => { encryption: jsonSchema.encryption, synonyms: schemaData?.synonyms?.filter(synonym => synonym.synonymEntityId === jsonSchema.GUID) || [], isUDTRef, + dimension: jsonSchema.dimension, + subtype: jsonSchema.subtype, ...(canHaveIdentity(jsonSchema.mode) && { identity: jsonSchema.identity }), }; }, diff --git a/properties_pane/field_level/fieldLevelConfig.json b/properties_pane/field_level/fieldLevelConfig.json index c6eafb4..61f68f4 100644 --- a/properties_pane/field_level/fieldLevelConfig.json +++ b/properties_pane/field_level/fieldLevelConfig.json @@ -9797,6 +9797,54 @@ making sure that you maintain a proper JSON format. "template": "textarea" } ], + "vector": [ + "name", + "code", + "isActivated", + { + "propertyKeyword": "type", + "typeDecorator": { + "value": "vctr", + "useAsTypeName": true + } + }, + { + "propertyName": "Dimension", + "propertyKeyword": "dimension", + "propertyType": "numeric", + "valueType": "integer", + "allowNegative": false, + "typeDecorator": { + "useAngleBrackets": true + }, + "minValue": 1 + }, + { + "propertyName": "Subtype", + "propertyKeyword": "subtype", + "shouldValidate": false, + "propertyType": "select", + "typeDecorator": true, + "options": ["", "int8", "float32", "float64"] + }, + { + "propertyName": "Comments", + "propertyKeyword": "description", + "propertyTooltip": "comments", + "addTimestampButton": false, + "propertyType": "details", + "template": "textarea" + }, + { + "propertyName": "Remarks", + "propertyKeyword": "comments", + "shouldValidate": false, + "propertyTooltip": "remarks", + "addTimestampButton": true, + "propertyType": "details", + "template": "textarea" + } + ], "object_udt": [ { "fieldKeyword": "name", diff --git a/types/object.json b/types/object.json index 6840894..2757ab0 100644 --- a/types/object.json +++ b/types/object.json @@ -19,7 +19,8 @@ "reference", "multiple", "object_udt", - "collection_udt" + "collection_udt", + "vector" ] } } diff --git a/types/vector.json b/types/vector.json new file mode 100644 index 0000000..d1bf8be --- /dev/null +++ b/types/vector.json @@ -0,0 +1,33 @@ +{ + "name": "vector", + "erdAbbreviation": "", + "dtdAbbreviation": "[...]", + "parentType": "array", + "useSample": false, + "hiddenOnEntity": "view", + "dependency": { + "type": "not", + "values": [ + { + "level": "model", + "key": "dbVersion", + "value": "12c" + }, + { + "level": "model", + "key": "dbVersion", + "value": "18c" + }, + { + "level": "model", + "key": "dbVersion", + "value": "19c" + }, + { + "level": "model", + "key": "dbVersion", + "value": "21c" + } + ] + } +} From 03af39967b7df3352ad99050f9aaa7aa8c5e0df5 Mon Sep 17 00:00:00 2001 From: Alik Rakhmonov Date: Thu, 20 Jun 2024 14:40:51 +0300 Subject: [PATCH 02/10] fix: properties --- types/vector.json | 3 +++ 1 file changed, 3 insertions(+) diff --git a/types/vector.json b/types/vector.json index d1bf8be..f2747f5 100644 --- a/types/vector.json +++ b/types/vector.json @@ -5,6 +5,9 @@ "parentType": "array", "useSample": false, "hiddenOnEntity": "view", + "defaultValues": { + "properties": [] + }, "dependency": { "type": "not", "values": [ From 7657231580f20bc2274c346ca33d4e4928eff55c Mon Sep 17 00:00:00 2001 From: Alik Rakhmonov Date: Thu, 20 Jun 2024 16:17:41 +0300 Subject: [PATCH 03/10] fix: adding children to vector --- types/vector.json | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/types/vector.json b/types/vector.json index f2747f5..d20b900 100644 --- a/types/vector.json +++ b/types/vector.json @@ -8,6 +8,20 @@ "defaultValues": { "properties": [] }, + "subtypes": { + "vector": { + "childValueType": "number" + }, + "int8": { + "childValueType": "number" + }, + "float32": { + "childValueType": "number" + }, + "float64": { + "childValueType": "number" + } + }, "dependency": { "type": "not", "values": [ From 87351966521ae32368918516bc8630581cd03a65 Mon Sep 17 00:00:00 2001 From: Alik Rakhmonov Date: Fri, 21 Jun 2024 15:57:58 +0300 Subject: [PATCH 04/10] fix: changing from 23ai should convert vector to blob --- .../model_level/modelLevelConfig.json | 24 ++++++++++++++----- types/vector.json | 7 ++++++ 2 files changed, 25 insertions(+), 6 deletions(-) diff --git a/properties_pane/model_level/modelLevelConfig.json b/properties_pane/model_level/modelLevelConfig.json index f38829c..d134c42 100644 --- a/properties_pane/model_level/modelLevelConfig.json +++ b/properties_pane/model_level/modelLevelConfig.json @@ -2,9 +2,9 @@ * Copyright © 2016-2021 by IntegrIT S.A. dba Hackolade. All rights reserved. * * The copyright to the computer software herein is the property of IntegrIT S.A. -* The software may be used and/or copied only with the written permission of -* IntegrIT S.A. or in accordance with the terms and conditions stipulated in -* the agreement/contract under which the software has been supplied. +* The software may be used and/or copied only with the written permission of +* IntegrIT S.A. or in accordance with the terms and conditions stipulated in +* the agreement/contract under which the software has been supplied. In order to define custom properties for any object's properties pane, you may copy/paste from the following, @@ -71,8 +71,8 @@ making sure that you maintain a proper JSON format. ] }, // “groupInput” can have the following states - 0 items, 1 item, and many items. -// “blockInput” has only 2 states - 0 items or 1 item. -// This gives us an easy way to represent it as an object and not as an array internally which is beneficial for processing +// “blockInput” has only 2 states - 0 items or 1 item. +// This gives us an easy way to represent it as an object and not as an array internally which is beneficial for processing // and forward-engineering in particular. { "propertyName": "Block", @@ -100,7 +100,7 @@ making sure that you maintain a proper JSON format. "propertyKeyword": "keyList", "propertyType": "fieldList", "template": "orderedList" - }, + }, { "propertyName": "List with attribute", "propertyKeyword": "keyListOrder", @@ -147,6 +147,18 @@ making sure that you maintain a proper JSON format. ] }, "adapter": "adaptUnavailableTypes" + }, + { + "dependency": { + "type": "not", + "values": [ + { + "key": "dbVersion", + "value": "23ai" + } + ] + }, + "adapter": "adaptUnavailableTypes" } ] }, diff --git a/types/vector.json b/types/vector.json index d20b900..42bfeff 100644 --- a/types/vector.json +++ b/types/vector.json @@ -22,6 +22,13 @@ "childValueType": "number" } }, + "compatibleWith": { + "type": "lobs", + "parentType": "jsonArray", + "mode": "blob", + "subtype": "array", + "properties": [] + }, "dependency": { "type": "not", "values": [ From 4e751177eb21879c414596d9bdc1297ce7e61a8d Mon Sep 17 00:00:00 2001 From: Alik Rakhmonov Date: Fri, 21 Jun 2024 16:35:48 +0300 Subject: [PATCH 05/10] fix: missing additional properties --- properties_pane/field_level/fieldLevelConfig.json | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/properties_pane/field_level/fieldLevelConfig.json b/properties_pane/field_level/fieldLevelConfig.json index 61f68f4..1d61538 100644 --- a/properties_pane/field_level/fieldLevelConfig.json +++ b/properties_pane/field_level/fieldLevelConfig.json @@ -9827,6 +9827,15 @@ making sure that you maintain a proper JSON format. "typeDecorator": true, "options": ["", "int8", "float32", "float64"] }, + { + "propertyName": "Not null", + "propertyKeyword": "required", + "propertyType": "checkbox" + }, + "minItems", + "maxItems", + "uniqueItems", + "additionalItems", { "propertyName": "Comments", "propertyKeyword": "description", From 943766d2a842b33b3975d79ae5d12a498a518665 Mon Sep 17 00:00:00 2001 From: Vitalii Yarmus Date: Fri, 21 Jun 2024 17:37:25 +0300 Subject: [PATCH 06/10] Update vector type config to make type conversion work properly --- properties_pane/field_level/fieldLevelConfig.json | 11 +++++++++-- types/vector.json | 7 ++++--- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/properties_pane/field_level/fieldLevelConfig.json b/properties_pane/field_level/fieldLevelConfig.json index 61f68f4..1604cc8 100644 --- a/properties_pane/field_level/fieldLevelConfig.json +++ b/properties_pane/field_level/fieldLevelConfig.json @@ -9824,8 +9824,15 @@ making sure that you maintain a proper JSON format. "propertyKeyword": "subtype", "shouldValidate": false, "propertyType": "select", - "typeDecorator": true, - "options": ["", "int8", "float32", "float64"] + "typeDecorator": { + "useOptionName": true + }, + "options": [ + "", + { "name": "int8", "value": "vector" }, + { "name": "float32", "value": "vector" }, + { "name": "float64", "value": "vector" } + ] }, { "propertyName": "Comments", diff --git a/types/vector.json b/types/vector.json index d20b900..977748b 100644 --- a/types/vector.json +++ b/types/vector.json @@ -6,19 +6,20 @@ "useSample": false, "hiddenOnEntity": "view", "defaultValues": { + "subtype": "", "properties": [] }, "subtypes": { "vector": { "childValueType": "number" }, - "int8": { + "vector": { "childValueType": "number" }, - "float32": { + "vector": { "childValueType": "number" }, - "float64": { + "vector": { "childValueType": "number" } }, From 64e093a620e1b01bdbadf780923097cb773c752e Mon Sep 17 00:00:00 2001 From: Vitalii Yarmus Date: Fri, 21 Jun 2024 18:00:09 +0300 Subject: [PATCH 07/10] FE: fix vector subtype FE --- .../ddlProvider/ddlHelpers/columnDefinitionHelper.js | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/forward_engineering/ddlProvider/ddlHelpers/columnDefinitionHelper.js b/forward_engineering/ddlProvider/ddlHelpers/columnDefinitionHelper.js index 7221885..5b4658b 100644 --- a/forward_engineering/ddlProvider/ddlHelpers/columnDefinitionHelper.js +++ b/forward_engineering/ddlProvider/ddlHelpers/columnDefinitionHelper.js @@ -124,7 +124,12 @@ module.exports = ({ _, wrap, assignTemplates, templates, commentIfDeactivated, w return ` INTERVAL DAY${_.isNumber(dayPrecision) ? `(${dayPrecision})` : ''} TO SECOND${_.isNumber(fractSecPrecision) ? `(${fractSecPrecision})` : ''}`; }; - const decorateVector = (type, dimensionNumber, dimensionFormat) => { + const decorateVector = (type, dimensionNumber, subtype) => { + const dimensionFormat = { + 'vector': 'int8', + 'vector': 'float32', + 'vector': 'float64', + }[subtype]; const dimension = dimensionNumber || '*'; const format = _.toUpper(dimensionFormat || '*'); return ` ${type}(${dimension}, ${format})`; From 1a34064915bb0e3bf4f45f4c09c0bc7d8c79f1fd Mon Sep 17 00:00:00 2001 From: Vitalii Yarmus Date: Tue, 25 Jun 2024 13:41:15 +0300 Subject: [PATCH 08/10] Update default dbVersion, update disable config for jsonSubquery --- properties_pane/defaultData.json | 2 +- reverse_engineering/helpers/oracleHelper.js | 2 +- types/joinSubquery.json | 26 ++++++++++++++++++--- 3 files changed, 25 insertions(+), 5 deletions(-) diff --git a/properties_pane/defaultData.json b/properties_pane/defaultData.json index d583b1f..7cea5b8 100644 --- a/properties_pane/defaultData.json +++ b/properties_pane/defaultData.json @@ -1,7 +1,7 @@ { "model": { "modelName": "New model", - "dbVersion": "21c", + "dbVersion": "23ai", "dbVendor": "Oracle" }, "container": { diff --git a/reverse_engineering/helpers/oracleHelper.js b/reverse_engineering/helpers/oracleHelper.js index 24ea7fa..fd4e9ef 100644 --- a/reverse_engineering/helpers/oracleHelper.js +++ b/reverse_engineering/helpers/oracleHelper.js @@ -471,7 +471,7 @@ const execute = (command, options = {}, binds = []) => { const getDbVersion = async logger => { const versions = ['12c', '18c', '19c', '21c', '23ai']; - const defaultVersion = '21c'; + const defaultVersion = '23ai'; try { const versionTable = await execute( diff --git a/types/joinSubquery.json b/types/joinSubquery.json index 2c2fbb9..caa8fc7 100644 --- a/types/joinSubquery.json +++ b/types/joinSubquery.json @@ -17,9 +17,29 @@ "type": "and", "values": [ { - "level": "model", - "key": "dbVersion", - "value": "23ai" + "type": "not", + "values": [ + { + "level": "model", + "key": "dbVersion", + "value": "12c" + }, + { + "level": "model", + "key": "dbVersion", + "value": "18c" + }, + { + "level": "model", + "key": "dbVersion", + "value": "19c" + }, + { + "level": "model", + "key": "dbVersion", + "value": "21c" + } + ] }, { "level": "root", From 57fb1d074af21db9ab12f00dffb8ef5399170318 Mon Sep 17 00:00:00 2001 From: Vitalii Yarmus Date: Tue, 25 Jun 2024 14:30:07 +0300 Subject: [PATCH 09/10] Revert "Update default dbVersion, update disable config for jsonSubquery" This reverts commit 1a34064915bb0e3bf4f45f4c09c0bc7d8c79f1fd. --- properties_pane/defaultData.json | 2 +- reverse_engineering/helpers/oracleHelper.js | 2 +- types/joinSubquery.json | 26 +++------------------ 3 files changed, 5 insertions(+), 25 deletions(-) diff --git a/properties_pane/defaultData.json b/properties_pane/defaultData.json index 7cea5b8..d583b1f 100644 --- a/properties_pane/defaultData.json +++ b/properties_pane/defaultData.json @@ -1,7 +1,7 @@ { "model": { "modelName": "New model", - "dbVersion": "23ai", + "dbVersion": "21c", "dbVendor": "Oracle" }, "container": { diff --git a/reverse_engineering/helpers/oracleHelper.js b/reverse_engineering/helpers/oracleHelper.js index fd4e9ef..24ea7fa 100644 --- a/reverse_engineering/helpers/oracleHelper.js +++ b/reverse_engineering/helpers/oracleHelper.js @@ -471,7 +471,7 @@ const execute = (command, options = {}, binds = []) => { const getDbVersion = async logger => { const versions = ['12c', '18c', '19c', '21c', '23ai']; - const defaultVersion = '23ai'; + const defaultVersion = '21c'; try { const versionTable = await execute( diff --git a/types/joinSubquery.json b/types/joinSubquery.json index caa8fc7..2c2fbb9 100644 --- a/types/joinSubquery.json +++ b/types/joinSubquery.json @@ -17,29 +17,9 @@ "type": "and", "values": [ { - "type": "not", - "values": [ - { - "level": "model", - "key": "dbVersion", - "value": "12c" - }, - { - "level": "model", - "key": "dbVersion", - "value": "18c" - }, - { - "level": "model", - "key": "dbVersion", - "value": "19c" - }, - { - "level": "model", - "key": "dbVersion", - "value": "21c" - } - ] + "level": "model", + "key": "dbVersion", + "value": "23ai" }, { "level": "root", From 4df842903e4983753f5d618857310f0e4f824f46 Mon Sep 17 00:00:00 2001 From: Vitalii Yarmus Date: Tue, 25 Jun 2024 14:30:51 +0300 Subject: [PATCH 10/10] Improve joinSubquery dependency --- types/joinSubquery.json | 26 +++++++++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) diff --git a/types/joinSubquery.json b/types/joinSubquery.json index 2c2fbb9..caa8fc7 100644 --- a/types/joinSubquery.json +++ b/types/joinSubquery.json @@ -17,9 +17,29 @@ "type": "and", "values": [ { - "level": "model", - "key": "dbVersion", - "value": "23ai" + "type": "not", + "values": [ + { + "level": "model", + "key": "dbVersion", + "value": "12c" + }, + { + "level": "model", + "key": "dbVersion", + "value": "18c" + }, + { + "level": "model", + "key": "dbVersion", + "value": "19c" + }, + { + "level": "model", + "key": "dbVersion", + "value": "21c" + } + ] }, { "level": "root",