diff --git a/.github/workflows/maven-sonar-build.yml b/.github/workflows/maven-sonar-build.yml index e0295d0cb1c6..5e5a03e86159 100644 --- a/.github/workflows/maven-sonar-build.yml +++ b/.github/workflows/maven-sonar-build.yml @@ -16,7 +16,7 @@ on: push: branches: - main - - '0.[0-9]+.[0-9]+' + - '[0-9]+.[0-9]+.[0-9]+' paths: - "openmetadata-service/**" - "openmetadata-ui/**" @@ -116,4 +116,4 @@ jobs: with: github_token: ${{ secrets.GITHUB_TOKEN }} fail_on_test_failures: true - report_paths: 'openmetadata-service/target/surefire-reports/TEST-*.xml' \ No newline at end of file + report_paths: 'openmetadata-service/target/surefire-reports/TEST-*.xml' diff --git a/openmetadata-service/src/main/java/org/openmetadata/csv/CsvUtil.java b/openmetadata-service/src/main/java/org/openmetadata/csv/CsvUtil.java index 61d83fc6ae65..d09e4b7ccc72 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/csv/CsvUtil.java +++ b/openmetadata-service/src/main/java/org/openmetadata/csv/CsvUtil.java @@ -112,7 +112,7 @@ public static List fieldToInternalArray(String field) { } /** - * Parses a field containing key-value pairs separated by semicolons, correctly handling quotes. + * Parses a field containing key-value pairs separated by FIELD_SEPARATOR, correctly handling quotes. * Each key-value pair may also be enclosed in quotes, especially if it contains delimiter like (SEPARATOR , FIELD_SEPARATOR). * Input Example: * "key1:value1;key2:value2;\"key3:value;with;semicolon\"" @@ -124,7 +124,8 @@ public static List fieldToExtensionStrings(String field) throws IOExcept return List.of(); } - // Replace semicolons within quoted strings with a placeholder + // Case when semicolon is part of the fieldValue - Replace semicolons within quoted strings with + // a placeholder String preprocessedField = Pattern.compile("\"([^\"]*)\"") // Matches content inside double quotes .matcher(field) @@ -146,9 +147,7 @@ public static List fieldToExtensionStrings(String field) throws IOExcept .flatMap(CSVRecord::stream) .map( value -> - value - .replace("__SEMICOLON__", ";") - .replace("\\n", "\n")) // Restore original semicolons and newlines + value.replace("__SEMICOLON__", ";")) // Restore original semicolons and newlines .map( value -> value.startsWith("\"") && value.endsWith("\"") // Remove outer quotes if present @@ -158,6 +157,48 @@ public static List fieldToExtensionStrings(String field) throws IOExcept } } + /** + * Parses a field containing column values separated by SEPARATOR, correctly handling quotes. + * Each value enclosed in quotes, especially if it contains delimiter like SEPARATOR. + * Input Example: + * "value1,value2,\"value,with,comma\"" + * Output: [value1, value2, value,with,comma] + * + */ + public static List fieldToColumns(String field) throws IOException { + if (field == null || field.isBlank()) { + return Collections.emptyList(); + } + + // Case when comma is part of the columnValue - Replace commas within quoted strings with a + // placeholder + String preprocessedField = + Pattern.compile("\"([^\"]*)\"") + .matcher(field) + .replaceAll(mr -> "\"" + mr.group(1).replace(",", "__COMMA__") + "\""); + + preprocessedField = preprocessedField.replace("\n", "\\n").replace("\"", "\\\""); + + CSVFormat format = CSVFormat.DEFAULT.withDelimiter(',').withQuote('"').withEscape('\\'); + + List columns; + try (CSVParser parser = CSVParser.parse(new StringReader(preprocessedField), format)) { + columns = + parser.getRecords().stream() + .flatMap(CSVRecord::stream) + .map(value -> value.replace("__COMMA__", ",")) + .map( + value -> + value.startsWith("\"") + && value.endsWith("\"") // Remove outer quotes if present + ? value.substring(1, value.length() - 1) + : value) + .collect(Collectors.toList()); + } + + return columns; + } + public static String quote(String field) { return String.format("\"%s\"", field); } @@ -270,6 +311,13 @@ private static String quoteCsvField(String str) { return str; } + private static String quoteCsvFieldForSeparator(String str) { + if (str.contains(SEPARATOR)) { + return quote(str); + } + return str; + } + public static List addExtension(List csvRecord, Object extension) { if (extension == null) { csvRecord.add(null); @@ -310,6 +358,8 @@ private static String formatMapValue(Map valueMap) { return formatEntityReference(valueMap); } else if (isTimeInterval(valueMap)) { return formatTimeInterval(valueMap); + } else if (isTableType(valueMap)) { + return formatTableRows(valueMap); } return valueMap.toString(); @@ -339,6 +389,10 @@ private static boolean isTimeInterval(Map valueMap) { return valueMap.containsKey("start") && valueMap.containsKey("end"); } + private static boolean isTableType(Map valueMap) { + return valueMap.containsKey("rows") && valueMap.containsKey("columns"); + } + private static String formatEntityReference(Map valueMap) { return valueMap.get("type") + ENTITY_TYPE_SEPARATOR + valueMap.get("fullyQualifiedName"); } @@ -346,4 +400,19 @@ private static String formatEntityReference(Map valueMap) { private static String formatTimeInterval(Map valueMap) { return valueMap.get("start") + ENTITY_TYPE_SEPARATOR + valueMap.get("end"); } + + private static String formatTableRows(Map valueMap) { + List columns = (List) valueMap.get("columns"); + List> rows = (List>) valueMap.get("rows"); + + return rows.stream() + .map( + row -> + columns.stream() + .map( + column -> + quoteCsvFieldForSeparator(row.getOrDefault(column, "").toString())) + .collect(Collectors.joining(SEPARATOR))) + .collect(Collectors.joining(INTERNAL_ARRAY_SEPARATOR)); + } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/csv/EntityCsv.java b/openmetadata-service/src/main/java/org/openmetadata/csv/EntityCsv.java index 9ad2cd76b2f6..34d540749a1d 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/csv/EntityCsv.java +++ b/openmetadata-service/src/main/java/org/openmetadata/csv/EntityCsv.java @@ -18,6 +18,7 @@ import static org.openmetadata.common.utils.CommonUtil.nullOrEmpty; import static org.openmetadata.csv.CsvUtil.ENTITY_TYPE_SEPARATOR; import static org.openmetadata.csv.CsvUtil.FIELD_SEPARATOR; +import static org.openmetadata.csv.CsvUtil.fieldToColumns; import static org.openmetadata.csv.CsvUtil.fieldToEntities; import static org.openmetadata.csv.CsvUtil.fieldToExtensionStrings; import static org.openmetadata.csv.CsvUtil.fieldToInternalArray; @@ -40,6 +41,8 @@ import java.util.Collections; import java.util.Comparator; import java.util.HashMap; +import java.util.Iterator; +import java.util.LinkedHashMap; import java.util.List; import java.util.Locale; import java.util.Map; @@ -66,6 +69,7 @@ import org.openmetadata.schema.type.csv.CsvFile; import org.openmetadata.schema.type.csv.CsvHeader; import org.openmetadata.schema.type.csv.CsvImportResult; +import org.openmetadata.schema.type.customproperties.TableConfig; import org.openmetadata.service.Entity; import org.openmetadata.service.TypeRegistry; import org.openmetadata.service.jdbi3.EntityRepository; @@ -379,7 +383,7 @@ private void validateExtension( parseEntityReferences(printer, csvRecord, fieldNumber, fieldValue.toString(), isList); } case "date-cp", "dateTime-cp", "time-cp" -> fieldValue = - getFormattedDateTimeField( + parseFormattedDateTimeField( printer, csvRecord, fieldNumber, @@ -392,19 +396,13 @@ private void validateExtension( fieldValue = enumKeys.isEmpty() ? null : enumKeys; } case "timeInterval" -> fieldValue = - handleTimeInterval(printer, csvRecord, fieldNumber, fieldName, fieldValue); - case "number", "integer", "timestamp" -> { - try { - fieldValue = Long.parseLong(fieldValue.toString()); - } catch (NumberFormatException e) { - importFailure( - printer, - invalidCustomPropertyValue( - fieldNumber, fieldName, customPropertyType, fieldValue.toString()), - csvRecord); - fieldValue = null; - } - } + parseTimeInterval(printer, csvRecord, fieldNumber, fieldName, fieldValue); + case "number", "integer", "timestamp" -> fieldValue = + parseLongField( + printer, csvRecord, fieldNumber, fieldName, customPropertyType, fieldValue); + case "table-cp" -> fieldValue = + parseTableType(printer, csvRecord, fieldNumber, fieldName, fieldValue, propertyConfig); + default -> {} } // Validate the field against the JSON schema @@ -448,7 +446,7 @@ private Object parseEntityReferences( return isList ? entityReferences : entityReferences.isEmpty() ? null : entityReferences.get(0); } - protected String getFormattedDateTimeField( + protected String parseFormattedDateTimeField( CSVPrinter printer, CSVRecord csvRecord, int fieldNumber, @@ -484,7 +482,7 @@ protected String getFormattedDateTimeField( } } - private Map handleTimeInterval( + private Map parseTimeInterval( CSVPrinter printer, CSVRecord csvRecord, int fieldNumber, String fieldName, Object fieldValue) throws IOException { List timestampValues = fieldToEntities(fieldValue.toString()); @@ -511,6 +509,70 @@ private Map handleTimeInterval( return timestampMap; } + private Object parseLongField( + CSVPrinter printer, + CSVRecord csvRecord, + int fieldNumber, + String fieldName, + String customPropertyType, + Object fieldValue) + throws IOException { + try { + return Long.parseLong(fieldValue.toString()); + } catch (NumberFormatException e) { + importFailure( + printer, + invalidCustomPropertyValue( + fieldNumber, fieldName, customPropertyType, fieldValue.toString()), + csvRecord); + return null; + } + } + + private Object parseTableType( + CSVPrinter printer, + CSVRecord csvRecord, + int fieldNumber, + String fieldName, + Object fieldValue, + String propertyConfig) + throws IOException { + List tableValues = listOrEmpty(fieldToInternalArray(fieldValue.toString())); + List> rows = new ArrayList<>(); + TableConfig tableConfig = + JsonUtils.treeToValue(JsonUtils.readTree(propertyConfig), TableConfig.class); + + for (String row : tableValues) { + List columns = listOrEmpty(fieldToColumns(row)); + Map rowMap = new LinkedHashMap<>(); + Iterator columnIterator = tableConfig.getColumns().iterator(); + Iterator valueIterator = columns.iterator(); + + if (columns.size() > tableConfig.getColumns().size()) { + importFailure( + printer, + invalidCustomPropertyValue( + fieldNumber, + fieldName, + "table", + "Column count should be less than or equal to " + tableConfig.getColumns().size()), + csvRecord); + return null; + } + + while (columnIterator.hasNext() && valueIterator.hasNext()) { + rowMap.put(columnIterator.next(), valueIterator.next()); + } + + rows.add(rowMap); + } + + Map tableJson = new LinkedHashMap<>(); + tableJson.put("rows", rows); + tableJson.put("columns", tableConfig.getColumns()); + return tableJson; + } + private void validateAndUpdateExtension( CSVPrinter printer, CSVRecord csvRecord, diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/resources/glossary/GlossaryResourceTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/resources/glossary/GlossaryResourceTest.java index f1594b74c5b5..383aec3d954a 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/resources/glossary/GlossaryResourceTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/resources/glossary/GlossaryResourceTest.java @@ -48,10 +48,12 @@ import static org.openmetadata.service.util.TestUtils.validateTagLabel; import java.io.IOException; +import java.util.ArrayList; import java.util.Comparator; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.UUID; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; @@ -87,6 +89,7 @@ import org.openmetadata.schema.type.TagLabel.TagSource; import org.openmetadata.schema.type.TaskStatus; import org.openmetadata.schema.type.csv.CsvImportResult; +import org.openmetadata.schema.type.customproperties.TableConfig; import org.openmetadata.service.Entity; import org.openmetadata.service.exception.CatalogExceptionMessage; import org.openmetadata.service.jdbi3.EntityRepository.EntityUpdater; @@ -643,6 +646,44 @@ invalidCustomPropertyKeyRecord, invalidCustomPropertyKey(11, "invalidCustomPrope 11, "glossaryTermDateCp", DATECP_TYPE.getDisplayName(), "dd-MM-yyyy")) }; assertRows(result, expectedRows); + + // Create glossaryTerm with Invalid custom property of type table + TableConfig tableConfig = + new TableConfig().withColumns(Set.of("columnName1", "columnName2", "columnName3")); + CustomProperty glossaryTermEnumCp = + new CustomProperty() + .withName("glossaryTermTableCp") + .withDescription("table type custom property ") + .withPropertyType(TABLE_TYPE.getEntityReference()) + .withCustomPropertyConfig( + new CustomPropertyConfig() + .withConfig(Map.of("columns", new ArrayList<>(tableConfig.getColumns())))); + entityType = + typeResourceTest.getEntityByName( + Entity.GLOSSARY_TERM, "customProperties", ADMIN_AUTH_HEADERS); + entityType = + typeResourceTest.addAndCheckCustomProperty( + entityType.getId(), glossaryTermEnumCp, OK, ADMIN_AUTH_HEADERS); + String invalidTableTypeRecord = + ",g1,dsp1,dsc1,h1;h2;h3,,term1;http://term1,PII.None,,,,\"glossaryTermTableCp:row_1_col1_Value,row_1_col2_Value,row_1_col3_Value,row_1_col4_Value|row_2_col1_Value,row_2_col2_Value,row_2_col3_Value,row_2_col4_Value\""; + String invalidTableTypeValue = + ",g1,dsp1,dsc1,h1;h2;h3,,term1;http://term1,PII.None,,,,\"glossaryTermTableCp:row_1_col1_Value,row_1_col2_Value,row_1_col3_Value,row_1_col4_Value|row_2_col1_Value,row_2_col2_Value,row_2_col3_Value,row_2_col4_Value\""; + csv = createCsv(GlossaryCsv.HEADERS, listOf(invalidTableTypeValue), null); + result = importCsv(glossaryName, csv, false); + Awaitility.await().atMost(4, TimeUnit.SECONDS).until(() -> true); + assertSummary(result, ApiStatus.PARTIAL_SUCCESS, 2, 1, 1); + expectedRows = + new String[] { + resultsHeader, + getFailedRecord( + invalidTableTypeRecord, + invalidCustomPropertyValue( + 11, + "glossaryTermTableCp", + "table", + "Column count should be less than or equal to " + tableConfig.getMaxColumns())) + }; + assertRows(result, expectedRows); } @Test @@ -767,7 +808,23 @@ void testGlossaryImportExport() throws IOException { .withName("glossaryTermEnumCpMulti") .withDescription("enum type custom property with multiselect = true") .withPropertyType(ENUM_TYPE.getEntityReference()) - .withCustomPropertyConfig(enumConfig) + .withCustomPropertyConfig(enumConfig), + new CustomProperty() + .withName("glossaryTermTableCol1Cp") + .withDescription("table type custom property with 1 column") + .withPropertyType(TABLE_TYPE.getEntityReference()) + .withCustomPropertyConfig( + new CustomPropertyConfig() + .withConfig( + Map.of("columns", List.of("columnName1", "columnName2", "columnName3")))), + new CustomProperty() + .withName("glossaryTermTableCol3Cp") + .withDescription("table type custom property with 3 columns") + .withPropertyType(TABLE_TYPE.getEntityReference()) + .withCustomPropertyConfig( + new CustomPropertyConfig() + .withConfig( + Map.of("columns", List.of("columnName1", "columnName2", "columnName3")))), }; for (CustomProperty customProperty : customProperties) { @@ -801,12 +858,13 @@ void testGlossaryImportExport() throws IOException { ",g2,dsp2,new-dsc3,h1;h3;h3,,term2;https://term2,PII.NonSensitive,user:%s,user:%s,%s,\"glossaryTermEnumCpMulti:val3|val2|val1|val4|val5;glossaryTermEnumCpSingle:singleVal1;glossaryTermIntegerCp:7777;glossaryTermMarkdownCp:# Sample Markdown Text;glossaryTermNumberCp:123456;\"\"glossaryTermQueryCp:select col,row from table where id ='30';\"\";glossaryTermStringCp:sample string content;glossaryTermTimeCp:10:08:45;glossaryTermTimeIntervalCp:1726142300000:17261420000;glossaryTermTimestampCp:1726142400000\"", user1, user2, "Approved"), String.format( - "importExportTest.g1,g11,dsp2,new-dsc11,h1;h3;h3,,,,user:%s,team:%s,%s,", + "importExportTest.g1,g11,dsp2,new-dsc11,h1;h3;h3,,,,user:%s,team:%s,%s,\"\"\"glossaryTermTableCol1Cp:row_1_col1_Value,,\"\";\"\"glossaryTermTableCol3Cp:row_1_col1_Value,row_1_col2_Value,row_1_col3_Value|row_2_col1_Value,row_2_col2_Value,row_2_col3_Value\"\"\"", reviewerRef.get(0), team11, "Draft")); // Add new row to existing rows List newRecords = - listOf(",g3,dsp0,dsc0,h1;h2;h3,,term0;http://term0,PII.Sensitive,,,Approved,"); + listOf( + ",g3,dsp0,dsc0,h1;h2;h3,,term0;http://term0,PII.Sensitive,,,Approved,\"\"\"glossaryTermTableCol1Cp:row_1_col1_Value,,\"\";\"\"glossaryTermTableCol3Cp:row_1_col1_Value,row_1_col2_Value,row_1_col3_Value|row_2_col1_Value,row_2_col2_Value,row_2_col3_Value\"\"\""); testImportExport( glossary.getName(), GlossaryCsv.HEADERS, createRecords, updateRecords, newRecords); } diff --git a/openmetadata-ui/src/main/resources/ui/playwright/constant/glossaryImportExport.ts b/openmetadata-ui/src/main/resources/ui/playwright/constant/glossaryImportExport.ts index eba6395a8e0e..520224305dd4 100644 --- a/openmetadata-ui/src/main/resources/ui/playwright/constant/glossaryImportExport.ts +++ b/openmetadata-ui/src/main/resources/ui/playwright/constant/glossaryImportExport.ts @@ -2,6 +2,7 @@ export const CUSTOM_PROPERTIES_TYPES = { STRING: 'String', MARKDOWN: 'Markdown', SQL_QUERY: 'Sql Query', + TABLE: 'Table', }; export const FIELD_VALUES_CUSTOM_PROPERTIES = { @@ -17,4 +18,8 @@ This project is designed to **simplify** and *automate* daily tasks. It aims to: 2. **Real-Time Analytics**: Get up-to-date insights on task progress. 3. **Automation**: Automate repetitive workflows using custom scripts.`, SQL_QUERY: 'SELECT * FROM table_name WHERE id="20";', + TABLE: { + columns: ['pw-import-export-column1', 'pw-import-export-column2'], + rows: 'pw-import-export-row1-column1,pw-import-export-row1-column2', + }, }; diff --git a/openmetadata-ui/src/main/resources/ui/playwright/e2e/Pages/GlossaryImportExport.spec.ts b/openmetadata-ui/src/main/resources/ui/playwright/e2e/Pages/GlossaryImportExport.spec.ts index c28eeba14f22..9d3394b4fa73 100644 --- a/openmetadata-ui/src/main/resources/ui/playwright/e2e/Pages/GlossaryImportExport.spec.ts +++ b/openmetadata-ui/src/main/resources/ui/playwright/e2e/Pages/GlossaryImportExport.spec.ts @@ -12,7 +12,10 @@ */ import { expect, test } from '@playwright/test'; import { CUSTOM_PROPERTIES_ENTITIES } from '../../constant/customProperty'; -import { CUSTOM_PROPERTIES_TYPES } from '../../constant/glossaryImportExport'; +import { + CUSTOM_PROPERTIES_TYPES, + FIELD_VALUES_CUSTOM_PROPERTIES, +} from '../../constant/glossaryImportExport'; import { GlobalSettingOptions } from '../../constant/settings'; import { SidebarItem } from '../../constant/sidebar'; import { Glossary } from '../../support/glossary/Glossary'; @@ -96,7 +99,9 @@ test.describe('Glossary Bulk Import Export', () => { propertyName, customPropertyData: entity, customType: property, - enumWithDescriptionConfig: entity.enumWithDescriptionConfig, + tableConfig: { + columns: FIELD_VALUES_CUSTOM_PROPERTIES.TABLE.columns, + }, }); } }); diff --git a/openmetadata-ui/src/main/resources/ui/playwright/utils/customProperty.ts b/openmetadata-ui/src/main/resources/ui/playwright/utils/customProperty.ts index 1133df89195a..5700f34343ed 100644 --- a/openmetadata-ui/src/main/resources/ui/playwright/utils/customProperty.ts +++ b/openmetadata-ui/src/main/resources/ui/playwright/utils/customProperty.ts @@ -57,7 +57,7 @@ export interface CustomProperty { }; } -export const fillTextInputDetails = async ( +export const fillTableColumnInputDetails = async ( page: Page, text: string, columnName: string @@ -230,9 +230,9 @@ export const setValueForProperty = async (data: { const values = value.split(','); await page.locator('[data-testid="add-new-row"]').click(); - await fillTextInputDetails(page, values[0], 'pw-column1'); + await fillTableColumnInputDetails(page, values[0], 'pw-column1'); - await fillTextInputDetails(page, values[1], 'pw-column2'); + await fillTableColumnInputDetails(page, values[1], 'pw-column2'); await page.locator('[data-testid="update-table-type-property"]').click(); diff --git a/openmetadata-ui/src/main/resources/ui/playwright/utils/importUtils.ts b/openmetadata-ui/src/main/resources/ui/playwright/utils/importUtils.ts index a720bc85c66b..ced6575cb1c1 100644 --- a/openmetadata-ui/src/main/resources/ui/playwright/utils/importUtils.ts +++ b/openmetadata-ui/src/main/resources/ui/playwright/utils/importUtils.ts @@ -16,6 +16,7 @@ import { FIELD_VALUES_CUSTOM_PROPERTIES, } from '../constant/glossaryImportExport'; import { descriptionBox, uuid } from './common'; +import { fillTableColumnInputDetails } from './customProperty'; export const createGlossaryTermRowDetails = () => { return { @@ -173,6 +174,19 @@ const editGlossaryCustomProperty = async ( await page.getByTestId('inline-save-btn').click(); } + + if (type === CUSTOM_PROPERTIES_TYPES.TABLE) { + const columns = FIELD_VALUES_CUSTOM_PROPERTIES.TABLE.columns; + const values = FIELD_VALUES_CUSTOM_PROPERTIES.TABLE.rows.split(','); + + await page.locator('[data-testid="add-new-row"]').click(); + + await fillTableColumnInputDetails(page, values[0], columns[0]); + + await fillTableColumnInputDetails(page, values[1], columns[1]); + + await page.locator('[data-testid="update-table-type-property"]').click(); + } }; export const fillCustomPropertyDetails = async ( diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Modals/ModalWithCustomProperty/ModalWithCustomPropertyEditor.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/Modals/ModalWithCustomProperty/ModalWithCustomPropertyEditor.component.tsx index 9312ecb7a71a..abd5b9b0810a 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Modals/ModalWithCustomProperty/ModalWithCustomPropertyEditor.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/Modals/ModalWithCustomProperty/ModalWithCustomPropertyEditor.component.tsx @@ -40,18 +40,21 @@ export const ModalWithCustomPropertyEditor = ({ visible, }: ModalWithCustomPropertyEditorProps) => { const { t } = useTranslation(); + const [isLoading, setIsLoading] = useState(false); const [isSaveLoading, setIsSaveLoading] = useState(false); - const [customPropertyValue, setCustomPropertyValue] = - useState(); - const [customPropertyTypes, setCustomPropertyTypes] = useState(); + + const [extensionObject, setExtensionObject] = useState(); + + const [customPropertyEntityRecord, setCustomPropertyEntityRecord] = + useState(); const fetchTypeDetail = async () => { setIsLoading(true); try { const response = await getTypeByFQN(entityType); - setCustomPropertyTypes(response); - setCustomPropertyValue( + setCustomPropertyEntityRecord(response); + setExtensionObject( convertCustomPropertyStringToEntityExtension(value ?? '', response) ); } catch (err) { @@ -65,15 +68,15 @@ export const ModalWithCustomPropertyEditor = ({ setIsSaveLoading(true); await onSave( convertEntityExtensionToCustomPropertyString( - customPropertyValue, - customPropertyTypes + extensionObject, + customPropertyEntityRecord ) ); setIsSaveLoading(false); }; const onExtensionUpdate = async (data: GlossaryTerm) => { - setCustomPropertyValue(data.extension); + setExtensionObject(data.extension); }; useEffect(() => { @@ -117,7 +120,7 @@ export const ModalWithCustomPropertyEditor = ({ hasEditAccess hasPermission isRenderedInRightPanel - entityDetails={{ extension: customPropertyValue } as GlossaryTerm} + entityDetails={{ extension: extensionObject } as GlossaryTerm} entityType={EntityType.GLOSSARY_TERM} handleExtensionUpdate={onExtensionUpdate} /> diff --git a/openmetadata-ui/src/main/resources/ui/src/components/Modals/ModalWithCustomProperty/ModalWithMarkdownEditor.interface.ts b/openmetadata-ui/src/main/resources/ui/src/components/Modals/ModalWithCustomProperty/ModalWithMarkdownEditor.interface.ts index 6696df303564..ae801b2e3969 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/Modals/ModalWithCustomProperty/ModalWithMarkdownEditor.interface.ts +++ b/openmetadata-ui/src/main/resources/ui/src/components/Modals/ModalWithCustomProperty/ModalWithMarkdownEditor.interface.ts @@ -12,13 +12,15 @@ */ import { EntityType } from '../../../enums/entity.enum'; import { EntityReference } from '../../../generated/entity/type'; +import { TableTypePropertyValueType } from '../../common/CustomPropertyTable/CustomPropertyTable.interface'; export type ExtensionDataTypes = | string | string[] | EntityReference | EntityReference[] - | { start: string; end: string }; + | { start: number; end: number } + | Partial; export interface ExtensionDataProps { [key: string]: ExtensionDataTypes; diff --git a/openmetadata-ui/src/main/resources/ui/src/components/common/CustomPropertyTable/property-value.less b/openmetadata-ui/src/main/resources/ui/src/components/common/CustomPropertyTable/property-value.less index 9e775878a355..be546896cc06 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/common/CustomPropertyTable/property-value.less +++ b/openmetadata-ui/src/main/resources/ui/src/components/common/CustomPropertyTable/property-value.less @@ -56,6 +56,7 @@ .ant-space-item:first-child { width: 100%; } + overflow-x: scroll; } .custom-property-card { .ant-card-body { diff --git a/openmetadata-ui/src/main/resources/ui/src/mocks/CSV.mock.ts b/openmetadata-ui/src/main/resources/ui/src/mocks/CSV.mock.ts index 1aa529cd30d4..848c75d4b19f 100644 --- a/openmetadata-ui/src/main/resources/ui/src/mocks/CSV.mock.ts +++ b/openmetadata-ui/src/main/resources/ui/src/mocks/CSV.mock.ts @@ -123,8 +123,8 @@ export const MOCK_GLOSSARY_TERM_CUSTOM_PROPERTIES_EXTENSION_OBJECT: ExtensionDat stringCp: 'select * from table where id="23";;', timeCp: '03:04:06', timerIntervalCp: { - end: '1727532820197', - start: '1727532807278', + end: 1727532820197, + start: 1727532807278, }, timeStampCp: '1727532807278', }; diff --git a/openmetadata-ui/src/main/resources/ui/src/utils/CSV/CSV.utils.test.tsx b/openmetadata-ui/src/main/resources/ui/src/utils/CSV/CSV.utils.test.tsx index 4bb7a40a4373..fd24dcee802e 100644 --- a/openmetadata-ui/src/main/resources/ui/src/utils/CSV/CSV.utils.test.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/utils/CSV/CSV.utils.test.tsx @@ -10,6 +10,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ +import { ExtensionDataProps } from '../../components/Modals/ModalWithCustomProperty/ModalWithMarkdownEditor.interface'; import { EntityType } from '../../enums/entity.enum'; import { MOCK_GLOSSARY_TERM_CUSTOM_PROPERTIES, @@ -163,5 +164,14 @@ describe('CSVUtils', () => { MOCK_GLOSSARY_TERM_CUSTOM_PROPERTIES_CONVERTED_EXTENSION_CSV_STRING ); }); + + it('should return string correctly which contains undefined as value for property', () => { + const convertedCSVEntities = convertEntityExtensionToCustomPropertyString( + { dateCp: undefined } as unknown as ExtensionDataProps, + MOCK_GLOSSARY_TERM_CUSTOM_PROPERTIES + ); + + expect(convertedCSVEntities).toStrictEqual(`dateCp:undefined`); + }); }); }); diff --git a/openmetadata-ui/src/main/resources/ui/src/utils/CSV/CSV.utils.tsx b/openmetadata-ui/src/main/resources/ui/src/utils/CSV/CSV.utils.tsx index 4852d04f1c21..c6f23d1d1987 100644 --- a/openmetadata-ui/src/main/resources/ui/src/utils/CSV/CSV.utils.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/utils/CSV/CSV.utils.tsx @@ -11,17 +11,27 @@ * limitations under the License. */ import { TypeColumn } from '@inovua/reactdatagrid-community/types'; -import { compact, get, isEmpty, isUndefined, startCase } from 'lodash'; +import { + compact, + get, + isEmpty, + isString, + isUndefined, + startCase, +} from 'lodash'; import React from 'react'; import { ReactComponent as SuccessBadgeIcon } from '../..//assets/svg/success-badge.svg'; import { ReactComponent as FailBadgeIcon } from '../../assets/svg/fail-badge.svg'; +import { TableTypePropertyValueType } from '../../components/common/CustomPropertyTable/CustomPropertyTable.interface'; import { ExtensionDataProps, ExtensionDataTypes, } from '../../components/Modals/ModalWithCustomProperty/ModalWithMarkdownEditor.interface'; +import { TABLE_TYPE_CUSTOM_PROPERTY } from '../../constants/CustomProperty.constants'; import { SEMICOLON_SPLITTER } from '../../constants/regex.constants'; import { EntityType } from '../../enums/entity.enum'; import { + Config, CustomProperty, EntityReference, Type, @@ -154,6 +164,12 @@ export const getCSVStringFromColumnsAndDataSource = ( return [header, ...compact(rows)].join('\n'); }; +/** + * + * @param value The value of the custom property in string format + * @param customProperty The custom property object + * @returns The value of the custom property in the correct type + */ const convertCustomPropertyStringToValueExtensionBasedOnType = ( value: string, customProperty?: CustomProperty @@ -182,6 +198,7 @@ const convertCustomPropertyStringToValueExtensionBasedOnType = ( } as EntityReference; }); } + case 'enum': { if (value.includes('|')) { return value.split('|'); @@ -194,15 +211,56 @@ const convertCustomPropertyStringToValueExtensionBasedOnType = ( const [start, end] = value.split(':'); return { - start, - end, + start: Number(start), + end: Number(end), + }; + } + + case TABLE_TYPE_CUSTOM_PROPERTY: { + // step 1: get the columns from the custom property config + const columns = + (customProperty?.customPropertyConfig?.config as Config)?.columns ?? []; + + // step 2: split the value by row + const rowStringList = value.split('|'); + + // step 3: convert the rowStringList into objects with column names as keys + const rows = rowStringList.map((row) => { + // Step 1: Replace commas inside double quotes with a placeholder + const preprocessedInput = row.replace(/"([^"]*)"/g, (_, p1) => { + return `${p1.replace(/,/g, '__COMMA__')}`; + }); + + // Step 2: Split the row by comma + const rowValues = preprocessedInput.split(','); + + // create an object with column names as keys + return columns.reduce((acc: Record, column, index) => { + // replace the placeholder with comma + acc[column] = rowValues[index].replaceAll('__COMMA__', ','); + + return acc; + }, {} as Record); + }); + + // return the columns and rows + return { + columns: columns, + rows: rows, }; } + default: return value; } }; +/** + * + * @param value The value of the custom property in object format + * @param customProperty The custom property object + * @returns The value of the custom property in string format + */ const convertCustomPropertyValueExtensionToStringBasedOnType = ( value: ExtensionDataTypes, customProperty: CustomProperty @@ -234,6 +292,30 @@ const convertCustomPropertyValueExtensionToStringBasedOnType = ( return `${interval.start}:${interval.end}`; } + case TABLE_TYPE_CUSTOM_PROPERTY: { + const tableTypeValue = value as TableTypePropertyValueType; + + // step 1: get the columns from the custom property config + const columns = tableTypeValue?.columns ?? []; + + // step 2: get the rows from the value + const rows = tableTypeValue?.rows ?? []; + + // step 3: convert the rows into a string + const rowStringList = rows.map((row) => { + return columns + .map((column) => { + const value = row[column] ?? ''; + + // if value contains comma, wrap it in quotes + return value.includes(',') ? `"${value}"` : value; + }) + .join(','); + }); + + return `${rowStringList.join('|')}`; + } + default: return value; } @@ -247,29 +329,34 @@ export const convertCustomPropertyStringToEntityExtension = ( return {}; } - const keyAndValueTypes: Record = {}; - - const result: ExtensionDataProps = {}; + // Step 1: Create a map of custom properties by name + const customPropertiesMapByName: Record = {}; customPropertyType.customProperties?.forEach( - (cp) => (keyAndValueTypes[cp.name] = cp) + (cp) => (customPropertiesMapByName[cp.name] = cp) ); - // Split the input into pairs using `;` and handle quoted strings properly + // Step 2: Split the input into pairs using `;` and handle quoted strings properly const pairs = value.split(SEMICOLON_SPLITTER); + // Step 3: Create a map of key-value pairs + const result: ExtensionDataProps = {}; + + // Step 4: Iterate over the pairs and convert them to key-value pairs pairs.forEach((pair) => { const cleanedText = removeOuterEscapes(pair); - const [key, ...valueParts] = cleanedText.split(':'); - const value = valueParts.join(':').trim(); // Join back in case of multiple `:` + const [propertyName, ...propertyValueParts] = cleanedText.split(':'); + const propertyValue = propertyValueParts.join(':').trim(); // Join back in case of multiple `:` + + const trimmedPropertyName = propertyName.trim(); // Clean up quotes if they are around the value - if (key && value) { - result[key.trim()] = + if (trimmedPropertyName && propertyValue) { + result[trimmedPropertyName] = convertCustomPropertyStringToValueExtensionBasedOnType( - value, - keyAndValueTypes[key] + propertyValue, + customPropertiesMapByName[trimmedPropertyName] ); } }); @@ -277,6 +364,12 @@ export const convertCustomPropertyStringToEntityExtension = ( return result; }; +/** + * + * @param value The value of the custom property in object format + * @param customPropertyType The custom property object + * @returns The value of the custom property in string format + */ export const convertEntityExtensionToCustomPropertyString = ( value?: ExtensionDataProps, customPropertyType?: Type @@ -285,37 +378,54 @@ export const convertEntityExtensionToCustomPropertyString = ( return; } - const keyAndValueTypes: Record = {}; + // Step 1: Create a map of custom properties by name + const customPropertiesMapByName: Record = {}; + customPropertyType?.customProperties?.forEach( - (cp) => (keyAndValueTypes[cp.name] = cp) + (cp) => (customPropertiesMapByName[cp.name] = cp) ); - let convertedString = ''; - + // Step 2: Convert the object into an array of key-value pairs const objectArray = Object.entries(value ?? {}); + // Step 3: Convert the key-value pairs into a string + let convertedString = ''; objectArray.forEach(([key, value], index) => { const isLastElement = objectArray.length - 1 === index; - if (keyAndValueTypes[key]) { + // Check if the key exists in the custom properties map + if (customPropertiesMapByName[key]) { + // Convert the value to a string based on the type const stringValue = convertCustomPropertyValueExtensionToStringBasedOnType( value, - keyAndValueTypes[key] + customPropertiesMapByName[key] ); + const endValue = isLastElement ? '' : ';'; + + const hasSeparator = + isString(stringValue) && + (stringValue.includes(',') || stringValue.includes(';')); + + // Check if the property type is markdown or sqlQuery or string and add quotes around the value if ( ['markdown', 'sqlQuery', 'string'].includes( - keyAndValueTypes[key].propertyType.name ?? '' - ) + customPropertiesMapByName[key]?.propertyType?.name ?? '' + ) && + hasSeparator ) { - convertedString += `"${`${key}:${stringValue}`}"${ - isLastElement ? '' : ';' - }`; + convertedString += `"${`${key}:${stringValue}`}"${endValue}`; + } else if ( + // Check if the property type is table and add quotes around the value + customPropertiesMapByName[key]?.propertyType?.name === + TABLE_TYPE_CUSTOM_PROPERTY + ) { + convertedString += `"${`${key}:${stringValue}`}"${endValue}`; } else { - convertedString += `${key}:${stringValue}${isLastElement ? '' : ';'}`; + convertedString += `${key}:${stringValue}${endValue}`; } } }); - return convertedString; + return `${convertedString}`; }; diff --git a/openmetadata-ui/src/main/resources/ui/src/utils/TagClassBase.test.ts b/openmetadata-ui/src/main/resources/ui/src/utils/TagClassBase.test.ts index 61efade41071..a393d968c229 100644 --- a/openmetadata-ui/src/main/resources/ui/src/utils/TagClassBase.test.ts +++ b/openmetadata-ui/src/main/resources/ui/src/utils/TagClassBase.test.ts @@ -16,6 +16,12 @@ import tagClassBase, { TagClassBase } from './TagClassBase'; jest.mock('../rest/searchAPI'); + +jest.mock('./StringsUtils', () => ({ + getEncodedFqn: jest.fn().mockReturnValue('test'), + escapeESReservedCharacters: jest.fn().mockReturnValue('test'), +})); + describe('TagClassBase', () => { beforeEach(() => { (searchQuery as jest.Mock).mockClear(); diff --git a/openmetadata-ui/src/main/resources/ui/src/utils/TagClassBase.ts b/openmetadata-ui/src/main/resources/ui/src/utils/TagClassBase.ts index f8cdf0d45b6c..a69c641afa8c 100644 --- a/openmetadata-ui/src/main/resources/ui/src/utils/TagClassBase.ts +++ b/openmetadata-ui/src/main/resources/ui/src/utils/TagClassBase.ts @@ -13,11 +13,14 @@ import { PAGE_SIZE } from '../constants/constants'; import { SearchIndex } from '../enums/search.enum'; import { searchQuery } from '../rest/searchAPI'; +import { escapeESReservedCharacters, getEncodedFqn } from './StringsUtils'; class TagClassBase { public async getTags(searchText: string, page: number) { + // this is to esacpe and encode any chars which is known by ES search internally + const encodedValue = getEncodedFqn(escapeESReservedCharacters(searchText)); const res = await searchQuery({ - query: `*${searchText}*`, + query: `*${encodedValue}*`, filters: 'disabled:false', pageNumber: page, pageSize: PAGE_SIZE,