diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/searchIndex/SearchIndexApp.java b/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/searchIndex/SearchIndexApp.java index adb45d380433..244ea2b33ec5 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/searchIndex/SearchIndexApp.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/searchIndex/SearchIndexApp.java @@ -1,6 +1,8 @@ package org.openmetadata.service.apps.bundles.searchIndex; import static org.openmetadata.schema.system.IndexingError.ErrorSource.READER; +import static org.openmetadata.service.Entity.TEST_CASE_RESOLUTION_STATUS; +import static org.openmetadata.service.Entity.TEST_CASE_RESULT; import static org.openmetadata.service.apps.scheduler.AbstractOmAppJobListener.APP_RUN_STATS; import static org.openmetadata.service.apps.scheduler.AppScheduler.ON_DEMAND_JOB; import static org.openmetadata.service.workflows.searchIndex.ReindexingUtil.ENTITY_NAME_LIST_KEY; @@ -90,6 +92,7 @@ public class SearchIndexApp extends AbstractNativeApplication { "storedProcedure", "storageService", "testCaseResolutionStatus", + "testCaseResult", "apiService", "apiEndpoint", "apiCollection", @@ -101,7 +104,8 @@ public class SearchIndexApp extends AbstractNativeApplication { ReportData.ReportDataType.WEB_ANALYTIC_USER_ACTIVITY_REPORT_DATA.value(), ReportData.ReportDataType.WEB_ANALYTIC_ENTITY_VIEW_REPORT_DATA.value(), ReportData.ReportDataType.AGGREGATED_COST_ANALYSIS_REPORT_DATA.value(), - "testCaseResolutionStatus"); + TEST_CASE_RESOLUTION_STATUS, + TEST_CASE_RESULT); private final List paginatedSources = new ArrayList<>(); private Processor entityProcessor; private Processor entityTimeSeriesProcessor; @@ -174,6 +178,8 @@ private void cleanUpStaleJobsFromRuns() { } private void initializeJob() { + List paginatedEntityTimeSeriesSources = new ArrayList<>(); + // Remove any Stale Jobs cleanUpStaleJobsFromRuns(); @@ -205,9 +211,11 @@ private void initializeJob() { if (!CommonUtil.nullOrEmpty(jobData.getAfterCursor())) { source.setCursor(jobData.getAfterCursor()); } - paginatedSources.add(source); + paginatedEntityTimeSeriesSources.add(source); } }); + // Add Time Series Sources at the End of the List to Process them last + paginatedSources.addAll(paginatedEntityTimeSeriesSources); if (searchRepository.getSearchType().equals(ElasticSearchConfiguration.SearchType.OPENSEARCH)) { this.entityProcessor = new OpenSearchEntitiesProcessor(totalRecords); this.entityTimeSeriesProcessor = new OpenSearchEntityTimeSeriesProcessor(totalRecords); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/CollectionDAO.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/CollectionDAO.java index bd18e717eb70..a5a471df3667 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/CollectionDAO.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/CollectionDAO.java @@ -4582,6 +4582,30 @@ void insert( @Bind("json") String json, @Bind("incidentStateId") String incidentStateId); + @SqlQuery( + """ + SELECT dqdts1.json FROM + data_quality_data_time_series dqdts1 + INNER JOIN ( + SELECT tc.fqnHash + FROM entity_relationship er + INNER JOIN test_case tc ON er.toId = tc.id + where fromEntity = 'testSuite' AND toEntity = 'testCase' and fromId = :testSuiteId + ) ts ON dqdts1.entityFQNHash = ts.fqnHash + LEFT JOIN data_quality_data_time_series dqdts2 ON + (dqdts1.entityFQNHash = dqdts2.entityFQNHash and dqdts1.timestamp < dqdts2.timestamp) + WHERE dqdts2.entityFQNHash IS NULL""") + List listLastTestCaseResultsForTestSuite(@BindMap Map params); + + @SqlQuery( + """ + SELECT dqdts1.json FROM + data_quality_data_time_series dqdts1 + LEFT JOIN data_quality_data_time_series dqdts2 ON + (dqdts1.entityFQNHash = dqdts2.entityFQNHash and dqdts1.timestamp < dqdts2.timestamp) + WHERE dqdts2.entityFQNHash IS NULL AND dqdts1.entityFQNHash = :testCaseFQN""") + String listLastTestCaseResult(@BindFQN("testCaseFQN") String testCaseFQN); + default void insert( String testCaseFQN, String extension, @@ -4597,6 +4621,10 @@ default void insert( json, incidentStateId != null ? incidentStateId.toString() : null); } + + default List listLastTestCaseResultsForTestSuite(UUID testSuiteId) { + return listLastTestCaseResultsForTestSuite(Map.of("testSuiteId", testSuiteId.toString())); + } } class EntitiesCountRowMapper implements RowMapper { diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TestCaseRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TestCaseRepository.java index 64e0228cdf13..40d15e09a1a5 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TestCaseRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TestCaseRepository.java @@ -12,6 +12,7 @@ import static org.openmetadata.service.Entity.TEST_DEFINITION; import static org.openmetadata.service.Entity.TEST_SUITE; import static org.openmetadata.service.Entity.getEntityByName; +import static org.openmetadata.service.Entity.getEntityTimeSeriesRepository; import static org.openmetadata.service.Entity.populateEntityFieldTags; import static org.openmetadata.service.exception.CatalogExceptionMessage.entityNotFound; import static org.openmetadata.service.security.mask.PIIMasker.maskSampleData; @@ -379,6 +380,7 @@ private ResultSummary getResultSummary( @SneakyThrows private TestCaseResult getTestCaseResult(TestCase testCase) { + TestCaseResult testCaseResult; if (testCase.getTestCaseResult() != null) { // we'll return the saved state if it exists otherwise we'll fetch it from the database // Should be the case if listing from the search repo. as the test case result @@ -387,10 +389,21 @@ private TestCaseResult getTestCaseResult(TestCase testCase) { } SearchListFilter searchListFilter = new SearchListFilter(); searchListFilter.addQueryParam("testCaseFQN", testCase.getFullyQualifiedName()); - EntityTimeSeriesRepository timeSeriesRepository = - Entity.getEntityTimeSeriesRepository(TEST_CASE_RESULT); - return (TestCaseResult) - timeSeriesRepository.latestFromSearch(Fields.EMPTY_FIELDS, searchListFilter, null); + TestCaseResultRepository timeSeriesRepository = + (TestCaseResultRepository) getEntityTimeSeriesRepository(TEST_CASE_RESULT); + try { + testCaseResult = + timeSeriesRepository.latestFromSearch(Fields.EMPTY_FIELDS, searchListFilter, null); + } catch (Exception e) { + // Index may not exist in the search index (e.g. reindexing with recreate index on). Fall back + // to database + LOG.debug( + "Error fetching test case result from search. Fetching from test case results from database", + e); + testCaseResult = + timeSeriesRepository.listLastTestCaseResult(testCase.getFullyQualifiedName()); + } + return testCaseResult; } public ResultList getTestCaseResults(String fqn, Long startTs, Long endTs) { diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TestCaseResultRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TestCaseResultRepository.java index d9c937fd8073..7af2339fb0f7 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TestCaseResultRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TestCaseResultRepository.java @@ -96,6 +96,21 @@ public Response addTestCaseResult( return Response.created(uriInfo.getRequestUri()).entity(testCaseResult).build(); } + public ResultList listLastTestCaseResultsForTestSuite(UUID testSuiteId) { + List json = + ((CollectionDAO.TestCaseResultTimeSeriesDAO) timeSeriesDao) + .listLastTestCaseResultsForTestSuite(testSuiteId); + List testCaseResults = JsonUtils.readObjects(json, TestCaseResult.class); + return new ResultList<>(testCaseResults, null, null, testCaseResults.size()); + } + + public TestCaseResult listLastTestCaseResult(String testCaseFQN) { + String json = + ((CollectionDAO.TestCaseResultTimeSeriesDAO) timeSeriesDao) + .listLastTestCaseResult(testCaseFQN); + return JsonUtils.readValue(json, TestCaseResult.class); + } + @Override protected void postCreate(TestCaseResult entity) { super.postCreate(entity); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TestSuiteRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TestSuiteRepository.java index 32656809143e..61654fb453fa 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TestSuiteRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TestSuiteRepository.java @@ -51,38 +51,6 @@ public class TestSuiteRepository extends EntityRepository { private static final String UPDATE_FIELDS = "tests"; private static final String PATCH_FIELDS = "tests"; - private static final String EXECUTION_SUMMARY_AGGS = - """ - { - "aggregations": { - "status_counts": { - "terms": { - "field": "testCaseResult.testCaseStatus" - } - } - } - } - """; - - private static final String ENTITY_EXECUTION_SUMMARY_AGGS = - """ - { - "aggregations": { - "entityLinks": { - "terms": { - "field": "entityLink.nonNormalized" - }, - "aggs": { - "status_counts": { - "terms": { - "field": "testCaseResult.testCaseStatus" - } - } - } - } - } - }"""; - private static final String ENTITY_EXECUTION_SUMMARY_FILTER = """ { @@ -275,14 +243,25 @@ public TestSummary getTestSummary(UUID testSuiteId) { @SneakyThrows private List getResultSummary(UUID testSuiteId) { List resultSummaries = new ArrayList<>(); + ResultList latestTestCaseResultResults; String groupBy = "testCaseFQN.keyword"; SearchListFilter searchListFilter = new SearchListFilter(); searchListFilter.addQueryParam("testSuiteId", testSuiteId.toString()); - EntityTimeSeriesRepository entityTimeSeriesRepository = + TestCaseResultRepository entityTimeSeriesRepository = (TestCaseResultRepository) getEntityTimeSeriesRepository(TEST_CASE_RESULT); - ResultList latestTestCaseResultResults = - entityTimeSeriesRepository.listLatestFromSearch( - EntityUtil.Fields.EMPTY_FIELDS, searchListFilter, groupBy, null); + try { + latestTestCaseResultResults = + entityTimeSeriesRepository.listLatestFromSearch( + EntityUtil.Fields.EMPTY_FIELDS, searchListFilter, groupBy, null); + } catch (Exception e) { + // Index may not exist in the search index (e.g. reindexing with recreate index on). Fall back + // to database + LOG.debug( + "Error fetching test case result from search. Fetching from test case results from database", + e); + latestTestCaseResultResults = + entityTimeSeriesRepository.listLastTestCaseResultsForTestSuite(testSuiteId); + } latestTestCaseResultResults .getData() diff --git a/openmetadata-service/src/main/resources/json/data/app/SearchIndexingApplication.json b/openmetadata-service/src/main/resources/json/data/app/SearchIndexingApplication.json index 3169c3634a42..d853ea037dc9 100644 --- a/openmetadata-service/src/main/resources/json/data/app/SearchIndexingApplication.json +++ b/openmetadata-service/src/main/resources/json/data/app/SearchIndexingApplication.json @@ -39,6 +39,7 @@ "storedProcedure", "dataProduct", "testCaseResolutionStatus", + "testCaseResult", "apiService", "apiEndpoint", "apiCollection" diff --git a/openmetadata-ui/src/main/resources/ui/src/enums/entity.enum.ts b/openmetadata-ui/src/main/resources/ui/src/enums/entity.enum.ts index 41ce4bffd6c3..aeb7dfcc8195 100644 --- a/openmetadata-ui/src/main/resources/ui/src/enums/entity.enum.ts +++ b/openmetadata-ui/src/main/resources/ui/src/enums/entity.enum.ts @@ -67,6 +67,7 @@ export enum EntityType { WEB_ANALYTIC_ENTITY_VIEW_REPORT_DATA = 'webAnalyticEntityViewReportData', WEB_ANALYTIC_USER_ACTIVITY_REPORT_DATA = 'webAnalyticUserActivityReportData', TEST_CASE_RESOLUTION_STATUS = 'test_case_resolution_status_search_index', + TEST_CASE_RESULT = 'test_case_result_search_index', EVENT_SUBSCRIPTION = 'eventsubscription', LINEAGE_EDGE = 'lineageEdge', API_SERVICE = 'apiService', diff --git a/openmetadata-ui/src/main/resources/ui/src/interface/search.interface.ts b/openmetadata-ui/src/main/resources/ui/src/interface/search.interface.ts index b4cb53d84b53..4dc9971edb48 100644 --- a/openmetadata-ui/src/main/resources/ui/src/interface/search.interface.ts +++ b/openmetadata-ui/src/main/resources/ui/src/interface/search.interface.ts @@ -45,7 +45,7 @@ import { PipelineService } from '../generated/entity/services/pipelineService'; import { SearchService } from '../generated/entity/services/searchService'; import { Team } from '../generated/entity/teams/team'; import { User } from '../generated/entity/teams/user'; -import { TestCase } from '../generated/tests/testCase'; +import { TestCase, TestCaseResult } from '../generated/tests/testCase'; import { TestCaseResolutionStatus } from '../generated/tests/testCaseResolutionStatus'; import { TestSuite } from '../generated/tests/testSuite'; import { TagLabel } from '../generated/type/tagLabel'; @@ -145,6 +145,15 @@ export interface TestCaseResolutionStatusSearchSource serviceType: string; description: string; } +export interface TestCaseResultSearchSource + extends SearchSourceBase, + TestCaseResult { + name: string; + displayName: string; + fullyQualifiedName: string; + serviceType: string; + description: string; +} export interface IngestionPipelineSearchSource extends SearchSourceBase, diff --git a/openmetadata-ui/src/main/resources/ui/src/utils/ApplicationSchemas/SearchIndexingApplication.json b/openmetadata-ui/src/main/resources/ui/src/utils/ApplicationSchemas/SearchIndexingApplication.json index 69e218615f5a..928d0c1a2707 100644 --- a/openmetadata-ui/src/main/resources/ui/src/utils/ApplicationSchemas/SearchIndexingApplication.json +++ b/openmetadata-ui/src/main/resources/ui/src/utils/ApplicationSchemas/SearchIndexingApplication.json @@ -59,6 +59,7 @@ "storedProcedure", "dataProduct", "testCaseResolutionStatus", + "testCaseResult", "apiService", "apiEndpoint", "apiCollection", diff --git a/openmetadata-ui/src/main/resources/ui/src/utils/mocks/ApplicationUtils.mock.ts b/openmetadata-ui/src/main/resources/ui/src/utils/mocks/ApplicationUtils.mock.ts index 359f7c8cf87c..ffbcefbaad73 100644 --- a/openmetadata-ui/src/main/resources/ui/src/utils/mocks/ApplicationUtils.mock.ts +++ b/openmetadata-ui/src/main/resources/ui/src/utils/mocks/ApplicationUtils.mock.ts @@ -193,6 +193,11 @@ export const MOCK_APPLICATION_ENTITY_STATS = { failedRecords: 0, successRecords: 4, }, + [EntityType.TEST_CASE_RESULT]: { + totalRecords: 4, + failedRecords: 0, + successRecords: 4, + }, }; export const MOCK_APPLICATION_ENTITY_STATS_DATA = [ @@ -412,4 +417,10 @@ export const MOCK_APPLICATION_ENTITY_STATS_DATA = [ failedRecords: 0, successRecords: 4, }, + { + name: EntityType.TEST_CASE_RESULT, + totalRecords: 4, + failedRecords: 0, + successRecords: 4, + }, ];