MINOR - added test case results to search reindex (#18277)

* feat: added test case results to search reindex * fix: failing typescript test case
open-metadata · Oct 17, 2024 · dff704b · dff704b
1 parent 9c191a9
commit dff704b
Show file tree

Hide file tree

Showing 10 changed files with 109 additions and 43 deletions.
diff --git a/...rvice/src/main/java/org/openmetadata/service/apps/bundles/searchIndex/SearchIndexApp.java b/...rvice/src/main/java/org/openmetadata/service/apps/bundles/searchIndex/SearchIndexApp.java
@@ -1,6 +1,8 @@
 package org.openmetadata.service.apps.bundles.searchIndex;
 
 import static org.openmetadata.schema.system.IndexingError.ErrorSource.READER;
+import static org.openmetadata.service.Entity.TEST_CASE_RESOLUTION_STATUS;
+import static org.openmetadata.service.Entity.TEST_CASE_RESULT;
 import static org.openmetadata.service.apps.scheduler.AbstractOmAppJobListener.APP_RUN_STATS;
 import static org.openmetadata.service.apps.scheduler.AppScheduler.ON_DEMAND_JOB;
 import static org.openmetadata.service.workflows.searchIndex.ReindexingUtil.ENTITY_NAME_LIST_KEY;
@@ -90,6 +92,7 @@ public class SearchIndexApp extends AbstractNativeApplication {
           "storedProcedure",
           "storageService",
           "testCaseResolutionStatus",
+          "testCaseResult",
           "apiService",
           "apiEndpoint",
           "apiCollection",
@@ -101,7 +104,8 @@ public class SearchIndexApp extends AbstractNativeApplication {
           ReportData.ReportDataType.WEB_ANALYTIC_USER_ACTIVITY_REPORT_DATA.value(),
           ReportData.ReportDataType.WEB_ANALYTIC_ENTITY_VIEW_REPORT_DATA.value(),
           ReportData.ReportDataType.AGGREGATED_COST_ANALYSIS_REPORT_DATA.value(),
-          "testCaseResolutionStatus");
+          TEST_CASE_RESOLUTION_STATUS,
+          TEST_CASE_RESULT);
   private final List<Source> paginatedSources = new ArrayList<>();
   private Processor entityProcessor;
   private Processor entityTimeSeriesProcessor;
@@ -174,6 +178,8 @@ private void cleanUpStaleJobsFromRuns() {
   }
 
   private void initializeJob() {
+    List<Source> paginatedEntityTimeSeriesSources = new ArrayList<>();
+
     // Remove any Stale Jobs
     cleanUpStaleJobsFromRuns();
 
@@ -205,9 +211,11 @@ private void initializeJob() {
                 if (!CommonUtil.nullOrEmpty(jobData.getAfterCursor())) {
                   source.setCursor(jobData.getAfterCursor());
                 }
-                paginatedSources.add(source);
+                paginatedEntityTimeSeriesSources.add(source);
               }
             });
+    // Add Time Series Sources at the End of the List to Process them last
+    paginatedSources.addAll(paginatedEntityTimeSeriesSources);
     if (searchRepository.getSearchType().equals(ElasticSearchConfiguration.SearchType.OPENSEARCH)) {
       this.entityProcessor = new OpenSearchEntitiesProcessor(totalRecords);
       this.entityTimeSeriesProcessor = new OpenSearchEntityTimeSeriesProcessor(totalRecords);

diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/CollectionDAO.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/CollectionDAO.java
@@ -4582,6 +4582,30 @@ void insert(
         @Bind("json") String json,
         @Bind("incidentStateId") String incidentStateId);
 
+    @SqlQuery(
+        """
+              SELECT dqdts1.json FROM
+              data_quality_data_time_series dqdts1
+              INNER JOIN (
+                  SELECT tc.fqnHash
+                  FROM entity_relationship er
+                  INNER JOIN test_case tc ON er.toId = tc.id
+                  where fromEntity = 'testSuite' AND toEntity = 'testCase' and fromId = :testSuiteId
+              ) ts ON dqdts1.entityFQNHash = ts.fqnHash
+              LEFT JOIN data_quality_data_time_series dqdts2 ON
+                  (dqdts1.entityFQNHash = dqdts2.entityFQNHash and dqdts1.timestamp < dqdts2.timestamp)
+              WHERE dqdts2.entityFQNHash IS NULL""")
+    List<String> listLastTestCaseResultsForTestSuite(@BindMap Map<String, String> params);
+
+    @SqlQuery(
+        """
+            SELECT dqdts1.json FROM
+            data_quality_data_time_series dqdts1
+            LEFT JOIN data_quality_data_time_series dqdts2 ON
+                (dqdts1.entityFQNHash = dqdts2.entityFQNHash and dqdts1.timestamp < dqdts2.timestamp)
+            WHERE dqdts2.entityFQNHash IS NULL AND dqdts1.entityFQNHash = :testCaseFQN""")
+    String listLastTestCaseResult(@BindFQN("testCaseFQN") String testCaseFQN);
+
     default void insert(
         String testCaseFQN,
         String extension,
@@ -4597,6 +4621,10 @@ default void insert(
           json,
           incidentStateId != null ? incidentStateId.toString() : null);
     }
+
+    default List<String> listLastTestCaseResultsForTestSuite(UUID testSuiteId) {
+      return listLastTestCaseResultsForTestSuite(Map.of("testSuiteId", testSuiteId.toString()));
+    }
   }
 
   class EntitiesCountRowMapper implements RowMapper<EntitiesCount> {

diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TestCaseRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TestCaseRepository.java
@@ -12,6 +12,7 @@
 import static org.openmetadata.service.Entity.TEST_DEFINITION;
 import static org.openmetadata.service.Entity.TEST_SUITE;
 import static org.openmetadata.service.Entity.getEntityByName;
+import static org.openmetadata.service.Entity.getEntityTimeSeriesRepository;
 import static org.openmetadata.service.Entity.populateEntityFieldTags;
 import static org.openmetadata.service.exception.CatalogExceptionMessage.entityNotFound;
 import static org.openmetadata.service.security.mask.PIIMasker.maskSampleData;
@@ -379,6 +380,7 @@ private ResultSummary getResultSummary(
 
   @SneakyThrows
   private TestCaseResult getTestCaseResult(TestCase testCase) {
+    TestCaseResult testCaseResult;
     if (testCase.getTestCaseResult() != null) {
       // we'll return the saved state if it exists otherwise we'll fetch it from the database
       // Should be the case if listing from the search repo. as the test case result
@@ -387,10 +389,21 @@ private TestCaseResult getTestCaseResult(TestCase testCase) {
     }
     SearchListFilter searchListFilter = new SearchListFilter();
     searchListFilter.addQueryParam("testCaseFQN", testCase.getFullyQualifiedName());
-    EntityTimeSeriesRepository<?> timeSeriesRepository =
-        Entity.getEntityTimeSeriesRepository(TEST_CASE_RESULT);
-    return (TestCaseResult)
-        timeSeriesRepository.latestFromSearch(Fields.EMPTY_FIELDS, searchListFilter, null);
+    TestCaseResultRepository timeSeriesRepository =
+        (TestCaseResultRepository) getEntityTimeSeriesRepository(TEST_CASE_RESULT);
+    try {
+      testCaseResult =
+          timeSeriesRepository.latestFromSearch(Fields.EMPTY_FIELDS, searchListFilter, null);
+    } catch (Exception e) {
+      // Index may not exist in the search index (e.g. reindexing with recreate index on). Fall back
+      // to database
+      LOG.debug(
+          "Error fetching test case result from search. Fetching from test case results from database",
+          e);
+      testCaseResult =
+          timeSeriesRepository.listLastTestCaseResult(testCase.getFullyQualifiedName());
+    }
+    return testCaseResult;
   }
 
   public ResultList<TestCaseResult> getTestCaseResults(String fqn, Long startTs, Long endTs) {

diff --git a/...tadata-service/src/main/java/org/openmetadata/service/jdbi3/TestCaseResultRepository.java b/...tadata-service/src/main/java/org/openmetadata/service/jdbi3/TestCaseResultRepository.java
@@ -96,6 +96,21 @@ public Response addTestCaseResult(
     return Response.created(uriInfo.getRequestUri()).entity(testCaseResult).build();
   }
 
+  public ResultList<TestCaseResult> listLastTestCaseResultsForTestSuite(UUID testSuiteId) {
+    List<String> json =
+        ((CollectionDAO.TestCaseResultTimeSeriesDAO) timeSeriesDao)
+            .listLastTestCaseResultsForTestSuite(testSuiteId);
+    List<TestCaseResult> testCaseResults = JsonUtils.readObjects(json, TestCaseResult.class);
+    return new ResultList<>(testCaseResults, null, null, testCaseResults.size());
+  }
+
+  public TestCaseResult listLastTestCaseResult(String testCaseFQN) {
+    String json =
+        ((CollectionDAO.TestCaseResultTimeSeriesDAO) timeSeriesDao)
+            .listLastTestCaseResult(testCaseFQN);
+    return JsonUtils.readValue(json, TestCaseResult.class);
+  }
+
   @Override
   protected void postCreate(TestCaseResult entity) {
     super.postCreate(entity);

diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TestSuiteRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TestSuiteRepository.java
@@ -51,38 +51,6 @@ public class TestSuiteRepository extends EntityRepository<TestSuite> {
   private static final String UPDATE_FIELDS = "tests";
   private static final String PATCH_FIELDS = "tests";
 
-  private static final String EXECUTION_SUMMARY_AGGS =
-      """
-        {
-          "aggregations": {
-              "status_counts": {
-                "terms": {
-                  "field": "testCaseResult.testCaseStatus"
-                }
-              }
-            }
-        }
-        """;
-
-  private static final String ENTITY_EXECUTION_SUMMARY_AGGS =
-      """
-  {
-    "aggregations": {
-      "entityLinks": {
-        "terms": {
-          "field": "entityLink.nonNormalized"
-        },
-        "aggs": {
-          "status_counts": {
-            "terms": {
-              "field": "testCaseResult.testCaseStatus"
-            }
-          }
-        }
-      }
-    }
-  }""";
-
   private static final String ENTITY_EXECUTION_SUMMARY_FILTER =
       """
   {
@@ -275,14 +243,25 @@ public TestSummary getTestSummary(UUID testSuiteId) {
   @SneakyThrows
   private List<ResultSummary> getResultSummary(UUID testSuiteId) {
     List<ResultSummary> resultSummaries = new ArrayList<>();
+    ResultList<TestCaseResult> latestTestCaseResultResults;
     String groupBy = "testCaseFQN.keyword";
     SearchListFilter searchListFilter = new SearchListFilter();
     searchListFilter.addQueryParam("testSuiteId", testSuiteId.toString());
-    EntityTimeSeriesRepository<TestCaseResult> entityTimeSeriesRepository =
+    TestCaseResultRepository entityTimeSeriesRepository =
         (TestCaseResultRepository) getEntityTimeSeriesRepository(TEST_CASE_RESULT);
-    ResultList<TestCaseResult> latestTestCaseResultResults =
-        entityTimeSeriesRepository.listLatestFromSearch(
-            EntityUtil.Fields.EMPTY_FIELDS, searchListFilter, groupBy, null);
+    try {
+      latestTestCaseResultResults =
+          entityTimeSeriesRepository.listLatestFromSearch(
+              EntityUtil.Fields.EMPTY_FIELDS, searchListFilter, groupBy, null);
+    } catch (Exception e) {
+      // Index may not exist in the search index (e.g. reindexing with recreate index on). Fall back
+      // to database
+      LOG.debug(
+          "Error fetching test case result from search. Fetching from test case results from database",
+          e);
+      latestTestCaseResultResults =
+          entityTimeSeriesRepository.listLastTestCaseResultsForTestSuite(testSuiteId);
+    }
 
     latestTestCaseResultResults
         .getData()

diff --git a/openmetadata-service/src/main/resources/json/data/app/SearchIndexingApplication.json b/openmetadata-service/src/main/resources/json/data/app/SearchIndexingApplication.json
@@ -39,6 +39,7 @@
       "storedProcedure",
       "dataProduct",
       "testCaseResolutionStatus",
+      "testCaseResult",
       "apiService",
       "apiEndpoint",
       "apiCollection"

diff --git a/openmetadata-ui/src/main/resources/ui/src/enums/entity.enum.ts b/openmetadata-ui/src/main/resources/ui/src/enums/entity.enum.ts
@@ -67,6 +67,7 @@ export enum EntityType {
   WEB_ANALYTIC_ENTITY_VIEW_REPORT_DATA = 'webAnalyticEntityViewReportData',
   WEB_ANALYTIC_USER_ACTIVITY_REPORT_DATA = 'webAnalyticUserActivityReportData',
   TEST_CASE_RESOLUTION_STATUS = 'test_case_resolution_status_search_index',
+  TEST_CASE_RESULT = 'test_case_result_search_index',
   EVENT_SUBSCRIPTION = 'eventsubscription',
   LINEAGE_EDGE = 'lineageEdge',
   API_SERVICE = 'apiService',

diff --git a/openmetadata-ui/src/main/resources/ui/src/interface/search.interface.ts b/openmetadata-ui/src/main/resources/ui/src/interface/search.interface.ts
@@ -45,7 +45,7 @@ import { PipelineService } from '../generated/entity/services/pipelineService';
 import { SearchService } from '../generated/entity/services/searchService';
 import { Team } from '../generated/entity/teams/team';
 import { User } from '../generated/entity/teams/user';
-import { TestCase } from '../generated/tests/testCase';
+import { TestCase, TestCaseResult } from '../generated/tests/testCase';
 import { TestCaseResolutionStatus } from '../generated/tests/testCaseResolutionStatus';
 import { TestSuite } from '../generated/tests/testSuite';
 import { TagLabel } from '../generated/type/tagLabel';
@@ -145,6 +145,15 @@ export interface TestCaseResolutionStatusSearchSource
   serviceType: string;
   description: string;
 }
+export interface TestCaseResultSearchSource
+  extends SearchSourceBase,
+    TestCaseResult {
+  name: string;
+  displayName: string;
+  fullyQualifiedName: string;
+  serviceType: string;
+  description: string;
+}
 
 export interface IngestionPipelineSearchSource
   extends SearchSourceBase,

diff --git a/...data-ui/src/main/resources/ui/src/utils/ApplicationSchemas/SearchIndexingApplication.json b/...data-ui/src/main/resources/ui/src/utils/ApplicationSchemas/SearchIndexingApplication.json
@@ -59,6 +59,7 @@
           "storedProcedure",
           "dataProduct",
           "testCaseResolutionStatus",
+          "testCaseResult",
           "apiService",
           "apiEndpoint",
           "apiCollection",

diff --git a/openmetadata-ui/src/main/resources/ui/src/utils/mocks/ApplicationUtils.mock.ts b/openmetadata-ui/src/main/resources/ui/src/utils/mocks/ApplicationUtils.mock.ts
@@ -193,6 +193,11 @@ export const MOCK_APPLICATION_ENTITY_STATS = {
     failedRecords: 0,
     successRecords: 4,
   },
+  [EntityType.TEST_CASE_RESULT]: {
+    totalRecords: 4,
+    failedRecords: 0,
+    successRecords: 4,
+  },
 };
 
 export const MOCK_APPLICATION_ENTITY_STATS_DATA = [
@@ -412,4 +417,10 @@ export const MOCK_APPLICATION_ENTITY_STATS_DATA = [
     failedRecords: 0,
     successRecords: 4,
   },
+  {
+    name: EntityType.TEST_CASE_RESULT,
+    totalRecords: 4,
+    failedRecords: 0,
+    successRecords: 4,
+  },
 ];