[Data Explorer] Implement data fetch logic in Discover 2.0

Issue Resolve:
opensearch-project#4397

Signed-off-by: ananzh <[email protected]>

src/plugins/data/common/search/search_source/search_source.ts

@@ -81,6 +81,8 @@
  */
 
 import { setWith } from '@elastic/safer-lodash-set';
+import { Observable, from } from 'rxjs';
+import { switchMap, tap, of } from 'rxjs/operators';
 import { uniqueId, uniq, extend, pick, difference, omit, isObject, keys, isFunction } from 'lodash';
 import { normalizeSortRequest } from './normalize_sort_request';
 import { filterDocvalueFields } from './filter_docvalue_fields';
@@ -296,6 +298,39 @@ export class SearchSource {
     return response;
   }
 
+  /**
+   * Fetch this source and return Observable
+   */
+  fetch$(options: ISearchOptions = {}) {
+    const { getConfig } = this.dependencies;
+
+    return from(this.requestIsStarting(options)).pipe(
+      switchMap(() => this.flatten()),
+      tap((searchRequest) => {
+        this.history = [searchRequest];
+        let response$: Observable<any>;
+
+        if (getConfig(UI_SETTINGS.COURIER_BATCH_SEARCHES)) {
+          response$ = from(this.legacyFetch(searchRequest, options));
+        } else {
+          const indexPattern = this.getField('index');
+          searchRequest.dataSourceId = indexPattern?.dataSourceRef?.id;
+
+          response$ = from(this.fetchSearch(searchRequest, options));
+        }
+
+        return response$;
+      }),
+      switchMap((response) => {
+        if ((response as any).error) {
+          throw new RequestFailure(null, response);
+        }
+
+        return of(response);
+      })
+    );
+  }
+
   /**
    *  Add a handler that will be notified whenever requests start
    *  @param  {Function} handler

src/plugins/discover/public/application/view_components/canvas/canvas.tsx

@@ -8,6 +8,7 @@ import { AppMountParameters } from '../../../../../../core/public';
 import { useOpenSearchDashboards } from '../../../../../opensearch_dashboards_react/public';
 import { DiscoverServices } from '../../../build_services';
 import { TopNav } from './top_nav';
+import { DiscoverTable } from './discover_table';
 
 interface CanvasProps {
   opts: {
@@ -17,11 +18,12 @@ interface CanvasProps {
 
 export const Canvas = ({ opts }: CanvasProps) => {
   const { services } = useOpenSearchDashboards<DiscoverServices>();
-
+  const { history: getHistory } = services;
+  const history = getHistory();
   return (
     <div>
       <TopNav opts={opts} />
-      Canvas
+      <DiscoverTable services={services} history={history} />
     </div>
   );
 };

src/plugins/discover/public/application/view_components/canvas/discover_table.tsx

@@ -0,0 +1,66 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+import React, { useState, useEffect } from 'react';
+import { History } from 'history';
+import { IndexPattern } from '../../../opensearch_dashboards_services';
+import { DiscoverServices } from '../../../build_services';
+import { SavedSearch } from '../../../saved_searches';
+import { DiscoverTableService } from './discover_table_service';
+import { fetchIndexPattern, fetchSavedSearch } from '../utils/index_pattern_helper';
+
+export interface DiscoverTableProps {
+  services: DiscoverServices;
+  history: History;
+}
+
+export const DiscoverTable = ({ history, services }: DiscoverTableProps) => {
+  const { core, chrome, data, uiSettings: config, toastNotifications } = services;
+  const [savedSearch, setSavedSearch] = useState<SavedSearch>();
+  const [indexPattern, setIndexPattern] = useState<IndexPattern | undefined>(undefined);
+  // ToDo: get id from data explorer since it is handling the routing logic
+  // Original angular code: const savedSearchId = $route.current.params.id;
+  const savedSearchId = '';
+  useEffect(() => {
+    const fetchData = async () => {
+      const indexPatternData = await fetchIndexPattern(data, config);
+      setIndexPattern(indexPatternData.loaded);
+
+      const savedSearchData = await fetchSavedSearch(
+        core,
+        '', // basePath
+        history,
+        savedSearchId,
+        services,
+        toastNotifications
+      );
+      if (savedSearchData && !savedSearchData?.searchSource.getField('index')) {
+        savedSearchData.searchSource.setField('index', indexPatternData);
+      }
+      setSavedSearch(savedSearchData);
+
+      if (savedSearchId) {
+        chrome.recentlyAccessed.add(
+          savedSearchData.getFullPath(),
+          savedSearchData.title,
+          savedSearchData.id
+        );
+      }
+    };
+    fetchData();
+  }, [data, config, core, chrome, toastNotifications, history, savedSearchId, services]);
+
+  if (!savedSearch || !savedSearch.searchSource || !indexPattern) {
+    // ToDo: handle loading state
+    return null;
+  }
+  return (
+    <DiscoverTableService
+      services={services}
+      savedSearch={savedSearch}
+      indexPattern={indexPattern}
+    />
+  );
+};

src/plugins/discover/public/application/view_components/canvas/discover_table_app.tsx

@@ -0,0 +1,66 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+import React, { useState, useEffect } from 'react';
+import { EuiPage, EuiPageBody, EuiFlexGroup, EuiFlexItem, EuiPageContent } from '@elastic/eui';
+import { DataGridTable } from '../../components/data_grid/data_grid_table';
+
+export const DiscoverTableApplication = ({ data$, indexPattern, savedSearch, services }) => {
+  const [fetchState, setFetchState] = useState<any>({
+    status: data$.getValue().status,
+    fetchCounter: 0,
+    fieldCounts: {},
+    rows: [],
+  });
+
+  const { rows } = fetchState;
+
+  useEffect(() => {
+    const subscription = data$.subscribe((next) => {
+      if (
+        (next.status && next.status !== fetchState.status) ||
+        (next.rows && next.rows !== fetchState.rows)
+      ) {
+        setFetchState({ ...fetchState, ...next });
+      }
+    });
+    return () => {
+      subscription.unsubscribe();
+    };
+  }, [data$, fetchState]);
+
+  // ToDo: implement columns, onAddColumn, onRemoveColumn, onMoveColumn, onSetColumns using config, indexPattern, appState
+
+  if (rows.length === 0) {
+    return <div>{'loading...'}</div>;
+  } else {
+    return (
+      <EuiPage className="dscCanvasAppPage">
+        <EuiPageBody className="dscCanvasAppPageBody">
+          <EuiFlexGroup className="dscCanvasAppPageBody__contents">
+            <EuiFlexItem>
+              <EuiPageContent>
+                <div className="dscDiscoverGrid">
+                  <DataGridTable
+                    columns={['_source']}
+                    indexPattern={indexPattern}
+                    onAddColumn={() => {}}
+                    onFilter={() => {}}
+                    onRemoveColumn={() => {}}
+                    onSetColumns={() => {}}
+                    onSort={() => {}}
+                    sort={[]}
+                    rows={rows}
+                    displayTimeColumn={true}
+                    services={services}
+                  />
+                </div>
+              </EuiPageContent>
+            </EuiFlexItem>
+          </EuiFlexGroup>
+        </EuiPageBody>
+      </EuiPage>
+    );
+  }
+};

src/plugins/discover/public/application/view_components/canvas/discover_table_service.tsx

@@ -0,0 +1,49 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+import React, { useEffect } from 'react';
+import { IndexPattern } from '../../../opensearch_dashboards_services';
+import { DiscoverServices } from '../../../build_services';
+import { SavedSearch } from '../../../saved_searches';
+import { useDiscoverTableService } from './utils/use_discover_canvas_service';
+import { DiscoverTableApplication } from './discover_table_app';
+
+export interface DiscoverTableAppProps {
+  services: DiscoverServices;
+  savedSearch: SavedSearch;
+  indexPattern: IndexPattern;
+}
+
+export const DiscoverTableService = ({
+  services,
+  savedSearch,
+  indexPattern,
+}: DiscoverTableAppProps) => {
+  const { data$, refetch$ } = useDiscoverTableService({
+    services,
+    savedSearch,
+    indexPattern,
+  });
+
+  // trigger manual fetch
+  // ToDo: remove this once we implement refetch data:
+  // Based on the controller, refetch$ should emit next when
+  // 1) appStateContainer interval and sort change
+  // 2) savedSearch id changes
+  // 3) timefilter.getRefreshInterval().pause === false
+  // 4) TopNavMenu updateQuery() is called
+  useEffect(() => {
+    refetch$.next();
+  }, [refetch$]);
+
+  return (
+    <DiscoverTableApplication
+      data$={data$}
+      indexPattern={indexPattern}
+      savedSearch={savedSearch}
+      services={services}
+    />
+  );
+};

src/plugins/discover/public/application/view_components/canvas/utils/use_discover_canvas_service.ts

@@ -0,0 +1,49 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+import { useMemo, useEffect } from 'react';
+import { DiscoverServices } from '../../../../build_services';
+import { SavedSearch } from '../../../../saved_searches';
+import { useSavedSearch } from '../../utils/use_saved_search';
+import { IndexPattern } from '../../../../opensearch_dashboards_services';
+
+export interface DiscoverTableServiceProps {
+  services: DiscoverServices;
+  savedSearch: SavedSearch;
+  indexPattern: IndexPattern;
+}
+
+export const useDiscoverTableService = ({
+  services,
+  savedSearch,
+  indexPattern,
+}: DiscoverTableServiceProps) => {
+  const searchSource = useMemo(() => {
+    savedSearch.searchSource.setField('index', indexPattern);
+    return savedSearch.searchSource;
+  }, [savedSearch, indexPattern]);
+
+  const { data$, refetch$ } = useSavedSearch({
+    searchSource,
+    services,
+    indexPattern,
+  });
+
+  useEffect(() => {
+    const dataSubscription = data$.subscribe((data) => {});
+    const refetchSubscription = refetch$.subscribe((refetch) => {});
+
+    return () => {
+      dataSubscription.unsubscribe();
+      refetchSubscription.unsubscribe();
+    };
+  }, [data$, refetch$]);
+
+  return {
+    data$,
+    refetch$,
+    indexPattern,
+  };
+};

src/plugins/discover/public/application/view_components/utils/get_sort_for_search_source.ts

@@ -0,0 +1,65 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ *
+ * Any modifications Copyright OpenSearch Contributors. See
+ * GitHub history for details.
+ */
+
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import { OpenSearchQuerySortValue, IndexPattern } from '../../../../opensearch_dashboards_services';
+import { SortOrder } from '../components/table_header/helpers';
+import { getSort } from './get_sort';
+import { getDefaultSort } from './get_default_sort';
+
+/**
+ * Prepares sort for search source, that's sending the request to OpenSearch
+ * - Adds default sort if necessary
+ * - Handles the special case when there's sorting by date_nanos typed fields
+ *   the addon of the numeric_type guarantees the right sort order
+ *   when there are indices with date and indices with date_nanos field
+ */
+export function getSortForSearchSource(
+  sort?: SortOrder[],
+  indexPattern?: IndexPattern,
+  defaultDirection: string = 'desc'
+): OpenSearchQuerySortValue[] {
+  if (!sort || !indexPattern) {
+    return [];
+  } else if (Array.isArray(sort) && sort.length === 0) {
+    sort = getDefaultSort(indexPattern, defaultDirection);
+  }
+  const { timeFieldName } = indexPattern;
+  return getSort(sort, indexPattern).map((sortPair: Record<string, string>) => {
+    if (indexPattern.isTimeNanosBased() && timeFieldName && sortPair[timeFieldName]) {
+      return {
+        [timeFieldName]: {
+          order: sortPair[timeFieldName],
+          numeric_type: 'date_nanos',
+        },
+      } as OpenSearchQuerySortValue;
+    }
+    return sortPair as OpenSearchQuerySortValue;
+  });
+}