Mark unmanaged indices for remote reindexing (#19976)

* mark unmanaged indices for remote reindexing * cl * remove duplicate remote indices --------- Co-authored-by: Mohamed OULD HOCINE <[email protected]>
Graylog2 · Jul 24, 2024 · 6d7a6a8 · 6d7a6a8
1 parent 4cf18e3
commit 6d7a6a8
Show file tree

Hide file tree

Showing 5 changed files with 63 additions and 17 deletions.
diff --git a/changelog/unreleased/pr-19976.toml b/changelog/unreleased/pr-19976.toml
@@ -0,0 +1,5 @@
+type = "a"
+message = "Indices not managed by Graylog are now marked and not selected by default in data node's remote reindex migration."
+
+pulls = ["19976"]
+issues = ["19974"]
diff --git a/...h2/src/main/java/org/graylog/storage/opensearch2/RemoteReindexingMigrationAdapterOS2.java b/...h2/src/main/java/org/graylog/storage/opensearch2/RemoteReindexingMigrationAdapterOS2.java
@@ -59,6 +59,7 @@
 import org.graylog2.indexer.migration.IndexerConnectionCheckResult;
 import org.graylog2.indexer.migration.LogEntry;
 import org.graylog2.indexer.migration.LogLevel;
+import org.graylog2.indexer.migration.RemoteIndex;
 import org.graylog2.indexer.migration.RemoteReindexIndex;
 import org.graylog2.indexer.migration.RemoteReindexMigration;
 import org.graylog2.indexer.migration.TaskStatus;
@@ -229,10 +230,14 @@ public IndexerConnectionCheckResult checkConnection(@Nonnull URI remoteHost, @Nu
             final RemoteReindexAllowlist reindexAllowlist = new RemoteReindexAllowlist(remoteHost, allowlist);
             reindexAllowlist.validate();
             final AggregatedConnectionResponse results = getAllIndicesFrom(remoteHost, username, password, trustUnknownCerts);
+            final List<RemoteIndex> indices = results.indices().stream()
+                    .map(i -> new RemoteIndex(i, indexSetRegistry.isManagedIndex(i)))
+                    .distinct()
+                    .toList();
             if (results.error() != null && !results.error().isEmpty()) {
                 return IndexerConnectionCheckResult.failure(results.error());
             } else {
-                return IndexerConnectionCheckResult.success(results.indices());
+                return IndexerConnectionCheckResult.success(indices);
             }
         } catch (Exception e) {
             return IndexerConnectionCheckResult.failure(e);

diff --git a/...og2-server/src/main/java/org/graylog2/indexer/migration/IndexerConnectionCheckResult.java b/...og2-server/src/main/java/org/graylog2/indexer/migration/IndexerConnectionCheckResult.java
@@ -19,9 +19,9 @@
 import java.util.Collections;
 import java.util.List;
 
-public record IndexerConnectionCheckResult(List<String> indices, String error) {
+public record IndexerConnectionCheckResult(List<RemoteIndex> indices, String error) {
 
-    public static IndexerConnectionCheckResult success(List<String> indexNames) {
+    public static IndexerConnectionCheckResult success(List<RemoteIndex> indexNames) {
         return new IndexerConnectionCheckResult(indexNames, null);
     }
 

diff --git a/graylog2-server/src/main/java/org/graylog2/indexer/migration/RemoteIndex.java b/graylog2-server/src/main/java/org/graylog2/indexer/migration/RemoteIndex.java
@@ -0,0 +1,20 @@
+/*
+ * Copyright (C) 2020 Graylog, Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the Server Side Public License, version 1,
+ * as published by MongoDB, Inc.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * Server Side Public License for more details.
+ *
+ * You should have received a copy of the Server Side Public License
+ * along with this program. If not, see
+ * <http://www.mongodb.com/licensing/server-side-public-license>.
+ */
+package org.graylog2.indexer.migration;
+
+public record RemoteIndex(String name, boolean managed) {
+}
diff --git a/...web-interface/src/components/datanode/migrations/remoteReindexing/MigrateExistingData.tsx b/...web-interface/src/components/datanode/migrations/remoteReindexing/MigrateExistingData.tsx
@@ -16,12 +16,12 @@
  */
 import * as React from 'react';
 import { useState } from 'react';
-import { Formik, Form } from 'formik';
 import type { FormikErrors } from 'formik';
+import { Formik, Form } from 'formik';
 import styled from 'styled-components';
 
 import { Alert, Input, Row, Col } from 'components/bootstrap';
-import { SearchForm, Spinner } from 'components/common';
+import { SearchForm, Spinner, Icon } from 'components/common';
 import { getValueFromInput } from 'util/FormsUtils';
 
 import type { RemoteReindexRequest } from '../../hooks/useRemoteReindexMigrationStatus';
@@ -41,17 +41,22 @@ const SearchContainer = styled.div`
   margin-top: 12px;
 `;
 
+export type RemoteIndex = {
+  name: string,
+  managed: boolean,
+}
+
 export type RemoteReindexCheckConnection = {
-  indices: string[],
+  indices: RemoteIndex[],
   error: any,
 }
 
 const MigrateExistingData = ({ currentStep, onTriggerStep, hideActions }: MigrationStepComponentProps) => {
   const [nextSteps, setNextSteps] = useState<MigrationActions[]>(['CHECK_REMOTE_INDEXER_CONNECTION']);
   const [errorMessage, setErrrorMessage] = useState<string | null>(null);
   const [isLoading, setIsLoading] = useState<boolean>(false);
-  const [availableIndices, setAvailableIndices] = useState<string[]>([]);
-  const [selectedIndices, setSelectedIndices] = useState<string[]>([]);
+  const [availableIndices, setAvailableIndices] = useState<RemoteIndex[]>([]);
+  const [selectedIndices, setSelectedIndices] = useState<RemoteIndex[]>([]);
   const [queryIndex, setQueryIndex] = useState<string>('');
 
   const handleConnectionCheck = (step: MigrationActions, data: MigrationState) => {
@@ -60,7 +65,7 @@ const MigrateExistingData = ({ currentStep, onTriggerStep, hideActions }: Migrat
 
       if (checkConnectionResult?.indices?.length) {
         setAvailableIndices(checkConnectionResult.indices);
-        setSelectedIndices(checkConnectionResult.indices);
+        setSelectedIndices(checkConnectionResult.indices.filter((i) => i.managed));
         setNextSteps(currentStep.next_steps.filter((next_step) => next_step === 'START_REMOTE_REINDEX_MIGRATION'));
       } else if (checkConnectionResult?.error) {
         setErrrorMessage(checkConnectionResult.error);
@@ -111,15 +116,15 @@ const MigrateExistingData = ({ currentStep, onTriggerStep, hideActions }: Migrat
     resetConnectionCheck();
   };
 
-  const handleSelectIndices = (indexToToggle: string) => {
+  const handleSelectIndices = (indexToToggle: RemoteIndex) => {
     if (selectedIndices.includes(indexToToggle)) {
       setSelectedIndices(selectedIndices.filter((index) => index !== indexToToggle));
     } else {
       setSelectedIndices([...selectedIndices, indexToToggle]);
     }
   };
 
-  const filteredIndices = queryIndex ? availableIndices.filter((index) => index.includes(queryIndex)) : availableIndices;
+  const filteredIndices = queryIndex ? availableIndices.filter((index) => index.name.includes(queryIndex)) : availableIndices;
   const filteredSelectedIndices = selectedIndices.filter((index) => filteredIndices.includes(index));
   const areAllIndicesSelected = filteredSelectedIndices.length === filteredIndices.length;
 
@@ -147,7 +152,7 @@ const MigrateExistingData = ({ currentStep, onTriggerStep, hideActions }: Migrat
                  name="hostname"
                  label="Hostname"
                  help="URI of the host to call the remote reindexing command against (http://example:9200)"
-                 placeholder="http://example:9200/"
+                 placeholder="http://example:9200"
                  type="text"
                  disabled={isLoading}
                  value={values.hostname}
@@ -226,10 +231,18 @@ const MigrateExistingData = ({ currentStep, onTriggerStep, hideActions }: Migrat
               <IndicesContainer>
                 {filteredIndices.map((index) => (
                   <Input type="checkbox"
-                         key={index}
-                         name={index}
-                         id={index}
-                         label={index}
+                         key={index.name}
+                         name={index.name}
+                         id={index.name}
+                         label={(
+                           <>
+                             <span>{index.name} </span>
+                             {!index.managed && (
+                               <Icon name="warning"
+                                     title="This is an index not managed by Graylog. If you import it, you will not be able to query it in Graylog." />
+                             )}
+                           </>
+                         )}
                          disabled={isLoading}
                          checked={filteredSelectedIndices.includes(index)}
                          onChange={() => handleSelectIndices(index)} />
@@ -246,7 +259,10 @@ const MigrateExistingData = ({ currentStep, onTriggerStep, hideActions }: Migrat
             <MigrationStepTriggerButtonToolbar hidden={hideActions}
                                                nextSteps={nextSteps || currentStep.next_steps}
                                                onTriggerStep={handleTriggerNextStep}
-                                               args={{ ...values, indices: filteredSelectedIndices } as RemoteReindexRequest} />
+                                               args={{
+                                                 ...values,
+                                                 indices: filteredSelectedIndices.map((i) => i.name),
+                                               } as RemoteReindexRequest} />
           )}
         </Form>
       )}