Skip to content

Commit

Permalink
Set and remove profile on index set level. Checks added and moved to …
Browse files Browse the repository at this point in the history 
…the service layer. (#17767)
  • Loading branch information
@luk-kaminski
luk-kaminski authored Dec 22, 2023
1 parent 1d15b4a commit 80a9e8e
Show file tree
Hide file tree
Showing 14 changed files with 439 additions and 33 deletions.
1 change: 1 addition & 0 deletions graylog2-server/src/main/java/org/graylog2/indexer/IndexMapping.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@ public abstract class IndexMapping implements IndexMappingTemplate {

@Override
public Template toTemplate(IndexSetConfig indexSetConfig, String indexPattern, Long order) {
//TODO: in next step, combine profile with customFieldMappings
return messageTemplate(indexPattern, indexSetConfig.indexAnalyzer(), order, indexSetConfig.customFieldMappings());
}

Expand Down
101 changes: 96 additions & 5 deletions ...erver/src/main/java/org/graylog2/indexer/fieldtypes/mapping/FieldTypeMappingsService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,22 +22,28 @@
import org.graylog2.indexer.indexset.IndexSetConfig;
import org.graylog2.indexer.indexset.IndexSetService;
import org.graylog2.indexer.indexset.MongoIndexSetService;
import org.graylog2.plugin.Message;
import org.graylog2.indexer.indexset.profile.IndexFieldTypeProfile;
import org.graylog2.indexer.indexset.profile.IndexFieldTypeProfileService;
import org.graylog2.rest.bulk.model.BulkOperationFailure;
import org.graylog2.rest.bulk.model.BulkOperationResponse;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import javax.inject.Inject;
import javax.ws.rs.BadRequestException;
import javax.ws.rs.NotFoundException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Optional;
import java.util.Set;
import java.util.stream.Collectors;

import static org.graylog2.plugin.Message.FIELDS_UNCHANGEABLE_BY_CUSTOM_MAPPINGS;

public class FieldTypeMappingsService {

private static final Logger LOG = LoggerFactory.getLogger(FieldTypeMappingsService.class);
Expand All @@ -46,21 +52,25 @@ public class FieldTypeMappingsService {
private final MongoIndexSet.Factory mongoIndexSetFactory;
private final MongoIndexSetService mongoIndexSetService;

private final IndexFieldTypeProfileService profileService;

@Inject
public FieldTypeMappingsService(final IndexSetService indexSetService,
final MongoIndexSet.Factory mongoIndexSetFactory,
final MongoIndexSetService mongoIndexSetService) {
final MongoIndexSetService mongoIndexSetService,
final IndexFieldTypeProfileService profileService) {
this.indexSetService = indexSetService;
this.mongoIndexSetFactory = mongoIndexSetFactory;
this.mongoIndexSetService = mongoIndexSetService;
this.profileService = profileService;
}

public void changeFieldType(final CustomFieldMapping customMapping,
final Set<String> indexSetsIds,
final boolean rotateImmediately) {
if (Message.FIELDS_UNCHANGEABLE_BY_CUSTOM_MAPPINGS.contains(customMapping.fieldName())) {
throw new IllegalArgumentException("Changing field type of " + customMapping.fieldName() + " is not allowed.");
}
checkFieldTypeCanBeChanged(customMapping.fieldName());
checkType(customMapping);

for (String indexSetId : indexSetsIds) {
try {
indexSetService.get(indexSetId).ifPresent(indexSetConfig -> {
Expand All @@ -76,6 +86,42 @@ public void changeFieldType(final CustomFieldMapping customMapping,
}
}

public void setProfile(final Set<String> indexSetsIds,
final String profileId,
final boolean rotateImmediately) {
checkProfile(profileId);
for (String indexSetId : indexSetsIds) {
try {
indexSetService.get(indexSetId).ifPresent(indexSetConfig -> {
var updatedIndexSetConfig = setProfileForIndexSet(profileId, indexSetConfig);
if (rotateImmediately) {
updatedIndexSetConfig.ifPresent(this::cycleIndexSet);
}
});
} catch (Exception ex) {
LOG.error("Failed to update field type in index set : " + indexSetId, ex);
throw ex;
}
}
}

public void removeProfileFromIndexSets(final Set<String> indexSetsIds,
final boolean rotateImmediately) {
for (String indexSetId : indexSetsIds) {
try {
indexSetService.get(indexSetId).ifPresent(indexSetConfig -> {
var updatedIndexSetConfig = removeProfileFromIndexSet(indexSetConfig);
if (rotateImmediately) {
updatedIndexSetConfig.ifPresent(this::cycleIndexSet);
}
});
} catch (Exception ex) {
LOG.error("Failed to update field type in index set : " + indexSetId, ex);
throw ex;
}
}
}

public Map<String, BulkOperationResponse> removeCustomMappingForFields(final List<String> fieldNames,
final Set<String> indexSetsIds,
final boolean rotateImmediately) {
Expand Down Expand Up @@ -142,8 +188,53 @@ private Optional<IndexSetConfig> storeMapping(final CustomFieldMapping customMap
));
}

private Optional<IndexSetConfig> setProfileForIndexSet(final String profileId,
final IndexSetConfig indexSetConfig) {
if (Objects.equals(indexSetConfig.fieldTypeProfile(), profileId)) {
return Optional.empty();
}
return Optional.of(mongoIndexSetService.save(
indexSetConfig.toBuilder()
.fieldTypeProfile(profileId)
.build()
));
}

private Optional<IndexSetConfig> removeProfileFromIndexSet(final IndexSetConfig indexSetConfig) {
if (indexSetConfig.fieldTypeProfile() == null) {
return Optional.empty();
}
return Optional.of(mongoIndexSetService.save(
indexSetConfig.toBuilder()
.fieldTypeProfile(null)
.build()
));
}

private void cycleIndexSet(final IndexSetConfig indexSetConfig) {
final MongoIndexSet mongoIndexSet = mongoIndexSetFactory.create(indexSetConfig);
mongoIndexSet.cycle();
}

private void checkType(final CustomFieldMapping customMapping) {
var type = CustomFieldMappings.AVAILABLE_TYPES.get(customMapping.type());
if (type == null) {
throw new BadRequestException("Invalid type provided: " + customMapping.type() + " - available types: " + CustomFieldMappings.AVAILABLE_TYPES.keySet());
}
}

private void checkProfile(final String profileId) {
final Optional<IndexFieldTypeProfile> fieldTypeProfile = profileService.get(profileId);
if (fieldTypeProfile.isPresent()) {
fieldTypeProfile.get().customFieldMappings().forEach(mapping -> checkFieldTypeCanBeChanged(mapping.fieldName()));
} else {
throw new NotFoundException("No profile with id : " + profileId);
}
}

private void checkFieldTypeCanBeChanged(final String fieldName) {
if (FIELDS_UNCHANGEABLE_BY_CUSTOM_MAPPINGS.contains(fieldName)) {
throw new BadRequestException("Unable to change field type of " + fieldName + ", not allowed to change type of these fields: " + FIELDS_UNCHANGEABLE_BY_CUSTOM_MAPPINGS);
}
}
}
12 changes: 12 additions & 0 deletions graylog2-server/src/main/java/org/graylog2/indexer/indexset/CustomFieldMappings.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,18 @@ public CustomFieldMappings mergeWith(final CustomFieldMapping changedMapping) {
return new CustomFieldMappings(modifiedMappings);
}

public CustomFieldMappings mergeWith(final CustomFieldMappings changedMappings) {
if (changedMappings == null || changedMappings.isEmpty()) {
return this;
}
final Set<CustomFieldMapping> modifiedMappings = new HashSet<>(this);
for (CustomFieldMapping changedMapping : changedMappings) {
modifiedMappings.removeIf(m -> changedMapping.fieldName().equals(m.fieldName()));
modifiedMappings.add(changedMapping);
}
return new CustomFieldMappings(modifiedMappings);
}

public boolean containsCustomMappingForField(final String fieldName) {
return stream().anyMatch(m -> m.fieldName().equals(fieldName));
}
Expand Down
15 changes: 12 additions & 3 deletions graylog2-server/src/main/java/org/graylog2/indexer/indexset/IndexSetConfig.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -150,6 +150,10 @@ public abstract class IndexSetConfig implements Comparable<IndexSetConfig> {
@JsonProperty("custom_field_mappings")
public abstract CustomFieldMappings customFieldMappings();

@JsonProperty("field_type_profile")
@Nullable
public abstract String fieldTypeProfile();

@JsonIgnore
public boolean isRegularIndex() {
final String indexTemplate = indexTemplateType().orElse(null);
Expand Down Expand Up @@ -179,7 +183,8 @@ public static IndexSetConfig create(@Id @ObjectId @JsonProperty("_id") @Nullable
@JsonProperty("index_optimization_max_num_segments") @Nullable Integer maxNumSegments,
@JsonProperty("index_optimization_disabled") @Nullable Boolean indexOptimizationDisabled,
@JsonProperty("field_type_refresh_interval") @Nullable Duration fieldTypeRefreshInterval,
@JsonProperty("custom_field_mappings") @Nullable CustomFieldMappings customFieldMappings
@JsonProperty("custom_field_mappings") @Nullable CustomFieldMappings customFieldMappings,
@JsonProperty("field_type_profile") @Nullable String fieldTypeProfile
) {

final boolean writableValue = isWritable == null || isWritable;
Expand Down Expand Up @@ -213,6 +218,7 @@ public static IndexSetConfig create(@Id @ObjectId @JsonProperty("_id") @Nullable
.indexOptimizationDisabled(indexOptimizationDisabled != null && indexOptimizationDisabled)
.fieldTypeRefreshInterval(fieldTypeRefreshIntervalValue)
.customFieldMappings(customFieldMappings == null ? new CustomFieldMappings() : customFieldMappings)
.fieldTypeProfile(fieldTypeProfile)
.build();
}

Expand All @@ -239,7 +245,7 @@ public static IndexSetConfig create(String id,
return create(id, title, description, isWritable, isRegular, indexPrefix, null, null, shards, replicas,
rotationStrategyClass, rotationStrategy, retentionStrategyClass, retentionStrategy, creationDate,
indexAnalyzer, indexTemplateName, indexTemplateType, indexOptimizationMaxNumSegments, indexOptimizationDisabled,
DEFAULT_FIELD_TYPE_REFRESH_INTERVAL, new CustomFieldMappings());
DEFAULT_FIELD_TYPE_REFRESH_INTERVAL, new CustomFieldMappings(), null);
}

// Compatibility creator after field type refresh interval has been introduced
Expand All @@ -263,7 +269,7 @@ public static IndexSetConfig create(String title,
return create(null, title, description, isWritable, isRegular, indexPrefix, null, null, shards, replicas,
rotationStrategyClass, rotationStrategy, retentionStrategyClass, retentionStrategy, creationDate,
indexAnalyzer, indexTemplateName, indexTemplateType, indexOptimizationMaxNumSegments, indexOptimizationDisabled,
DEFAULT_FIELD_TYPE_REFRESH_INTERVAL, new CustomFieldMappings());
DEFAULT_FIELD_TYPE_REFRESH_INTERVAL, new CustomFieldMappings(), null);
}

@Override
Expand Down Expand Up @@ -335,6 +341,9 @@ public abstract static class Builder {

public abstract Builder customFieldMappings(CustomFieldMappings customFieldMappings);

public abstract Builder fieldTypeProfile(String fieldTypeProfile);


public abstract IndexSetConfig build();
}
}
3 changes: 2 additions & 1 deletion ...og2-server/src/main/java/org/graylog2/indexer/indexset/profile/IndexFieldTypeProfile.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,12 +21,13 @@
import org.mongojack.Id;
import org.mongojack.ObjectId;

import javax.annotation.Nonnull;
import javax.annotation.Nullable;

public record IndexFieldTypeProfile(@JsonProperty(ID_FIELD_NAME) @Nullable @Id @ObjectId String id,
@JsonProperty(NAME_FIELD_NAME) String name,
@JsonProperty(DESCRIPTION_FIELD_NAME) String description,
@JsonProperty(CUSTOM_MAPPINGS_FIELD_NAME) CustomFieldMappings customFieldMappings) {
@JsonProperty(CUSTOM_MAPPINGS_FIELD_NAME) @Nonnull CustomFieldMappings customFieldMappings) {

public static final String ID_FIELD_NAME = "id";
public static final String NAME_FIELD_NAME = "name";
Expand Down
4 changes: 3 additions & 1 deletion ...server/src/main/java/org/graylog2/indexer/indexset/profile/IndexFieldTypeProfileData.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,11 +19,13 @@
import com.fasterxml.jackson.annotation.JsonProperty;
import org.graylog2.indexer.indexset.CustomFieldMappings;

import javax.annotation.Nonnull;

import static org.graylog2.indexer.indexset.profile.IndexFieldTypeProfile.CUSTOM_MAPPINGS_FIELD_NAME;
import static org.graylog2.indexer.indexset.profile.IndexFieldTypeProfile.DESCRIPTION_FIELD_NAME;
import static org.graylog2.indexer.indexset.profile.IndexFieldTypeProfile.NAME_FIELD_NAME;

public record IndexFieldTypeProfileData(@JsonProperty(NAME_FIELD_NAME) String name,
@JsonProperty(DESCRIPTION_FIELD_NAME) String description,
@JsonProperty(CUSTOM_MAPPINGS_FIELD_NAME) CustomFieldMappings customFieldMappings) {
@JsonProperty(CUSTOM_MAPPINGS_FIELD_NAME) @Nonnull CustomFieldMappings customFieldMappings) {
}
16 changes: 15 additions & 1 deletion ...ver/src/main/java/org/graylog2/indexer/indexset/profile/IndexFieldTypeProfileService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,13 +30,15 @@
import org.mongojack.WriteResult;

import javax.inject.Inject;
import javax.ws.rs.BadRequestException;
import java.util.List;
import java.util.Locale;

import static org.graylog2.indexer.indexset.profile.IndexFieldTypeProfile.CUSTOM_MAPPINGS_FIELD_NAME;
import static org.graylog2.indexer.indexset.profile.IndexFieldTypeProfile.DESCRIPTION_FIELD_NAME;
import static org.graylog2.indexer.indexset.profile.IndexFieldTypeProfile.ID_FIELD_NAME;
import static org.graylog2.indexer.indexset.profile.IndexFieldTypeProfile.NAME_FIELD_NAME;
import static org.graylog2.plugin.Message.FIELDS_UNCHANGEABLE_BY_CUSTOM_MAPPINGS;


public class IndexFieldTypeProfileService extends PaginatedDbService<IndexFieldTypeProfile> {
Expand All @@ -61,7 +63,14 @@ public IndexFieldTypeProfileService(final MongoConnection mongoConnection,
this.db.createIndex(new BasicDBObject(IndexFieldTypeProfile.NAME_FIELD_NAME, 1), new BasicDBObject("unique", false));
}

@Override
public IndexFieldTypeProfile save(final IndexFieldTypeProfile indexFieldTypeProfile) {
indexFieldTypeProfile.customFieldMappings().forEach(mapping -> checkFieldTypeCanBeChanged(mapping.fieldName()));
return super.save(indexFieldTypeProfile);
}

public boolean update(final String profileId, final IndexFieldTypeProfile updatedProfile) {
updatedProfile.customFieldMappings().forEach(mapping -> checkFieldTypeCanBeChanged(mapping.fieldName()));
final WriteResult<IndexFieldTypeProfile, ObjectId> writeResult = db.updateById(new ObjectId(profileId), updatedProfile);
return writeResult.getN() > 0;
}
Expand All @@ -76,7 +85,6 @@ public PageListResponse<IndexFieldTypeProfile> getPaginated(final int page,
getSortBuilder(order, sortField),
page,
perPage);
final int total = paginated.grandTotal().orElse(0L).intValue();
return PageListResponse.create("",
paginated,
sortField,
Expand All @@ -85,4 +93,10 @@ public PageListResponse<IndexFieldTypeProfile> getPaginated(final int page,
DEFAULTS);
}

private void checkFieldTypeCanBeChanged(final String fieldName) {
if (FIELDS_UNCHANGEABLE_BY_CUSTOM_MAPPINGS.contains(fieldName)) {
throw new BadRequestException("Unable to change field type of " + fieldName + ", not allowed to change type of these fields: " + FIELDS_UNCHANGEABLE_BY_CUSTOM_MAPPINGS);
}
}

}
Loading

0 comments on commit 80a9e8e

Please sign in to comment.