Skip to content

Commit

Permalink
Indicator for regular indices (#11313)
Browse files Browse the repository at this point in the history 
Adding a new field to the IndexSetConfig collection for being able to
explicitly indicate regular indices. Required by the Illuminate indices.

NOTE: for the sake of backward-compatibility the default value of the
new field is considered to be false. Should be removed after a migration
is provided.
  • Loading branch information
@devdanylo
devdanylo committed Sep 20, 2021
1 parent 647f8c2 commit aafae87
Show file tree
Hide file tree
Showing 13 changed files with 219 additions and 88 deletions.
79 changes: 23 additions & 56 deletions graylog2-server/src/main/java/org/graylog2/indexer/indexset/IndexSetConfig.java
View file
Expand Up @@ -18,6 +18,7 @@

import com.fasterxml.jackson.annotation.JsonAutoDetect;
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonIgnore;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.auto.value.AutoValue;
Expand Down Expand Up @@ -56,14 +57,6 @@ public abstract class IndexSetConfig implements Comparable<IndexSetConfig> {

private static final Duration DEFAULT_FIELD_TYPE_REFRESH_INTERVAL = Duration.standardSeconds(5L);

public static boolean isRegularIndex(@Nullable String templateType, boolean isWritable) {
// "isWritable == false" marks the restored-archive index set, which can also not be used as a default
return isWritable && (templateType == null || DEFAULT_INDEX_TEMPLATE_TYPE.equals(templateType));
}
public static boolean isRegularIndex(IndexSetConfig config) {
return isRegularIndex(config.indexTemplateType().orElse(null), config.isWritable());
}

@JsonProperty("id")
@Nullable
@Id
Expand All @@ -81,6 +74,13 @@ public static boolean isRegularIndex(IndexSetConfig config) {
@JsonProperty("writable")
public abstract boolean isWritable();

/**
* Indicates whether this index set is intended to
* store messages ingested by user, not by the system
*/
@JsonProperty("regular")
public abstract Optional<Boolean> isRegular();

@JsonProperty(FIELD_INDEX_PREFIX)
@NotBlank
@Pattern(regexp = INDEX_PREFIX_REGEX)
Expand Down Expand Up @@ -144,11 +144,19 @@ public static boolean isRegularIndex(IndexSetConfig config) {
@JsonProperty("field_type_refresh_interval")
public abstract Duration fieldTypeRefreshInterval();

@JsonIgnore
public boolean isRegularIndex() {
final String indexTemplate = indexTemplateType().orElse(null);
return isWritable() && (indexTemplate == null || DEFAULT_INDEX_TEMPLATE_TYPE.equals(indexTemplate) ||
isRegular().orElse(false));
}

@JsonCreator
public static IndexSetConfig create(@Id @ObjectId @JsonProperty("_id") @Nullable String id,
@JsonProperty("title") @NotBlank String title,
@JsonProperty("description") @Nullable String description,
@JsonProperty("writable") @Nullable Boolean isWritable,
@JsonProperty("regular") @Nullable Boolean isRegular,
@JsonProperty(FIELD_INDEX_PREFIX) @Pattern(regexp = INDEX_PREFIX_REGEX) String indexPrefix,
@JsonProperty("index_match_pattern") @Nullable String indexMatchPattern,
@JsonProperty("index_wildcard") @Nullable String indexWildcard,
Expand Down Expand Up @@ -179,6 +187,7 @@ public static IndexSetConfig create(@Id @ObjectId @JsonProperty("_id") @Nullable
.title(title)
.description(description)
.isWritable(writableValue)
.isRegular(isRegular)
.indexPrefix(indexPrefix)
.indexMatchPattern(indexMatchPattern)
.indexWildcard(indexWildcard)
Expand All @@ -198,58 +207,13 @@ public static IndexSetConfig create(@Id @ObjectId @JsonProperty("_id") @Nullable
.build();
}

public static IndexSetConfig create(String id,
String title,
String description,
boolean isWritable,
String indexPrefix,
int shards,
int replicas,
String rotationStrategyClass,
RotationStrategyConfig rotationStrategy,
String retentionStrategyClass,
RetentionStrategyConfig retentionStrategy,
ZonedDateTime creationDate,
String indexAnalyzer,
String indexTemplateName,
String indexTemplateType,
int indexOptimizationMaxNumSegments,
boolean indexOptimizationDisabled,
Duration fieldTypeRefreshInterval) {
return create(id, title, description, isWritable, indexPrefix, null, null, shards, replicas,
rotationStrategyClass, rotationStrategy, retentionStrategyClass, retentionStrategy, creationDate,
indexAnalyzer, indexTemplateName, indexTemplateType, indexOptimizationMaxNumSegments, indexOptimizationDisabled,
fieldTypeRefreshInterval);
}

public static IndexSetConfig create(String title,
String description,
boolean isWritable,
String indexPrefix,
int shards,
int replicas,
String rotationStrategyClass,
RotationStrategyConfig rotationStrategy,
String retentionStrategyClass,
RetentionStrategyConfig retentionStrategy,
ZonedDateTime creationDate,
String indexAnalyzer,
String indexTemplateName,
String indexTemplateType,
int indexOptimizationMaxNumSegments,
boolean indexOptimizationDisabled,
Duration fieldTypeRefreshInterval) {
return create(null, title, description, isWritable, indexPrefix, null, null, shards, replicas,
rotationStrategyClass, rotationStrategy, retentionStrategyClass, retentionStrategy, creationDate,
indexAnalyzer, indexTemplateName, indexTemplateType, indexOptimizationMaxNumSegments, indexOptimizationDisabled,
fieldTypeRefreshInterval);
}

// Compatibility creator after field type refresh interval has been introduced
public static IndexSetConfig create(String id,
String title,
String description,
boolean isWritable,
Boolean isRegular,
String indexPrefix,
int shards,
int replicas,
Expand All @@ -263,7 +227,7 @@ public static IndexSetConfig create(String id,
String indexTemplateType,
int indexOptimizationMaxNumSegments,
boolean indexOptimizationDisabled) {
return create(id, title, description, isWritable, indexPrefix, null, null, shards, replicas,
return create(id, title, description, isWritable, isRegular, indexPrefix, null, null, shards, replicas,
rotationStrategyClass, rotationStrategy, retentionStrategyClass, retentionStrategy, creationDate,
indexAnalyzer, indexTemplateName, indexTemplateType, indexOptimizationMaxNumSegments, indexOptimizationDisabled,
DEFAULT_FIELD_TYPE_REFRESH_INTERVAL);
Expand All @@ -273,6 +237,7 @@ public static IndexSetConfig create(String id,
public static IndexSetConfig create(String title,
String description,
boolean isWritable,
Boolean isRegular,
String indexPrefix,
int shards,
int replicas,
Expand All @@ -286,7 +251,7 @@ public static IndexSetConfig create(String title,
String indexTemplateType,
int indexOptimizationMaxNumSegments,
boolean indexOptimizationDisabled) {
return create(null, title, description, isWritable, indexPrefix, null, null, shards, replicas,
return create(null, title, description, isWritable, isRegular, indexPrefix, null, null, shards, replicas,
rotationStrategyClass, rotationStrategy, retentionStrategyClass, retentionStrategy, creationDate,
indexAnalyzer, indexTemplateName, indexTemplateType, indexOptimizationMaxNumSegments, indexOptimizationDisabled,
DEFAULT_FIELD_TYPE_REFRESH_INTERVAL);
Expand Down Expand Up @@ -320,6 +285,8 @@ public abstract static class Builder {

public abstract Builder isWritable(boolean isWritable);

public abstract Builder isRegular(@Nullable Boolean isRegular);

public abstract Builder indexPrefix(String indexPrefix);

public abstract Builder indexMatchPattern(String indexMatchPattern);
Expand Down
1 change: 1 addition & 0 deletions ...erver/src/main/java/org/graylog2/migrations/V20161116172100_DefaultIndexSetMigration.java
View file
Expand Up @@ -83,6 +83,7 @@ public void upgrade() {
final IndexSetConfig config = IndexSetConfig.builder()
.title("Default index set")
.description("The Graylog default index set")
.isRegular(true)
.indexPrefix(elasticsearchConfiguration.getIndexPrefix())
.shards(elasticsearchConfiguration.getShards())
.replicas(elasticsearchConfiguration.getReplicas())
Expand Down
1 change: 1 addition & 0 deletions ...ver/src/main/java/org/graylog2/migrations/V20190705071400_AddEventIndexSetsMigration.java
View file
Expand Up @@ -151,6 +151,7 @@ private IndexSet setupEventsIndexSet(String indexSetTitle, String indexSetDescri
.description(indexSetDescription)
.indexTemplateType(EVENT_TEMPLATE_TYPE)
.isWritable(true)
.isRegular(false)
.indexPrefix(indexPrefix)
.shards(elasticsearchConfiguration.getShards())
.replicas(elasticsearchConfiguration.getReplicas())
Expand Down
5 changes: 2 additions & 3 deletions graylog2-server/src/main/java/org/graylog2/rest/resources/streams/StreamResource.java
View file
Expand Up @@ -42,7 +42,6 @@
import org.graylog2.database.PaginatedList;
import org.graylog2.indexer.IndexSet;
import org.graylog2.indexer.IndexSetRegistry;
import org.graylog2.indexer.indexset.IndexSetConfig;
import org.graylog2.plugin.Message;
import org.graylog2.plugin.Tools;
import org.graylog2.plugin.alarms.AlertCondition;
Expand Down Expand Up @@ -171,7 +170,7 @@ public Response create(@ApiParam(name = "JSON body", required = true) final Crea
final IndexSet indexSet = stream.getIndexSet();
if (!indexSet.getConfig().isWritable()) {
throw new BadRequestException("Assigned index set must be writable!");
} else if (!IndexSetConfig.isRegularIndex(indexSet.getConfig())) {
} else if (!indexSet.getConfig().isRegularIndex()) {
throw new BadRequestException("Assigned index set is not usable");
}

Expand Down Expand Up @@ -329,7 +328,7 @@ public StreamResponse update(@ApiParam(name = "streamId", required = true)
throw new BadRequestException("Index set with ID <" + stream.getIndexSetId() + "> does not exist!");
} else if (!indexSet.get().getConfig().isWritable()) {
throw new BadRequestException("Assigned index set must be writable!");
} else if (!IndexSetConfig.isRegularIndex(indexSet.get().getConfig())) {
} else if (!indexSet.get().getConfig().isRegularIndex()) {
throw new BadRequestException("Assigned index set is not usable");
}

Expand Down
4 changes: 2 additions & 2 deletions ...g2-server/src/main/java/org/graylog2/rest/resources/system/indexer/IndexSetsResource.java
View file
Expand Up @@ -221,7 +221,7 @@ public IndexSetStats indexSetStatistics(@ApiParam(name = "id", required = true)
public IndexSetSummary save(@ApiParam(name = "Index set configuration", required = true)
@Valid @NotNull IndexSetSummary indexSet) {
try {
final IndexSetConfig indexSetConfig = indexSet.toIndexSetConfig();
final IndexSetConfig indexSetConfig = indexSet.toIndexSetConfig(true);

final Optional<IndexSetValidator.Violation> violation = indexSetValidator.validate(indexSetConfig);
if (violation.isPresent()) {
Expand Down Expand Up @@ -281,7 +281,7 @@ public IndexSetSummary setDefault(@ApiParam(name = "id", required = true)
final IndexSetConfig indexSet = indexSetService.get(id)
.orElseThrow(() -> new NotFoundException("Index set <" + id + "> does not exist"));

if (!IndexSetConfig.isRegularIndex(indexSet)) {
if (!indexSet.isRegularIndex()) {
throw new ClientErrorException("Index set not eligible as default", Response.Status.CONFLICT);
}

Expand Down
1 change: 1 addition & 0 deletions .../main/java/org/graylog2/rest/resources/system/indexer/requests/IndexSetUpdateRequest.java
View file
Expand Up @@ -122,6 +122,7 @@ public IndexSetConfig toIndexSetConfig(String id, IndexSetConfig oldConfig) {
.title(title())
.description(description())
.isWritable(isWritable())
.isRegular(oldConfig.isRegular().orElse(null))
.indexPrefix(oldConfig.indexPrefix())
.indexMatchPattern(oldConfig.indexMatchPattern())
.indexWildcard(oldConfig.indexWildcard())
Expand Down
7 changes: 5 additions & 2 deletions ...r/src/main/java/org/graylog2/rest/resources/system/indexer/responses/IndexSetSummary.java
View file
Expand Up @@ -114,6 +114,7 @@ public static IndexSetSummary create(@JsonProperty("id") @Nullable String id,
@JsonProperty("description") @Nullable String description,
@JsonProperty("default") boolean isDefault,
@JsonProperty("writable") boolean isWritable,
@JsonProperty("can_be_default") boolean canBeDefault,
@JsonProperty("index_prefix") @Pattern(regexp = IndexSetConfig.INDEX_PREFIX_REGEX) String indexPrefix,
@JsonProperty("shards") @Min(1) int shards,
@JsonProperty("replicas") @Min(0) int replicas,
Expand All @@ -127,7 +128,7 @@ public static IndexSetSummary create(@JsonProperty("id") @Nullable String id,
@JsonProperty("index_optimization_disabled") boolean indexOptimizationDisabled,
@JsonProperty("field_type_refresh_interval") Duration fieldTypeRefreshInterval,
@JsonProperty("index_template_type") @Nullable String templateType) {
return new AutoValue_IndexSetSummary(id, title, description, isDefault, IndexSetConfig.isRegularIndex(templateType, isWritable),
return new AutoValue_IndexSetSummary(id, title, description, isDefault, canBeDefault,
isWritable, indexPrefix, shards, replicas,
rotationStrategyClass, rotationStrategy, retentionStrategyClass, retentionStrategy, creationDate,
indexAnalyzer, indexOptimizationMaxNumSegments, indexOptimizationDisabled, fieldTypeRefreshInterval,
Expand All @@ -141,6 +142,7 @@ public static IndexSetSummary fromIndexSetConfig(IndexSetConfig indexSet, boolea
indexSet.description(),
isDefault,
indexSet.isWritable(),
indexSet.isRegularIndex(),
indexSet.indexPrefix(),
indexSet.shards(),
indexSet.replicas(),
Expand All @@ -157,12 +159,13 @@ public static IndexSetSummary fromIndexSetConfig(IndexSetConfig indexSet, boolea

}

public IndexSetConfig toIndexSetConfig() {
public IndexSetConfig toIndexSetConfig(boolean isRegular) {
final IndexSetConfig.Builder builder = IndexSetConfig.builder()
.id(id())
.title(title())
.description(description())
.isWritable(isWritable())
.isRegular(isRegular)
.indexPrefix(indexPrefix())
.shards(shards())
.replicas(replicas())
Expand Down
2 changes: 1 addition & 1 deletion graylog2-server/src/test/java/org/graylog2/indexer/MongoIndexSetTest.java
View file
Expand Up @@ -82,7 +82,7 @@ public class MongoIndexSetTest {
private final IndexSetConfig config = IndexSetConfig.create(
"Test",
"Test",
true,
true, true,
"graylog",
1,
0,
Expand Down

0 comments on commit aafae87

Please sign in to comment.