Skip to content

Commit

Permalink
Return only summary of existing searches/dashboards (#10280)
Browse files Browse the repository at this point in the history
* simple first version with a stripped down interface for the DTO and lax requirements on the web side for discussion

* Summaries for Dashboards and Searches (ViewDTO)

* Linked static strings back to the original ViewDTO to make changes in the future less error prone.

* Included a delegation from ViewService to ViewSummaryService so that the Service used everywhere is still ViewService

* Added .isRequired back to views in FE
Added properties to ViewSummaryDTO that are needed on the FE side and unproblematic on the Backend side

* replaced ViewDTO with ViewSummaryDTo to prevent deserialization errors

* fixed tests to use ViewSummaryService

* added SearchSummary class

* removed requirements (unnecessary for the given search)

* added/fixed tests to work with new Summary-Classes

* removed unnecessary class

* cleared up naming, invert filter criteria for better performance
  • Loading branch information
janheise committed Apr 12, 2021
1 parent 72eab6b commit c567bff
Show file tree
Hide file tree
Showing 17 changed files with 412 additions and 58 deletions.
Expand Up @@ -56,7 +56,7 @@
@JsonDeserialize(builder = Search.Builder.class)
public abstract class Search implements ContentPackable<SearchEntity> {
public static final String FIELD_REQUIRES = "requires";
private static final String FIELD_CREATED_AT = "created_at";
static final String FIELD_CREATED_AT = "created_at";
public static final String FIELD_OWNER = "owner";

// generated during build to help quickly find a query by id.
Expand Down
@@ -0,0 +1,85 @@
/*
* Copyright (C) 2020 Graylog, Inc.
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the Server Side Public License, version 1,
* as published by MongoDB, Inc.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* Server Side Public License for more details.
*
* You should have received a copy of the Server Side Public License
* along with this program. If not, see
* <http://www.mongodb.com/licensing/server-side-public-license>.
*/
package org.graylog.plugins.views.search;

import com.fasterxml.jackson.annotation.JsonAutoDetect;
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
import com.fasterxml.jackson.databind.annotation.JsonPOJOBuilder;
import com.google.auto.value.AutoValue;
import org.joda.time.DateTime;
import org.joda.time.DateTimeZone;
import org.mongojack.Id;
import org.mongojack.ObjectId;

import javax.annotation.Nullable;
import java.util.Optional;

import static org.graylog.plugins.views.search.Search.FIELD_CREATED_AT;
import static org.graylog.plugins.views.search.Search.FIELD_OWNER;

@AutoValue
@JsonAutoDetect
@JsonDeserialize(builder = SearchSummary.Builder.class)
@JsonIgnoreProperties(ignoreUnknown = true)
public abstract class SearchSummary {
@Id
@ObjectId
@Nullable
@JsonProperty
public abstract String id();

@JsonProperty(FIELD_OWNER)
public abstract Optional<String> owner();

@JsonProperty(FIELD_CREATED_AT)
public abstract DateTime createdAt();

public abstract Builder toBuilder();

public static Builder builder() {
return Builder.create();
}

@AutoValue.Builder
@JsonPOJOBuilder(withPrefix = "")
@JsonIgnoreProperties(ignoreUnknown = true)
public abstract static class Builder {
@Id
@JsonProperty
public abstract Builder id(String id);

@JsonProperty(FIELD_OWNER)
public abstract Builder owner(@Nullable String owner);

@JsonProperty(FIELD_CREATED_AT)
public abstract Builder createdAt(DateTime createdAt);

abstract SearchSummary autoBuild();

@JsonCreator
public static Builder create() {
return new AutoValue_SearchSummary.Builder().createdAt(DateTime.now(DateTimeZone.UTC));
}

public SearchSummary build() {
return autoBuild();
}
}
}
Expand Up @@ -21,9 +21,11 @@
import org.bson.types.ObjectId;
import org.graylog.plugins.views.search.Search;
import org.graylog.plugins.views.search.SearchRequirements;
import org.graylog.plugins.views.search.SearchSummary;
import org.graylog2.bindings.providers.MongoJackObjectMapperProvider;
import org.graylog2.database.MongoConnection;
import org.graylog2.database.PaginatedList;
import org.joda.time.Instant;
import org.mongojack.DBCursor;
import org.mongojack.DBQuery;
import org.mongojack.DBSort;
Expand All @@ -47,6 +49,7 @@
*/
public class SearchDbService {
protected final JacksonDBCollection<Search, ObjectId> db;
protected final JacksonDBCollection<SearchSummary, ObjectId> summarydb;
private final SearchRequirements.Factory searchRequirementsFactory;

@Inject
Expand All @@ -59,6 +62,10 @@ protected SearchDbService(MongoConnection mongoConnection,
ObjectId.class,
mapper.get());
db.createIndex(new BasicDBObject("created_at", 1), new BasicDBObject("unique", false));
summarydb = JacksonDBCollection.wrap(mongoConnection.getDatabase().getCollection("searches"),
SearchSummary.class,
ObjectId.class,
mapper.get());
}

public Optional<Search> get(String id) {
Expand Down Expand Up @@ -117,4 +124,15 @@ private Search requirementsForSearch(Search search) {
return searchRequirementsFactory.create(search)
.rebuildRequirements(Search::requires, (s, newRequirements) -> s.toBuilder().requires(newRequirements).build());
}

Stream<SearchSummary> findSummaries() {
return Streams.stream((Iterable<SearchSummary>) summarydb.find());
}

public Set<String> getExpiredSearches(final Set<String> neverDeleteIds, final Instant mustNotBeOlderThan) {
return this.findSummaries()
.filter(search -> !neverDeleteIds.contains(search.id()) && search.createdAt().isBefore(mustNotBeOlderThan))
.map(search -> search.id())
.collect(Collectors.toSet());
}
}
Expand Up @@ -16,8 +16,8 @@
*/
package org.graylog.plugins.views.search.db;

import org.graylog.plugins.views.search.views.ViewDTO;
import org.graylog.plugins.views.search.views.ViewService;
import org.graylog.plugins.views.search.views.ViewSummaryDTO;
import org.graylog.plugins.views.search.views.ViewSummaryService;
import org.graylog2.plugin.periodical.Periodical;
import org.joda.time.Duration;
import org.joda.time.Instant;
Expand All @@ -32,15 +32,15 @@
public class SearchesCleanUpJob extends Periodical {
private static final Logger LOG = LoggerFactory.getLogger(SearchesCleanUpJob.class);

private final ViewService viewService;
private final ViewSummaryService viewSummaryService;
private final SearchDbService searchDbService;
private final Instant mustNotBeOlderThan;

@Inject
public SearchesCleanUpJob(ViewService viewService,
public SearchesCleanUpJob(ViewSummaryService viewSummaryService,
SearchDbService searchDbService,
@Named("views_maximum_search_age") Duration maximumSearchAge) {
this.viewService = viewService;
this.viewSummaryService = viewSummaryService;
this.searchDbService = searchDbService;
this.mustNotBeOlderThan = Instant.now().minus(maximumSearchAge);
}
Expand Down Expand Up @@ -87,9 +87,7 @@ protected Logger getLogger() {

@Override
public void doRun() {
final Set<String> requiredIds = viewService.streamAll().map(ViewDTO::searchId).collect(Collectors.toSet());
searchDbService.streamAll()
.filter(search -> search.createdAt().isBefore(mustNotBeOlderThan) && !requiredIds.contains(search.id()))
.forEach(search -> searchDbService.delete(search.id()));
final Set<String> requiredIds = viewSummaryService.streamAll().map(ViewSummaryDTO::searchId).collect(Collectors.toSet());
searchDbService.getExpiredSearches(requiredIds, mustNotBeOlderThan).forEach(id -> searchDbService.delete(id));
}
}
Expand Up @@ -24,6 +24,7 @@
import org.apache.shiro.authz.annotation.RequiresAuthentication;
import org.graylog.plugins.views.search.views.ViewDTO;
import org.graylog.plugins.views.search.views.ViewService;
import org.graylog.plugins.views.search.views.ViewSummaryDTO;
import org.graylog2.database.PaginatedList;
import org.graylog2.rest.models.PaginatedResponse;
import org.graylog2.search.SearchQuery;
Expand Down Expand Up @@ -65,22 +66,22 @@ public DashboardsResource(ViewService dbService) {
@GET
@ApiOperation("Get a list of all dashboards")
@Timed
public PaginatedResponse<ViewDTO> views(@ApiParam(name = "page") @QueryParam("page") @DefaultValue("1") int page,
@ApiParam(name = "per_page") @QueryParam("per_page") @DefaultValue("50") int perPage,
@ApiParam(name = "sort",
public PaginatedResponse<ViewSummaryDTO> views(@ApiParam(name = "page") @QueryParam("page") @DefaultValue("1") int page,
@ApiParam(name = "per_page") @QueryParam("per_page") @DefaultValue("50") int perPage,
@ApiParam(name = "sort",
value = "The field to sort the result on",
required = true,
allowableValues = "id,title,created_at") @DefaultValue(ViewDTO.FIELD_TITLE) @QueryParam("sort") String sortField,
@ApiParam(name = "order", value = "The sort direction", allowableValues = "asc, desc") @DefaultValue("asc") @QueryParam("order") String order,
@ApiParam(name = "query") @QueryParam("query") String query) {
@ApiParam(name = "order", value = "The sort direction", allowableValues = "asc, desc") @DefaultValue("asc") @QueryParam("order") String order,
@ApiParam(name = "query") @QueryParam("query") String query) {

if (!ViewDTO.SORT_FIELDS.contains(sortField.toLowerCase(ENGLISH))) {
sortField = ViewDTO.FIELD_TITLE;
}

try {
final SearchQuery searchQuery = searchQueryParser.parse(query);
final PaginatedList<ViewDTO> result = dbService.searchPaginatedByType(
final PaginatedList<ViewSummaryDTO> result = dbService.searchSummariesPaginatedByType(
ViewDTO.Type.DASHBOARD,
searchQuery,
view -> isPermitted(ViewsRestPermissions.VIEW_READ, view.id())
Expand Down
Expand Up @@ -23,6 +23,7 @@
import org.apache.shiro.authz.annotation.RequiresAuthentication;
import org.graylog.plugins.views.search.views.ViewDTO;
import org.graylog.plugins.views.search.views.ViewService;
import org.graylog.plugins.views.search.views.ViewSummaryDTO;
import org.graylog2.database.PaginatedList;
import org.graylog2.rest.models.PaginatedResponse;
import org.graylog2.search.SearchQuery;
Expand Down Expand Up @@ -62,22 +63,22 @@ public SavedSearchesResource(ViewService dbService) {

@GET
@ApiOperation("Get a list of all searches")
public PaginatedResponse<ViewDTO> views(@ApiParam(name = "page") @QueryParam("page") @DefaultValue("1") int page,
@ApiParam(name = "per_page") @QueryParam("per_page") @DefaultValue("50") int perPage,
@ApiParam(name = "sort",
public PaginatedResponse<ViewSummaryDTO> views(@ApiParam(name = "page") @QueryParam("page") @DefaultValue("1") int page,
@ApiParam(name = "per_page") @QueryParam("per_page") @DefaultValue("50") int perPage,
@ApiParam(name = "sort",
value = "The field to sort the result on",
required = true,
allowableValues = "id,title,created_at") @DefaultValue(ViewDTO.FIELD_TITLE) @QueryParam("sort") String sortField,
@ApiParam(name = "order", value = "The sort direction", allowableValues = "asc, desc") @DefaultValue("asc") @QueryParam("order") String order,
@ApiParam(name = "query") @QueryParam("query") String query) {
@ApiParam(name = "order", value = "The sort direction", allowableValues = "asc, desc") @DefaultValue("asc") @QueryParam("order") String order,
@ApiParam(name = "query") @QueryParam("query") String query) {

if (!ViewDTO.SORT_FIELDS.contains(sortField.toLowerCase(ENGLISH))) {
sortField = ViewDTO.FIELD_TITLE;
}

try {
final SearchQuery searchQuery = searchQueryParser.parse(query);
final PaginatedList<ViewDTO> result = dbService.searchPaginatedByType(
final PaginatedList<ViewSummaryDTO> result = dbService.searchSummariesPaginatedByType(
ViewDTO.Type.SEARCH,
searchQuery,
view -> isPermitted(ViewsRestPermissions.VIEW_READ, view.id()),
Expand Down
Expand Up @@ -48,17 +48,20 @@ public class ViewService extends PaginatedDbService<ViewDTO> {
private final ClusterConfigService clusterConfigService;
private final ViewRequirements.Factory viewRequirementsFactory;
private final EntityOwnershipService entityOwnerShipService;
private final ViewSummaryService viewSummaryService;

@Inject
protected ViewService(MongoConnection mongoConnection,
MongoJackObjectMapperProvider mapper,
ClusterConfigService clusterConfigService,
ViewRequirements.Factory viewRequirementsFactory,
EntityOwnershipService entityOwnerShipService) {
EntityOwnershipService entityOwnerShipService,
ViewSummaryService viewSummaryService) {
super(mongoConnection, mapper, ViewDTO.class, COLLECTION_NAME);
this.clusterConfigService = clusterConfigService;
this.viewRequirementsFactory = viewRequirementsFactory;
this.entityOwnerShipService = entityOwnerShipService;
this.viewSummaryService = viewSummaryService;
}

private PaginatedList<ViewDTO> searchPaginated(DBQuery.Query query,
Expand Down Expand Up @@ -105,6 +108,16 @@ public PaginatedList<ViewDTO> searchPaginatedByType(ViewDTO.Type type,
);
}

public PaginatedList<ViewSummaryDTO> searchSummariesPaginatedByType(final ViewDTO.Type type,
final SearchQuery query,
final Predicate<ViewSummaryDTO> filter,
final String order,
final String sortField,
final int page,
final int perPage) {
return viewSummaryService.searchPaginatedByType(type, query, filter, order, sortField, page, perPage);
}

public void saveDefault(ViewDTO dto) {
if (isNullOrEmpty(dto.id())) {
throw new IllegalArgumentException("ViewDTO needs an ID to be configured as default view");
Expand Down

0 comments on commit c567bff

Please sign in to comment.