Skip to content

Commit

Permalink
GH-2121 - Optimize cyclic schema queries.
Browse files Browse the repository at this point in the history 
Those queries will now return nodes and relationships instead of plain path(s).
This format is now the only one supported by SDN for generated queries.

Intentionally closes #2121
As a side-effect this commit also
closes #2107
closes #2109
closes #2119

Polishing after rebase.
  • Loading branch information
@meistermeier
meistermeier committed Jan 25, 2021
1 parent b5cb295 commit cc73484
Show file tree
Hide file tree
Showing 14 changed files with 675 additions and 141 deletions.
13 changes: 11 additions & 2 deletions src/main/java/org/springframework/data/neo4j/core/Neo4jTemplate.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
import java.util.Optional;
Expand Down Expand Up @@ -614,7 +615,11 @@ public List<T> getResults() {

public Optional<T> getSingleResult() {
try {
return fetchSpec.one();
Optional<T> one = fetchSpec.one();
if (preparedQuery.resultsHaveBeenAggregated()) {
return one.map(aggregatedResults -> (T) ((LinkedHashSet<?>) aggregatedResults).iterator().next());
}
return one;
} catch (NoSuchRecordException e) {
// This exception is thrown by the driver in both cases when there are 0 or 1+n records
// So there has been an incorrect result size, but not to few results but to many.
Expand All @@ -623,7 +628,11 @@ public Optional<T> getSingleResult() {
}

public T getRequiredSingleResult() {
return fetchSpec.one().orElseThrow(() -> new NoResultException(1, preparedQuery.getCypherQuery()));
Optional<T> one = fetchSpec.one();
if (preparedQuery.resultsHaveBeenAggregated()) {
one = one.map(aggregatedResults -> (T) ((LinkedHashSet<?>) aggregatedResults).iterator().next());
}
return one.orElseThrow(() -> new NoResultException(1, preparedQuery.getCypherQuery()));
}
}
}
31 changes: 19 additions & 12 deletions src/main/java/org/springframework/data/neo4j/core/PreparedQuery.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -164,30 +164,37 @@ private static class AggregatingMappingFunction implements BiFunction<TypeSystem
private Collection<?> aggregateList(TypeSystem t, Value value) {

if (MappingSupport.isListContainingOnly(t.LIST(), t.PATH()).test(value)) {
Set<Object> result = new LinkedHashSet<>();
for (Value path : value.values()) {
result.addAll(aggregatePath(t, path, Collections.emptyList()));
}
return result;
return new LinkedHashSet<Object>(aggregatePath(t, value, Collections.emptyList()));
}
return value.asList(v -> target.apply(t, v));
}

private Collection<?> aggregatePath(TypeSystem t, Value value,
List<Map.Entry<String, Value>> additionalValues) {
Path path = value.asPath();

// We are using linked hash sets here so that the order of nodes will be stable and match that of the path.
Set<Object> result = new LinkedHashSet<>();
Set<Value> nodes = new LinkedHashSet<>();
Set<Value> relationships = new LinkedHashSet<>();
Node lastNode = null;
for (Path.Segment segment : path) {
nodes.add(Values.value(segment.start()));
lastNode = segment.end();
relationships.add(Values.value(segment.relationship()));

List<Path> paths = value.hasType(t.PATH())
? Collections.singletonList(value.asPath())
: value.asList(Value::asPath);

for (Path path : paths) {
Node lastNode = null;
for (Path.Segment segment : path) {
Node start = segment.start();
if (start != null) {
nodes.add(Values.value(start));
}
lastNode = segment.end();
relationships.add(Values.value(segment.relationship()));
}
if (lastNode != null) {
nodes.add(Values.value(lastNode));
}
}
nodes.add(Values.value(lastNode));

// This loop synthesizes a node, it's relationship and all related nodes for all nodes in a path.
// All other nodes must be assumed to somehow related
Expand Down
12 changes: 9 additions & 3 deletions src/main/java/org/springframework/data/neo4j/core/ReactiveNeo4jTemplate.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
import java.util.function.Function;
Expand Down Expand Up @@ -632,8 +633,8 @@ final class DefaultReactiveExecutableQuery<T> implements ExecutableQuery<T> {
public Flux<T> getResults() {

return fetchSpec.all().switchOnFirst((signal, f) -> {
if (preparedQuery.resultsHaveBeenAggregated()) {
return f.flatMap(nested -> Flux.fromIterable((Collection<T>) nested).distinct());
if (signal.hasValue() && preparedQuery.resultsHaveBeenAggregated()) {
return f.flatMap(nested -> Flux.fromIterable((Collection<T>) nested).distinct()).distinct();
}
return f;
});
Expand All @@ -645,7 +646,12 @@ public Flux<T> getResults() {
*/
public Mono<T> getSingleResult() {
try {
return fetchSpec.one();
return fetchSpec.one().map(t -> {
if (t instanceof LinkedHashSet) {
return (T) ((LinkedHashSet<?>) t).iterator().next();
}
return t;
});
} catch (NoSuchRecordException e) {
// This exception is thrown by the driver in both cases when there are 0 or 1+n records
// So there has been an incorrect result size, but not to few results but to many.
Expand Down
170 changes: 138 additions & 32 deletions src/main/java/org/springframework/data/neo4j/core/mapping/CypherGenerator.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,18 +25,22 @@
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.function.Predicate;
import java.util.function.UnaryOperator;
import java.util.stream.Collectors;

import org.apiguardian.api.API;
import org.neo4j.cypherdsl.core.Condition;
import org.neo4j.cypherdsl.core.Conditions;
import org.neo4j.cypherdsl.core.Cypher;
import org.neo4j.cypherdsl.core.Expression;
import org.neo4j.cypherdsl.core.FunctionInvocation;
import org.neo4j.cypherdsl.core.Functions;
import org.neo4j.cypherdsl.core.ListComprehension;
import org.neo4j.cypherdsl.core.MapProjection;
import org.neo4j.cypherdsl.core.NamedPath;
import org.neo4j.cypherdsl.core.Node;
Expand Down Expand Up @@ -100,7 +104,13 @@ public StatementBuilder.OrderableOngoingReadingAndWith prepareMatchOf(NodeDescri
* @return An ongoing match
*/
public StatementBuilder.OrderableOngoingReadingAndWith prepareMatchOf(NodeDescription<?> nodeDescription,
@Nullable Condition condition) {
@Nullable Condition condition) {

return prepareMatchOf(nodeDescription, condition, Collections.emptyList());
}

public StatementBuilder.OrderableOngoingReadingAndWith prepareMatchOf(NodeDescription<?> nodeDescription,
@Nullable Condition condition, List<String> includedProperties) {

String primaryLabel = nodeDescription.getPrimaryLabel();
List<String> additionalLabels = nodeDescription.getAdditionalLabels();
Expand All @@ -111,7 +121,99 @@ public StatementBuilder.OrderableOngoingReadingAndWith prepareMatchOf(NodeDescri
expressions.add(Constants.NAME_OF_ROOT_NODE);
expressions.add(Functions.id(rootNode).as(Constants.NAME_OF_INTERNAL_ID));

return match(rootNode).where(conditionOrNoCondition(condition)).with(expressions.toArray(new Expression[] {}));
if (nodeDescription.containsPossibleCircles(includedProperties)) {
return createPathMatchWithCondition(nodeDescription, includedProperties, condition, rootNode);
} else {
return match(rootNode).where(conditionOrNoCondition(condition)).with(expressions.toArray(new Expression[] {}));
}
}

private StatementBuilder.OrderableOngoingReadingAndWithWithoutWhere createPathMatchWithCondition(
NodeDescription<?> nodeDescription, List<String> includedProperties, @Nullable Condition condition, Node rootNode) {

return createPathMatchWithCondition(null, nodeDescription, includedProperties, condition, rootNode);
}

public StatementBuilder.OrderableOngoingReadingAndWithWithoutWhere createPathMatchWithCondition(
@Nullable StatementBuilder.OngoingReadingWithoutWhere previousMatches,
NodeDescription<?> nodeDescription, List<String> includedProperties, @Nullable Condition condition, Node rootNode) {

List<Expression> expressions1 = new ArrayList<>();
List<Expression> expressions2 = new ArrayList<>();

String aliasedPathName = "pathPattern";
Predicate<String> includeField = s -> includedProperties.isEmpty() || includedProperties.contains(s);
Collection<RelationshipDescription> relationships = getRelationshipDescriptionsUpAndDown(nodeDescription, includeField);
RelationshipPattern patternPath = createRelationships(rootNode, relationships);
NamedPath path = Cypher.path("p").definedBy(patternPath);

// nested nodes flatMap: reduce(...reduce(...))
SymbolicName outerNodesAccumulator = Cypher.name("a");
SymbolicName outerNodesVariable = Cypher.name("b");
SymbolicName innerNodesAccumulator = Cypher.name("c");
SymbolicName innerNodesVariable = Cypher.name("d");
SymbolicName innerNodesListIterator = Cypher.name("e");
ListComprehension innerNodesListComprehension = Cypher.listWith(innerNodesListIterator)
.in(Cypher.name(aliasedPathName)).returning(Functions.nodes(innerNodesListIterator));

FunctionInvocation innerNodesReduce = createInnerReduce(innerNodesAccumulator, innerNodesVariable,
innerNodesListComprehension);

FunctionInvocation outerNodesReduce = createOuterReduce(outerNodesAccumulator, outerNodesVariable,
innerNodesReduce);

// nested relationships flatMap: reduce(...reduce(...))
SymbolicName outerRelationshipsAccumulator = Cypher.name("f");
SymbolicName outerRelationshipsVariable = Cypher.name("g");
SymbolicName innerRelationshipsAccumulator = Cypher.name("h");
SymbolicName innerRelationshipsVariable = Cypher.name("i");
SymbolicName innerRelationshipsListIterator = Cypher.name("j");
ListComprehension innerRelationshipsListComprehension = Cypher.listWith(innerRelationshipsListIterator)
.in(Cypher.name(aliasedPathName)).returning(Functions.relationships(innerRelationshipsListIterator));

FunctionInvocation innerRelationshipReduce = createInnerReduce(innerRelationshipsAccumulator,
innerRelationshipsVariable, innerRelationshipsListComprehension);

FunctionInvocation outerRelationshipsReduce = createOuterReduce(outerRelationshipsAccumulator,
outerRelationshipsVariable, innerRelationshipReduce);

// WITH n, collect(p) as pathPattern
expressions1.add(Constants.NAME_OF_ROOT_NODE);
expressions1.add(Functions.collect(path).as(aliasedPathName));
// WITH n, reduce(nodes) as __sm__, reduce(relationships) as __sr__
expressions2.add(Constants.NAME_OF_ROOT_NODE);
expressions2.add(outerNodesReduce.as(Constants.NAME_OF_SYNTHESIZED_RELATED_NODES));
expressions2.add(outerRelationshipsReduce.as(Constants.NAME_OF_SYNTHESIZED_RELATIONS));

StatementBuilder.OngoingReadingWithoutWhere match = match(path);

if (previousMatches != null) {
match = previousMatches.match(path);
}

return match
.where(conditionOrNoCondition(condition))
.with(expressions1.toArray(new Expression[]{}))
.with(expressions2.toArray(new Expression[]{}));
}

private FunctionInvocation createOuterReduce(SymbolicName outerNodesAccumulator, SymbolicName outerNodesVariable, FunctionInvocation innerNodesReduce) {
return Functions.reduce(outerNodesVariable)
.in(innerNodesReduce)
.map(Cypher.caseExpression()
.when(outerNodesVariable.in(outerNodesAccumulator))
.then(outerNodesAccumulator)
.elseDefault(outerNodesAccumulator.add(outerNodesVariable)))
.accumulateOn(outerNodesAccumulator)
.withInitialValueOf(Cypher.listOf());
}

private FunctionInvocation createInnerReduce(SymbolicName innerNodesAccumulator, SymbolicName innerNodesVariable, ListComprehension innerNodesListComprehension) {
return Functions.reduce(innerNodesVariable)
.in(innerNodesListComprehension)
.map(innerNodesAccumulator.add(innerNodesVariable))
.accumulateOn(innerNodesAccumulator)
.withInitialValueOf(Cypher.listOf());
}

/**
Expand Down Expand Up @@ -332,8 +434,8 @@ public Statement prepareDeleteOf(
.build();
}

public Expression createReturnStatementForMatch(NodeDescription<?> nodeDescription) {
return createReturnStatementForMatch(nodeDescription, null);
public Expression[] createReturnStatementForMatch(NodeDescription<?> nodeDescription) {
return createReturnStatementForMatch(nodeDescription, Collections.emptyList());
}

/**
Expand Down Expand Up @@ -372,20 +474,25 @@ public Expression createReturnStatementForMatch(NodeDescription<?> nodeDescripti

/**
* @param nodeDescription Description of the root node
* @param inputProperties A list of Java properties of the domain to be included. Those properties are compared with
* @param includedProperties A list of Java properties of the domain to be included. Those properties are compared with
* the field names of graph properties respectively relationships.
* @return An expresion to be returned by a Cypher statement
*/
public Expression createReturnStatementForMatch(NodeDescription<?> nodeDescription,
@Nullable List<String> inputProperties) {
public Expression[] createReturnStatementForMatch(NodeDescription<?> nodeDescription,
List<String> includedProperties) {

Predicate<String> includeField = s -> inputProperties == null || inputProperties.isEmpty()
|| inputProperties.contains(s);

SymbolicName nodeName = Constants.NAME_OF_ROOT_NODE;
List<RelationshipDescription> processedRelationships = new ArrayList<>();

return projectPropertiesAndRelationships(nodeDescription, nodeName, includeField, processedRelationships);
if (nodeDescription.containsPossibleCircles(includedProperties)) {
List<Expression> returnExpressions = new ArrayList<>();
Node rootNode = anyNode(Constants.NAME_OF_ROOT_NODE);
returnExpressions.add(rootNode.as(Constants.NAME_OF_SYNTHESIZED_ROOT_NODE));
returnExpressions.add(Cypher.name(Constants.NAME_OF_SYNTHESIZED_RELATED_NODES));
returnExpressions.add(Cypher.name(Constants.NAME_OF_SYNTHESIZED_RELATIONS));
return returnExpressions.toArray(new Expression[]{});
} else {
Predicate<String> includeField = s -> includedProperties.isEmpty() || includedProperties.contains(s);
return new Expression[]{projectPropertiesAndRelationships(nodeDescription, Constants.NAME_OF_ROOT_NODE, includeField, processedRelationships)};
}
}

// recursive entry point for relationships in return statement
Expand All @@ -398,30 +505,23 @@ private MapProjection projectAllPropertiesAndRelationships(NodeDescription<?> no
}

private MapProjection projectPropertiesAndRelationships(NodeDescription<?> nodeDescription, SymbolicName nodeName,
Predicate<String> includeProperty, List<RelationshipDescription> processedRelationships) {
Predicate<String> includedProperties, List<RelationshipDescription> processedRelationships) {

List<Object> propertiesProjection = projectNodeProperties(nodeDescription, nodeName, includeProperty);
List<Object> propertiesProjection = projectNodeProperties(nodeDescription, nodeName, includedProperties);
List<Object> contentOfProjection = new ArrayList<>(propertiesProjection);

Collection<RelationshipDescription> relationships = getRelationshipDescriptionsUpAndDown(nodeDescription);
relationships.removeIf(r -> !includeProperty.test(r.getFieldName()));
Collection<RelationshipDescription> relationships = getRelationshipDescriptionsUpAndDown(nodeDescription, includedProperties);
relationships.removeIf(r -> !includedProperties.test(r.getFieldName()));

if (nodeDescription.containsPossibleCircles()) {
Node node = anyNode(nodeName);
RelationshipPattern pattern = createRelationships(node, relationships);
NamedPath p = Cypher.path("p").definedBy(pattern);
contentOfProjection.add(Constants.NAME_OF_PATHS);
contentOfProjection.add(Cypher.listBasedOn(p).returning(p));
} else {
contentOfProjection.addAll(generateListsFor(relationships, nodeName, processedRelationships));
}
contentOfProjection.addAll(generateListsFor(relationships, nodeName, processedRelationships));
return Cypher.anyNode(nodeName).project(contentOfProjection);
}

@NonNull
static Collection<RelationshipDescription> getRelationshipDescriptionsUpAndDown(NodeDescription<?> nodeDescription) {
Collection<RelationshipDescription> relationships = new HashSet<>(nodeDescription.getRelationships());
static Collection<RelationshipDescription> getRelationshipDescriptionsUpAndDown(NodeDescription<?> nodeDescription,
Predicate<String> includedProperties) {

Collection<RelationshipDescription> relationships = new HashSet<>(nodeDescription.getRelationships());
for (NodeDescription<?> childDescription : nodeDescription.getChildNodeDescriptionsInHierarchy()) {
childDescription.getRelationships().forEach(concreteRelationship -> {

Expand All @@ -432,19 +532,25 @@ static Collection<RelationshipDescription> getRelationshipDescriptionsUpAndDown(
}
});
}
return relationships;

return relationships.stream().filter(relationshipDescription ->
includedProperties.test(relationshipDescription.getFieldName()))
.collect(Collectors.toSet());
}

private RelationshipPattern createRelationships(Node node, Collection<RelationshipDescription> relationshipDescriptions) {
RelationshipPattern relationship;

Direction determinedDirection = determineDirection(relationshipDescriptions);
if (Direction.OUTGOING.equals(determinedDirection)) {
relationship = node.relationshipTo(anyNode(), collectFirstLevelRelationshipTypes(relationshipDescriptions));
relationship = node.relationshipTo(anyNode(), collectFirstLevelRelationshipTypes(relationshipDescriptions))
.min(0).max(1);
} else if (Direction.INCOMING.equals(determinedDirection)) {
relationship = node.relationshipFrom(anyNode(), collectFirstLevelRelationshipTypes(relationshipDescriptions));
relationship = node.relationshipFrom(anyNode(), collectFirstLevelRelationshipTypes(relationshipDescriptions))
.min(0).max(1);
} else {
relationship = node.relationshipBetween(anyNode(), collectFirstLevelRelationshipTypes(relationshipDescriptions));
relationship = node.relationshipBetween(anyNode(), collectFirstLevelRelationshipTypes(relationshipDescriptions))
.min(0).max(1);
}

Set<RelationshipDescription> processedRelationshipDescriptions = new HashSet<>(relationshipDescriptions);
Expand Down

0 comments on commit cc73484

Please sign in to comment.