Skip to content

Commit

Permalink
Remove write columns from NodeSim stats result
Browse files Browse the repository at this point in the history 
We've already split NodeSim stats proc and config out. There is no
reason for it to use the `write()` method or the `WriteResult` from the
write mode proc anymore.

Co-Authored-By: Jacob Sznajdman <breakanalysis@gmail.com>
  • Loading branch information
@jjaderberg @breakanalysis
jjaderberg and breakanalysis committed Feb 7, 2020
1 parent 1fb6b9f commit 9d32db7
Show file tree
Hide file tree
Showing 4 changed files with 308 additions and 183 deletions.
180 changes: 1 addition & 179 deletions proc/src/main/java/org/neo4j/graphalgo/nodesim/NodeSimilarityBaseProc.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,17 +23,10 @@
import org.neo4j.graphalgo.AlgoBaseProc;
import org.neo4j.graphalgo.AlgorithmFactory;
import org.neo4j.graphalgo.api.Graph;
import org.neo4j.graphalgo.compat.MapUtil;
import org.neo4j.graphalgo.core.CypherMapWrapper;
import org.neo4j.graphalgo.core.utils.ProgressTimer;
import org.neo4j.graphalgo.core.write.RelationshipExporter;
import org.neo4j.graphalgo.newapi.GraphCreateConfig;
import org.neo4j.graphalgo.result.AbstractResultBuilder;

import java.util.Collections;
import java.util.Map;
import java.util.Optional;
import java.util.stream.Stream;

public abstract class NodeSimilarityBaseProc<CONFIG extends NodeSimilarityBaseConfig> extends AlgoBaseProc<NodeSimilarity, NodeSimilarityResult, CONFIG> {

Expand All @@ -54,82 +47,7 @@ protected AlgorithmFactory<NodeSimilarity, CONFIG> algorithmFactory(CONFIG confi
return new NodeSimilarityFactory<>();
}

public Stream<WriteResult> write(
ComputationResult<NodeSimilarity, NodeSimilarityResult, CONFIG> computationResult
) {
CONFIG config = computationResult.config();

if (computationResult.isGraphEmpty()) {
return Stream.of(
new WriteResult(
computationResult.createMillis(),
0,
0,
0,
0,
0,
Collections.emptyMap(),
config.toMap()
)
);
}

NodeSimilarityResult result = computationResult.result();
NodeSimilarity algorithm = computationResult.algorithm();
SimilarityGraphResult similarityGraphResult = result.maybeGraphResult().get();
Graph similarityGraph = similarityGraphResult.similarityGraph();

WriteResultBuilder resultBuilder = new WriteResultBuilder();
resultBuilder
.withNodesCompared(similarityGraphResult.comparedNodes())
.withRelationshipsWritten(similarityGraphResult.similarityGraph().relationshipCount());
resultBuilder.withCreateMillis(computationResult.createMillis());
resultBuilder.withComputeMillis(computationResult.computeMillis());
resultBuilder.withConfig(config);

boolean shouldComputeHistogram = callContext
.outputFields()
.anyMatch(s -> s.equalsIgnoreCase("similarityDistribution"));
if (shouldWrite(config) && similarityGraph.relationshipCount() > 0) {
NodeSimilarityWriteConfig writeConfig = (NodeSimilarityWriteConfig) config;

String writeRelationshipType = writeConfig.writeRelationshipType();
String writeProperty = writeConfig.writeProperty();

runWithExceptionLogging(
"NodeSimilarity write-back failed",
() -> {
try (ProgressTimer ignored = ProgressTimer.start(resultBuilder::withWriteMillis)) {
RelationshipExporter exporter = RelationshipExporter
.of(api, similarityGraph, algorithm.getTerminationFlag())
.withLog(log)
.build();
if (shouldComputeHistogram) {
DoubleHistogram histogram = new DoubleHistogram(5);
exporter.write(
writeRelationshipType,
writeProperty,
(node1, node2, similarity) -> {
histogram.recordValue(similarity);
return true;
}
);
resultBuilder.withHistogram(histogram);
} else {
exporter.write(writeRelationshipType, writeProperty);
}
}
}
);
} else if (shouldComputeHistogram) {
try (ProgressTimer ignored = resultBuilder.timePostProcessing()) {
resultBuilder.withHistogram(computeHistogram(similarityGraph));
}
}
return Stream.of(resultBuilder.build());
}

private DoubleHistogram computeHistogram(Graph similarityGraph) {
DoubleHistogram computeHistogram(Graph similarityGraph) {
DoubleHistogram histogram = new DoubleHistogram(5);
similarityGraph.forEachNode(nodeId -> {
similarityGraph.forEachRelationship(nodeId, Double.NaN, (node1, node2, property) -> {
Expand All @@ -141,100 +59,4 @@ private DoubleHistogram computeHistogram(Graph similarityGraph) {
return histogram;
}

public static class WriteResult {
public final long loadMillis;
public final long computeMillis;
public final long writeMillis;
public final long postProcessingMillis;

public final long nodesCompared;
public final long relationshipsWritten;

public final Map<String, Object> similarityDistribution;
public final Map<String, Object> configuration;

WriteResult(
long loadMillis,
long computeMillis,
long writeMillis,
long postProcessingMillis,
long nodesCompared,
long relationshipsWritten,
Map<String, Object> similarityDistribution,
Map<String, Object> configuration
) {
this.loadMillis = loadMillis;
this.computeMillis = computeMillis;
this.writeMillis = writeMillis;
this.postProcessingMillis = postProcessingMillis;
this.nodesCompared = nodesCompared;
this.relationshipsWritten = relationshipsWritten;
this.similarityDistribution = similarityDistribution;
this.configuration = configuration;
}
}

static class WriteResultBuilder extends AbstractResultBuilder<WriteResult> {

private long nodesCompared = 0L;

private long postProcessingMillis = -1L;

private Optional<DoubleHistogram> maybeHistogram = Optional.empty();

public WriteResultBuilder withNodesCompared(long nodesCompared) {
this.nodesCompared = nodesCompared;
return this;
}

WriteResultBuilder withHistogram(DoubleHistogram histogram) {
this.maybeHistogram = Optional.of(histogram);
return this;
}

void setPostProcessingMillis(long postProcessingMillis) {
this.postProcessingMillis = postProcessingMillis;
}

ProgressTimer timePostProcessing() {
return ProgressTimer.start(this::setPostProcessingMillis);
}

private Map<String, Object> distribution() {
if (maybeHistogram.isPresent()) {
DoubleHistogram definitelyHistogram = maybeHistogram.get();
return MapUtil.map(
"min", definitelyHistogram.getMinValue(),
"max", definitelyHistogram.getMaxValue(),
"mean", definitelyHistogram.getMean(),
"stdDev", definitelyHistogram.getStdDeviation(),
"p1", definitelyHistogram.getValueAtPercentile(1),
"p5", definitelyHistogram.getValueAtPercentile(5),
"p10", definitelyHistogram.getValueAtPercentile(10),
"p25", definitelyHistogram.getValueAtPercentile(25),
"p50", definitelyHistogram.getValueAtPercentile(50),
"p75", definitelyHistogram.getValueAtPercentile(75),
"p90", definitelyHistogram.getValueAtPercentile(90),
"p95", definitelyHistogram.getValueAtPercentile(95),
"p99", definitelyHistogram.getValueAtPercentile(99),
"p100", definitelyHistogram.getValueAtPercentile(100)
);
}
return Collections.emptyMap();
}

@Override
public WriteResult build() {
return new WriteResult(
createMillis,
computeMillis,
writeMillis,
postProcessingMillis,
nodesCompared,
relationshipsWritten,
distribution(),
config.toMap()
);
}
}
}
139 changes: 137 additions & 2 deletions proc/src/main/java/org/neo4j/graphalgo/nodesim/NodeSimilarityStatsProc.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,15 +19,21 @@
*/
package org.neo4j.graphalgo.nodesim;

import org.HdrHistogram.DoubleHistogram;
import org.neo4j.graphalgo.AlgoBaseProc;
import org.neo4j.graphalgo.api.Graph;
import org.neo4j.graphalgo.compat.MapUtil;
import org.neo4j.graphalgo.core.CypherMapWrapper;
import org.neo4j.graphalgo.core.utils.ProgressTimer;
import org.neo4j.graphalgo.newapi.GraphCreateConfig;
import org.neo4j.graphalgo.result.AbstractResultBuilder;
import org.neo4j.graphalgo.results.MemoryEstimateResult;
import org.neo4j.procedure.Description;
import org.neo4j.procedure.Mode;
import org.neo4j.procedure.Name;
import org.neo4j.procedure.Procedure;

import java.util.Collections;
import java.util.Map;
import java.util.Optional;
import java.util.stream.Stream;
Expand All @@ -38,15 +44,15 @@ public class NodeSimilarityStatsProc extends NodeSimilarityBaseProc<NodeSimilari

@Procedure(name = "gds.nodeSimilarity.stats", mode = Mode.WRITE)
@Description(STATS_DESCRIPTION)
public Stream<NodeSimilarityBaseProc.WriteResult> stats(
public Stream<StatsResult> stats(
@Name(value = "graphName") Object graphNameOrConfig,
@Name(value = "configuration", defaultValue = "{}") Map<String, Object> configuration
) {
AlgoBaseProc.ComputationResult<NodeSimilarity, NodeSimilarityResult, NodeSimilarityStatsConfig> result = compute(
graphNameOrConfig,
configuration
);
return write(result);
return stats(result);
}

@Procedure(value = "gds.nodeSimilarity.stats.estimate", mode = READ)
Expand All @@ -67,4 +73,133 @@ protected NodeSimilarityStatsConfig newConfig(
) {
return NodeSimilarityStatsConfig.of(username, graphName, maybeImplicitCreate, config);
}

public Stream<StatsResult> stats(ComputationResult<NodeSimilarity, NodeSimilarityResult, NodeSimilarityStatsConfig> computationResult) {
NodeSimilarityStatsConfig config = computationResult.config();

if (computationResult.isGraphEmpty()) {
return Stream.of(
new StatsResult(
computationResult.createMillis(),
0,
0,
0,
Collections.emptyMap(),
config.toMap()
)
);
}

NodeSimilarityResult result = computationResult.result();
SimilarityGraphResult similarityGraphResult = result.maybeGraphResult().get();
Graph similarityGraph = similarityGraphResult.similarityGraph();

StatsResultBuilder resultBuilder = new StatsResultBuilder();
resultBuilder
.withNodesCompared(similarityGraphResult.comparedNodes())
.withRelationshipsWritten(similarityGraph.relationshipCount());
resultBuilder.withCreateMillis(computationResult.createMillis());
resultBuilder.withComputeMillis(computationResult.computeMillis());
resultBuilder.withConfig(config);

boolean shouldComputeHistogram = callContext
.outputFields()
.anyMatch(s -> s.equalsIgnoreCase("similarityDistribution"));
if (shouldComputeHistogram) {
try (ProgressTimer ignored = resultBuilder.timePostProcessing()) {
resultBuilder.withHistogram(computeHistogram(similarityGraph));
}
}
return Stream.of(resultBuilder.build());
}


public static final class StatsResult {

public long createMillis;
public long computeMillis;
public long postProcessingMillis;
public long nodesCompared;
public Map<String, Object> similarityDistribution;
public Map<String, Object> configuration;

StatsResult(
long createMillis,
long computeMillis,
long postProcessingMillis,
long nodesCompared,
Map<String, Object> communityDistribution,
Map<String, Object> configuration

) {
this.createMillis = createMillis;
this.computeMillis = computeMillis;
this.postProcessingMillis = postProcessingMillis;
this.nodesCompared = nodesCompared;
this.similarityDistribution = communityDistribution;
this.configuration = configuration;
}
}

static class StatsResultBuilder extends AbstractResultBuilder<StatsResult> {

private long nodesCompared = 0L;

private long postProcessingMillis = -1L;

private Optional<DoubleHistogram> maybeHistogram = Optional.empty();

public StatsResultBuilder withNodesCompared(long nodesCompared) {
this.nodesCompared = nodesCompared;
return this;
}

StatsResultBuilder withHistogram(DoubleHistogram histogram) {
this.maybeHistogram = Optional.of(histogram);
return this;
}

void setPostProcessingMillis(long postProcessingMillis) {
this.postProcessingMillis = postProcessingMillis;
}

ProgressTimer timePostProcessing() {
return ProgressTimer.start(this::setPostProcessingMillis);
}

private Map<String, Object> distribution() {
if (maybeHistogram.isPresent()) {
DoubleHistogram definitelyHistogram = maybeHistogram.get();
return MapUtil.map(
"min", definitelyHistogram.getMinValue(),
"max", definitelyHistogram.getMaxValue(),
"mean", definitelyHistogram.getMean(),
"stdDev", definitelyHistogram.getStdDeviation(),
"p1", definitelyHistogram.getValueAtPercentile(1),
"p5", definitelyHistogram.getValueAtPercentile(5),
"p10", definitelyHistogram.getValueAtPercentile(10),
"p25", definitelyHistogram.getValueAtPercentile(25),
"p50", definitelyHistogram.getValueAtPercentile(50),
"p75", definitelyHistogram.getValueAtPercentile(75),
"p90", definitelyHistogram.getValueAtPercentile(90),
"p95", definitelyHistogram.getValueAtPercentile(95),
"p99", definitelyHistogram.getValueAtPercentile(99),
"p100", definitelyHistogram.getValueAtPercentile(100)
);
}
return Collections.emptyMap();
}

@Override
public StatsResult build() {
return new StatsResult(
createMillis,
computeMillis,
postProcessingMillis,
nodesCompared,
distribution(),
config.toMap()
);
}
}
}
Loading

0 comments on commit 9d32db7

Please sign in to comment.