Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 20 additions & 0 deletions src/main/java/org/apache/sysds/hops/estim/EstimatorBasicAvg.java
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,26 @@ public double estim(MatrixBlock m, OpCode op) {
return estimIntern(m.getDataCharacteristics(), null, op).getSparsity();
}

protected void obtainSparsityInformation(MMNode node) {
MatrixBlock mb = node.getData();
if(mb != null && mb.getDataCharacteristics().getSparsity() == -1) {
if(mb.getNonZeros() != 0) {
mb.getDataCharacteristics().setNonZeros(mb.getNonZeros());
}
else {
mb.recomputeNonZeros();
mb.getDataCharacteristics().setNonZeros(mb.getNonZeros());
}
}
}

@Override
public MMNode getInputMMNode(MatrixBlock mb) {
MMNode node = new MMNode(mb);
obtainSparsityInformation(node);
return node;
}

private DataCharacteristics estimIntern(DataCharacteristics dc1, DataCharacteristics dc2, OpCode op) {
switch (op) {
case MM:
Expand Down
20 changes: 20 additions & 0 deletions src/main/java/org/apache/sysds/hops/estim/EstimatorBasicWorst.java
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,26 @@ public double estim(MatrixBlock m, OpCode op) {
return estimIntern(m.getDataCharacteristics(), null, op).getSparsity();
}

protected void obtainSparsityInformation(MMNode node) {
MatrixBlock mb = node.getData();
if(mb != null && mb.getDataCharacteristics().getSparsity() == -1) {
if(mb.getNonZeros() != 0) {
mb.getDataCharacteristics().setNonZeros(mb.getNonZeros());
}
else {
mb.recomputeNonZeros();
mb.getDataCharacteristics().setNonZeros(mb.getNonZeros());
}
}
}

@Override
public MMNode getInputMMNode(MatrixBlock mb) {
MMNode node = new MMNode(mb);
obtainSparsityInformation(node);
return node;
}

private DataCharacteristics estimIntern(DataCharacteristics mc1, DataCharacteristics mc2, OpCode op) {
switch (op) {
case MM:
Expand Down
11 changes: 9 additions & 2 deletions src/main/java/org/apache/sysds/hops/estim/EstimatorBitsetMM.java
Original file line number Diff line number Diff line change
Expand Up @@ -93,7 +93,14 @@ public double estim(MatrixBlock m, OpCode op) {
return OptimizerUtils.getSparsity(outMap.getNumRows(),
outMap.getNumColumns(), outMap.getNonZeros());
}


@Override
public MMNode getInputMMNode(MatrixBlock mb) {
MMNode node = new MMNode(mb);
getCachedSynopsis(node);
return node;
}

private BitsetMatrix getCachedSynopsis(MMNode node) {
if( node == null )
return null;
Expand All @@ -104,7 +111,7 @@ else if( !node.isLeaf() )
estim(node); //recursively obtain synopsis
return (BitsetMatrix) node.getSynopsis();
}

private static BitsetMatrix estimInternal(BitsetMatrix m1Map, BitsetMatrix m2Map, OpCode op) {
switch(op) {
case MM: return m1Map.matMult(m2Map);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -88,7 +88,14 @@ public double estim(MatrixBlock m1, MatrixBlock m2, OpCode op) {
public double estim(MatrixBlock m, OpCode op) {
return estim(m, null, op);
}


@Override
public MMNode getInputMMNode(MatrixBlock mb) {
MMNode node = new MMNode(mb);
getCachedSynopsis(node);
return node;
}

private DensityMap getCachedSynopsis(MMNode node) {
if( node == null )
return null;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -115,6 +115,11 @@ public double estim(MatrixBlock m1, MatrixBlock m2) {
m1.getNumRows(), m2.getNumColumns(), graph.estimateNnz());
}

@Override
public MMNode getInputMMNode(MatrixBlock mb) {
throw new UnsupportedOperationException("Not implemented yet.");
}

private static LayeredGraph estimInternal(LayeredGraph lg1, LayeredGraph lg2, OpCode op) {
switch(op) {
case MM: return lg1.matMult(lg2);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -105,7 +105,14 @@ public double estim(MatrixBlock m1, OpCode op) {
MatrixHistogram h1 = new MatrixHistogram(m1, _useExtended);
return estimIntern(h1, null, op, null);
}


@Override
public MMNode getInputMMNode(MatrixBlock mb) {
MMNode node = new MMNode(mb);
getCachedSynopsis(node);
return node;
}

private MatrixHistogram getCachedSynopsis(MMNode node) {
if( node == null )
return null;
Expand All @@ -116,7 +123,7 @@ else if( !node.isLeaf() )
estim(node, false); //recursively obtain synopsis
return (MatrixHistogram) node.getSynopsis();
}

public double estimIntern(MatrixHistogram h1, MatrixHistogram h2, OpCode op, long[] misc) {
double msize = (double)h1.getRows()*h1.getCols();
switch (op) {
Expand Down Expand Up @@ -314,6 +321,7 @@ else if( !in.isEmpty() ) {
}
}
}
in.setNonZeros(IntStream.of(cNnz).sum());
}

public MatrixHistogram(int[] r, int[] r1e, int[] c, int[] c1e, int rmax, int cmax) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -72,6 +72,13 @@ public double estim(MatrixBlock m1, OpCode op) {
return DoubleStream.of(rsOut).average().orElse(0);
}

@Override
public MMNode getInputMMNode(MatrixBlock mb) {
MMNode node = new MMNode(mb);
throw new UnsupportedOperationException("TODO: Implement getCachedSynopsis similarly to EstimatorMatrixHistogram.");
// return node;
}

private double[] estimInternChain(MMNode node) {
return estimInternChain(node, null, null);
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -197,7 +197,12 @@ private static int[] computeColumnNnz(MatrixBlock in, int[] ix) {
ret[i] = nnz[ix[i]];
return ret;
}


@Override
public MMNode getInputMMNode(MatrixBlock mb) {
throw new UnsupportedOperationException("Not implemented yet.");
}

private static int[] computeRowNnz(MatrixBlock in, int[] ix) {
//copy nnz into reduced vector
int[] ret = new int[ix.length];
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -127,7 +127,12 @@ public double estim(MatrixBlock m1, MatrixBlock m2) {
return OptimizerUtils.getSparsity(
m1.getNumRows(), m2.getNumColumns(), nnz);
}


@Override
public MMNode getInputMMNode(MatrixBlock mb) {
throw new UnsupportedOperationException("Not implemented yet.");
}

private void initHashArrays(int m, int n, int l) {
if( h1 == null ) {
h1 = new double[m];
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -85,7 +85,9 @@ public static enum OpCode {
* @return sparsity
*/
public abstract double estim(MatrixBlock m, OpCode op);


public abstract MMNode getInputMMNode(MatrixBlock mb);

protected boolean isExactMetadataOp(OpCode op) {
return ArrayUtils.contains(EXACT_META_DATA_OPS, op);
}
Expand Down
13 changes: 11 additions & 2 deletions src/main/java/org/apache/sysds/hops/recompile/Recompiler.java
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,7 @@
import org.apache.sysds.common.Types.ValueType;
import org.apache.sysds.conf.CompilerConfig.ConfigType;
import org.apache.sysds.conf.ConfigurationManager;
import org.apache.sysds.conf.DMLConfig;
import org.apache.sysds.hops.DataGenOp;
import org.apache.sysds.hops.DataOp;
import org.apache.sysds.hops.FunctionOp;
Expand All @@ -61,6 +62,7 @@
import org.apache.sysds.hops.codegen.SpoofCompiler;
import org.apache.sysds.hops.rewrite.HopRewriteUtils;
import org.apache.sysds.hops.rewrite.ProgramRewriter;
import org.apache.sysds.hops.rewrite.SparsityOptimizer;
import org.apache.sysds.lops.Lop;
import org.apache.sysds.lops.compile.Dag;
import org.apache.sysds.lops.rewrite.LopRewriter;
Expand Down Expand Up @@ -337,7 +339,7 @@ else if( !codegen ) {
for( Hop hopRoot : hops )
rClearLops( hopRoot );
}

// get max parallelism constraint, see below
Hop.resetVisitStatus(hops);
int maxK = rGetMaxParallelism(hops);
Expand Down Expand Up @@ -396,7 +398,14 @@ else if( !codegen ) {
hops = SpoofCompiler.optimize(hops,
(status==null || !status.isInitialCodegen()));
}


if(ConfigurationManager.getDMLConfig().getBooleanValue(DMLConfig.SPARSITY_RECOMPILE)) {
Hop.resetVisitStatus(hops);
boolean executed = SparsityOptimizer.optimizeAndExecute(hops, ec);
if(executed)
return null;
}

// set max parallelism constraint to ensure compilation
// incl rewrites does not lose these hop-lop constraints
Hop.resetVisitStatus(hops);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -149,9 +149,6 @@ public ProgramRewriter(boolean staticRewrites, boolean dynamicRewrites)
if(ConfigurationManager.getDMLConfig().getBooleanValue(DMLConfig.SPARSITY_REWRITES)) {
_dagRuleSet.add( new RewriteMatrixMultChainOptimizationSparse() );
}
if( ConfigurationManager.getDMLConfig().getBooleanValue(DMLConfig.SPARSITY_RECOMPILE) ) {
_dagRuleSet.add(new SparsityOptimizer());
}
if( OptimizerUtils.ALLOW_ALGEBRAIC_SIMPLIFICATION ) {
_dagRuleSet.add( new RewriteAlgebraicSimplificationDynamic() ); //dependencies: cse
_dagRuleSet.add( new RewriteAlgebraicSimplificationStatic() ); //dependencies: cse
Expand Down
Loading
Loading