Skip to content
Snippets Groups Projects
Commit feb6f025 authored by Vojtech Moravec's avatar Vojtech Moravec
Browse files

CLI options.

parent ef2d12ac
No related branches found
No related tags found
No related merge requests found
......@@ -72,7 +72,7 @@ public class BigDataServer {
final String thumbnailDirectory = null;
final boolean enableManagerContext = false;
return new Parameters(port, hostname, new HashMap<String, String>(), thumbnailDirectory, enableManagerContext,
new CustomCompressionParameters("", "", 8, false, false));
new CustomCompressionParameters("", "", 8, false, false, -1));
}
public static void main(final String[] args) throws Exception {
......@@ -220,6 +220,7 @@ public class BigDataServer {
final String ENABLE_COMPRESSION_DIFF = "diff";
final String DUMP_FILE = "dump";
final String TRAIN_FILE = "train";
final String DIFF_THRESHOLD = "diffthreshold";
// create Options object
final Options options = new Options();
......@@ -281,6 +282,12 @@ public class BigDataServer {
.withDescription("Send compression difference")
.create(ENABLE_COMPRESSION_DIFF));
options.addOption(OptionBuilder
.withDescription("Render difference above this threshold")
.hasArg()
.withArgName(DIFF_THRESHOLD)
.create(DIFF_THRESHOLD));
if (Constants.ENABLE_EXPERIMENTAL_FEATURES) {
options.addOption(OptionBuilder
.withDescription("enable statistics and manager context. EXPERIMENTAL!")
......@@ -312,6 +319,10 @@ public class BigDataServer {
final boolean enableCompressionDiff = cmd.hasOption(ENABLE_COMPRESSION_DIFF);
final String trainFile = cmd.getOptionValue(TRAIN_FILE, "");
final int bitTarget = Integer.parseInt(cmd.getOptionValue(BIT_TARGET, "8"));
final int diffThreshold = Integer.parseInt(cmd.getOptionValue(DIFF_THRESHOLD, "-1"));
if (diffThreshold > -1) {
LOG.info("Diff threshold is set to: " + diffThreshold);
}
if ((enableCompression || enableCompressionDiff) && (trainFile.isEmpty())) {
throw new MissingArgumentException(String.format("!!! %s must be specified when %s or %s is specified !!!",
......@@ -319,7 +330,7 @@ public class BigDataServer {
}
final CustomCompressionParameters customCompParams = new CustomCompressionParameters(dumpFile, trainFile, bitTarget,
enableCompression, enableCompressionDiff);
enableCompression, enableCompressionDiff, diffThreshold);
LOG.info("Compression is " + (enableCompression ? "Matched" : "Not matched"));
LOG.info("Compression-Diff is " + (enableCompressionDiff ? "Matched" : "Not matched"));
......
......@@ -11,6 +11,9 @@ import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import compression.U16;
import compression.data.Chunk3D;
import compression.data.V3i;
import compression.data.V3l;
import compression.quantization.scalar.ScalarQuantizer;
import compression.utilities.Utils;
import org.eclipse.jetty.server.Request;
......@@ -149,59 +152,73 @@ public class CellHandler extends ContextHandler {
final int level = Integer.parseInt(parts[4]);
final Key key = new VolatileGlobalCellCache.Key(timepoint, setup, level, index);
VolatileCell<?> cell = cache.getLoadingVolatileCache().getIfPresent(key, cacheHints);
final int[] cellDims = new int[]{
Integer.parseInt(parts[5]),
Integer.parseInt(parts[6]),
Integer.parseInt(parts[7])};
final long[] cellMin = new long[]{
Long.parseLong(parts[8]),
Long.parseLong(parts[9]),
Long.parseLong(parts[10])};
if (cell == null) {
final int[] cellDims = new int[]{
Integer.parseInt(parts[5]),
Integer.parseInt(parts[6]),
Integer.parseInt(parts[7])};
final long[] cellMin = new long[]{
Long.parseLong(parts[8]),
Long.parseLong(parts[9]),
Long.parseLong(parts[10])};
cell = cache.getLoadingVolatileCache().get(key, cacheHints, new VolatileCellLoader<>(loader, timepoint, setup, level, cellDims, cellMin));
}
@SuppressWarnings("unchecked")
short[] data = ((VolatileCell<VolatileShortArray>) cell).getData().getCurrentStorageArray();
Chunk3D boxChunk = new Chunk3D(new V3i(cellDims[0], cellDims[1], cellDims[2]), data);
boxChunk.setOffsets(cellMin[0], cellMin[1], cellMin[2]);
//System.out.println(boxChunk.toString());
// TODO(Moravec): When chunking boxes of non-uniform dimensions, some data cells are missed!
if (cellDims[0] != 16 || cellDims[1] != 16 || cellDims[2] != 16) {
boxChunk.divideIntoChunks(new V3i(2, 2, 2));
}
//boxChunk.divideIntoChunks(new V3i(3,3,3));
if (compressionParams.shouldCompressData()) {
assert (quantizer != null) : "Compressor wasn't created";
data = quantizer.quantize(data);
} else if (compressionParams.renderDifference()) {
final int diffThreshold = compressionParams.getDiffThreshold();
assert (quantizer != null) : "Compressor wasn't created";
short[] compressedData = quantizer.quantize(data);
int e1 = 0, e2 = 0, e3 = 0, e4 = 0, e5 = 0;
int maxError = Integer.MIN_VALUE;
for (int i = 0; i < data.length; i++) {
final int diff = Math.abs(compressedData[i] - data[i]);
if (diff < 100) {
data[i] = 1000;
++e1;
} else if (diff < 200) {
data[i] = 2000;
++e2;
} else if (diff < 300) {
data[i] = 3000;
++e3;
} else if (diff < 400) {
data[i] = 4000;
++e4;
} else if (diff < 500) {
data[i] = 5000;
++e5;
} else {
data[i] = 0x0;
}
// assert (data[i] != 0 || (data[i] == 0 && compressedData[i] == 0)) : "BAD";
// data[i] = (short) (5 * Utils.u16BitsToShort(Math.abs(compressedData[i] - data[i])));
maxError = Math.max(maxError, diff);
// NOTE(Moravec): We know that we are not producing non-existing values.
// There was no data but now there is.
// if (data[i] == 0 && compressedData[i] != 0) {
// data[i] = (short) 0xffff;
// ++nonExistingDataCount;
// } else {
// data[i] = (short) 0x0;
// }
// if (compressedData[i] > data[i]) {
// data[i] = compressedData[i];
// } else {
// data[i] = 0;
// }
// if (diff > diffThreshold) {
// data[i] = (short) diff;
// } else {
// data[i] = 0;
// }
// NOTE(Moravec): Squared error
final short squaredError = (short) Math.floor(Math.pow(compressedData[i] - data[i], 2));
data[i] = squaredError;
}
if (maxError > 0) {
System.out.println("Max error: " + maxError);
}
LOG.info(String.format("E1: %.2f E2: %.2f E3: %.2f E4: %.2f E5: %.2f",
(float)e1/(float)data.length,
(float)e2/(float)data.length,
(float)e3/(float)data.length,
(float)e4/(float)data.length,
(float)e5/(float)data.length));
//LOG.warn("Not yet implemented.");
}
final byte[] buf = new byte[2 * data.length];
......
......@@ -5,6 +5,7 @@ public class CustomCompressionParameters {
private final String dumpFile;
private final String trainFile;
private final int bitTarget;
private final int diffThreshold;
public String getTrainFile() {
return trainFile;
......@@ -21,13 +22,15 @@ public class CustomCompressionParameters {
final String trainFile,
final int bitTarget,
final boolean enableRequestCompression,
final boolean renderDifference) {
final boolean renderDifference,
final int diffThreshold) {
this.dumpFile = dumpFile;
this.trainFile = trainFile;
this.bitTarget=bitTarget;
this.dumpRequestData = !this.dumpFile.isEmpty();
this.enableRequestCompression = enableRequestCompression;
this.renderDifference = renderDifference;
this.diffThreshold = diffThreshold;
}
public boolean shouldDumpRequestData() {
......@@ -46,4 +49,7 @@ public class CustomCompressionParameters {
return renderDifference;
}
public int getDiffThreshold() {
return diffThreshold;
}
}
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment