Compare commits
11 Commits
druid_0.18
...
main
| Author | SHA1 | Date | |
|---|---|---|---|
|
81b3cbded3 | ||
|
|
8c546e20d7 | ||
|
|
9a6c44112e | ||
|
|
38c22db84d | ||
|
|
25ab1b3f9d | ||
|
|
95e6e07ed9 | ||
|
|
00db131a55 | ||
|
|
eb64880203 | ||
|
|
14d06ca0bc | ||
|
|
ce13bd16de | ||
|
|
26bb13fd74 |
@@ -5,7 +5,7 @@
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
|
||||
<groupId>org.apache.druid.extensions</groupId>
|
||||
<artifactId>druid-hdrhistogram_0.18.1</artifactId>
|
||||
<artifactId>druid-hdrhistogram_26.0.0</artifactId>
|
||||
<name>druid-hdrhistogram</name>
|
||||
<version>1.0-SNAPSHOT</version>
|
||||
|
||||
@@ -14,7 +14,7 @@
|
||||
<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
|
||||
<maven.compiler.source>1.8</maven.compiler.source>
|
||||
<maven.compiler.target>1.8</maven.compiler.target>
|
||||
<druid.version>0.18.1</druid.version>
|
||||
<druid.version>26.0.0</druid.version>
|
||||
</properties>
|
||||
|
||||
<dependencies>
|
||||
@@ -45,6 +45,13 @@
|
||||
</dependency>
|
||||
|
||||
<!-- Tests -->
|
||||
<dependency>
|
||||
<groupId>org.easymock</groupId>
|
||||
<artifactId>easymock</artifactId>
|
||||
<version>4.3</version>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.apache.druid</groupId>
|
||||
<artifactId>druid-processing</artifactId>
|
||||
@@ -54,9 +61,17 @@
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.druid</groupId>
|
||||
<artifactId>druid-benchmarks</artifactId>
|
||||
<artifactId>druid-server</artifactId>
|
||||
<version>${druid.version}</version>
|
||||
<scope>test</scope>
|
||||
<type>test-jar</type>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.druid</groupId>
|
||||
<artifactId>druid-sql</artifactId>
|
||||
<version>${druid.version}</version>
|
||||
<type>test-jar</type>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>junit</groupId>
|
||||
|
||||
@@ -1,361 +1,388 @@
|
||||
package org.HdrHistogram; /**
|
||||
* Written by Gil Tene of Azul Systems, and released to the public domain,
|
||||
* as explained at http://creativecommons.org/publicdomain/zero/1.0/
|
||||
*
|
||||
* @author Gil Tene
|
||||
*/
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.ObjectInputStream;
|
||||
import java.nio.ByteBuffer;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.List;
|
||||
import java.util.zip.DataFormatException;
|
||||
|
||||
/**
|
||||
* <h3>A High Dynamic Range (HDR) Histogram</h3>
|
||||
* <p>
|
||||
* {@link ArrayHistogram} supports the recording and analyzing sampled data value counts across a configurable integer value
|
||||
* range with configurable value precision within the range. Value precision is expressed as the number of significant
|
||||
* digits in the value recording, and provides control over value quantization behavior across the value range and the
|
||||
* subsequent value resolution at any given level.
|
||||
* <p>
|
||||
* For example, a Histogram could be configured to track the counts of observed integer values between 0 and
|
||||
* 3,600,000,000 while maintaining a value precision of 3 significant digits across that range. Value quantization
|
||||
* within the range will thus be no larger than 1/1,000th (or 0.1%) of any value. This example Histogram could
|
||||
* be used to track and analyze the counts of observed response times ranging between 1 microsecond and 1 hour
|
||||
* in magnitude, while maintaining a value resolution of 1 microsecond up to 1 millisecond, a resolution of
|
||||
* 1 millisecond (or better) up to one second, and a resolution of 1 second (or better) up to 1,000 seconds. At its
|
||||
* maximum tracked value (1 hour), it would still maintain a resolution of 3.6 seconds (or better).
|
||||
* <p>
|
||||
* Histogram tracks value counts in <b><code>long</code></b> fields. Smaller field types are available in the
|
||||
* {@link IntCountsHistogram} and {@link ShortCountsHistogram} implementations of
|
||||
* {@link AbstractHistogram}.
|
||||
* <p>
|
||||
* Auto-resizing: When constructed with no specified value range range (or when auto-resize is turned on with {@link
|
||||
* ArrayHistogram#setAutoResize}) a {@link ArrayHistogram} will auto-resize its dynamic range to include recorded values as
|
||||
* they are encountered. Note that recording calls that cause auto-resizing may take longer to execute, as resizing
|
||||
* incurs allocation and copying of internal data structures.
|
||||
* <p>
|
||||
* See package description for {@link org.HdrHistogram} for details.
|
||||
*/
|
||||
|
||||
public class ArrayHistogram extends AbstractHistogram implements Histogramer{
|
||||
long totalCount;
|
||||
long[] counts;
|
||||
int normalizingIndexOffset;
|
||||
|
||||
@Override
|
||||
long getCountAtIndex(final int index) {
|
||||
return counts[normalizeIndex(index, normalizingIndexOffset, countsArrayLength)];
|
||||
}
|
||||
|
||||
@Override
|
||||
long getCountAtNormalizedIndex(final int index) {
|
||||
return counts[index];
|
||||
}
|
||||
|
||||
@Override
|
||||
void incrementCountAtIndex(final int index) {
|
||||
counts[normalizeIndex(index, normalizingIndexOffset, countsArrayLength)]++;
|
||||
}
|
||||
|
||||
@Override
|
||||
void addToCountAtIndex(final int index, final long value) {
|
||||
// 正常情况下normalizingIndexOffset = 0, index不用偏移
|
||||
counts[normalizeIndex(index, normalizingIndexOffset, countsArrayLength)] += value;
|
||||
}
|
||||
|
||||
@Override
|
||||
void setCountAtIndex(int index, long value) {
|
||||
counts[normalizeIndex(index, normalizingIndexOffset, countsArrayLength)] = value;
|
||||
}
|
||||
|
||||
@Override
|
||||
void setCountAtNormalizedIndex(int index, long value) {
|
||||
counts[index] = value;
|
||||
}
|
||||
|
||||
@Override
|
||||
int getNormalizingIndexOffset() {
|
||||
return normalizingIndexOffset;
|
||||
}
|
||||
|
||||
@Override
|
||||
void setNormalizingIndexOffset(int normalizingIndexOffset) {
|
||||
this.normalizingIndexOffset = normalizingIndexOffset;
|
||||
}
|
||||
|
||||
@Override
|
||||
void setIntegerToDoubleValueConversionRatio(double integerToDoubleValueConversionRatio) {
|
||||
nonConcurrentSetIntegerToDoubleValueConversionRatio(integerToDoubleValueConversionRatio);
|
||||
}
|
||||
|
||||
@Override
|
||||
void shiftNormalizingIndexByOffset(int offsetToAdd,
|
||||
boolean lowestHalfBucketPopulated,
|
||||
double newIntegerToDoubleValueConversionRatio) {
|
||||
nonConcurrentNormalizingIndexShift(offsetToAdd, lowestHalfBucketPopulated);
|
||||
}
|
||||
|
||||
@Override
|
||||
void clearCounts() {
|
||||
Arrays.fill(counts, 0);
|
||||
totalCount = 0;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Histogramer makeCopy() {
|
||||
return miniCopy();
|
||||
}
|
||||
|
||||
@Override
|
||||
public ArrayHistogram copy() {
|
||||
ArrayHistogram copy = new ArrayHistogram(this);
|
||||
copy.add(this);
|
||||
return copy;
|
||||
}
|
||||
|
||||
public ArrayHistogram miniCopy() {
|
||||
ArrayHistogram copy = new ArrayHistogram(lowestDiscernibleValue, maxValue < highestTrackableValue ? Math.max(maxValue, lowestDiscernibleValue * 2) : highestTrackableValue, numberOfSignificantValueDigits);
|
||||
copy.add(this);
|
||||
return copy;
|
||||
}
|
||||
|
||||
@Override
|
||||
public ArrayHistogram copyCorrectedForCoordinatedOmission(final long expectedIntervalBetweenValueSamples) {
|
||||
ArrayHistogram copy = new ArrayHistogram(this);
|
||||
copy.addWhileCorrectingForCoordinatedOmission(this, expectedIntervalBetweenValueSamples);
|
||||
return copy;
|
||||
}
|
||||
|
||||
@Override
|
||||
public long getTotalCount() {
|
||||
return totalCount;
|
||||
}
|
||||
|
||||
@Override
|
||||
void setTotalCount(final long totalCount) {
|
||||
this.totalCount = totalCount;
|
||||
}
|
||||
|
||||
@Override
|
||||
void incrementTotalCount() {
|
||||
totalCount++;
|
||||
}
|
||||
|
||||
@Override
|
||||
void addToTotalCount(final long value) {
|
||||
totalCount += value;
|
||||
}
|
||||
|
||||
@Override
|
||||
int _getEstimatedFootprintInBytes() {
|
||||
return (512 + (8 * counts.length));
|
||||
}
|
||||
|
||||
@Override
|
||||
void resize(long newHighestTrackableValue) {
|
||||
int oldNormalizedZeroIndex = normalizeIndex(0, normalizingIndexOffset, countsArrayLength);
|
||||
|
||||
establishSize(newHighestTrackableValue);
|
||||
|
||||
int countsDelta = countsArrayLength - counts.length;
|
||||
|
||||
counts = Arrays.copyOf(counts, countsArrayLength);
|
||||
|
||||
if (oldNormalizedZeroIndex != 0) {
|
||||
// We need to shift the stuff from the zero index and up to the end of the array:
|
||||
int newNormalizedZeroIndex = oldNormalizedZeroIndex + countsDelta;
|
||||
int lengthToCopy = (countsArrayLength - countsDelta) - oldNormalizedZeroIndex;
|
||||
System.arraycopy(counts, oldNormalizedZeroIndex, counts, newNormalizedZeroIndex, lengthToCopy);
|
||||
Arrays.fill(counts, oldNormalizedZeroIndex, newNormalizedZeroIndex, 0);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Construct an auto-resizing histogram with a lowest discernible value of 1 and an auto-adjusting
|
||||
* highestTrackableValue. Can auto-resize up to track values up to (Long.MAX_VALUE / 2).
|
||||
*
|
||||
* @param numberOfSignificantValueDigits Specifies the precision to use. This is the number of significant
|
||||
* decimal digits to which the histogram will maintain value resolution
|
||||
* and separation. Must be a non-negative integer between 0 and 5.
|
||||
*/
|
||||
public ArrayHistogram(final int numberOfSignificantValueDigits) {
|
||||
this(1, 2, numberOfSignificantValueDigits);
|
||||
setAutoResize(true);
|
||||
}
|
||||
|
||||
/**
|
||||
* Construct a Histogram given the Highest value to be tracked and a number of significant decimal digits. The
|
||||
* histogram will be constructed to implicitly track (distinguish from 0) values as low as 1.
|
||||
*
|
||||
* @param highestTrackableValue The highest value to be tracked by the histogram. Must be a positive
|
||||
* integer that is {@literal >=} 2.
|
||||
* @param numberOfSignificantValueDigits Specifies the precision to use. This is the number of significant
|
||||
* decimal digits to which the histogram will maintain value resolution
|
||||
* and separation. Must be a non-negative integer between 0 and 5.
|
||||
*/
|
||||
public ArrayHistogram(final long highestTrackableValue, final int numberOfSignificantValueDigits) {
|
||||
this(1, highestTrackableValue, numberOfSignificantValueDigits);
|
||||
}
|
||||
|
||||
/**
|
||||
* Construct a Histogram given the Lowest and Highest values to be tracked and a number of significant
|
||||
* decimal digits. Providing a lowestDiscernibleValue is useful is situations where the units used
|
||||
* for the histogram's values are much smaller that the minimal accuracy required. E.g. when tracking
|
||||
* time values stated in nanosecond units, where the minimal accuracy required is a microsecond, the
|
||||
* proper value for lowestDiscernibleValue would be 1000.
|
||||
*
|
||||
* @param lowestDiscernibleValue The lowest value that can be discerned (distinguished from 0) by the
|
||||
* histogram. Must be a positive integer that is {@literal >=} 1. May be
|
||||
* internally rounded down to nearest power of 2.
|
||||
* @param highestTrackableValue The highest value to be tracked by the histogram. Must be a positive
|
||||
* integer that is {@literal >=} (2 * lowestDiscernibleValue).
|
||||
* @param numberOfSignificantValueDigits Specifies the precision to use. This is the number of significant
|
||||
* decimal digits to which the histogram will maintain value resolution
|
||||
* and separation. Must be a non-negative integer between 0 and 5.
|
||||
*/
|
||||
public ArrayHistogram(final long lowestDiscernibleValue, final long highestTrackableValue,
|
||||
final int numberOfSignificantValueDigits) {
|
||||
this(lowestDiscernibleValue, highestTrackableValue, numberOfSignificantValueDigits, true);
|
||||
}
|
||||
|
||||
/**
|
||||
* Construct a histogram with the same range settings as a given source histogram,
|
||||
* duplicating the source's start/end timestamps (but NOT its contents)
|
||||
* @param source The source histogram to duplicate
|
||||
*/
|
||||
public ArrayHistogram(final AbstractHistogram source) {
|
||||
this(source, true);
|
||||
}
|
||||
|
||||
ArrayHistogram(final AbstractHistogram source, boolean allocateCountsArray) {
|
||||
super(source);
|
||||
if (allocateCountsArray) {
|
||||
counts = new long[countsArrayLength];
|
||||
}
|
||||
wordSizeInBytes = 8;
|
||||
}
|
||||
|
||||
ArrayHistogram(final long lowestDiscernibleValue, final long highestTrackableValue,
|
||||
final int numberOfSignificantValueDigits, boolean allocateCountsArray) {
|
||||
super(lowestDiscernibleValue, highestTrackableValue, numberOfSignificantValueDigits);
|
||||
if (allocateCountsArray) {
|
||||
counts = new long[countsArrayLength];
|
||||
}
|
||||
// 写死 = 8
|
||||
wordSizeInBytes = 8;
|
||||
}
|
||||
|
||||
/**
|
||||
* Construct a new histogram by decoding it from a ByteBuffer.
|
||||
* @param buffer The buffer to decode from
|
||||
* @param minBarForHighestTrackableValue Force highestTrackableValue to be set at least this high
|
||||
* @return The newly constructed histogram
|
||||
*/
|
||||
public static ArrayHistogram decodeFromByteBuffer(final ByteBuffer buffer,
|
||||
final long minBarForHighestTrackableValue) {
|
||||
return decodeFromByteBuffer(buffer, ArrayHistogram.class, minBarForHighestTrackableValue);
|
||||
}
|
||||
|
||||
/**
|
||||
* Construct a new histogram by decoding it from a compressed form in a ByteBuffer.
|
||||
* @param buffer The buffer to decode from
|
||||
* @param minBarForHighestTrackableValue Force highestTrackableValue to be set at least this high
|
||||
* @return The newly constructed histogram
|
||||
* @throws DataFormatException on error parsing/decompressing the buffer
|
||||
*/
|
||||
public static ArrayHistogram decodeFromCompressedByteBuffer(final ByteBuffer buffer,
|
||||
final long minBarForHighestTrackableValue)
|
||||
throws DataFormatException {
|
||||
return decodeFromCompressedByteBuffer(buffer, ArrayHistogram.class, minBarForHighestTrackableValue);
|
||||
}
|
||||
|
||||
private void readObject(final ObjectInputStream o)
|
||||
throws IOException, ClassNotFoundException {
|
||||
o.defaultReadObject();
|
||||
}
|
||||
|
||||
/**
|
||||
* Construct a new Histogram by decoding it from a String containing a base64 encoded
|
||||
* compressed histogram representation.
|
||||
*
|
||||
* @param base64CompressedHistogramString A string containing a base64 encoding of a compressed histogram
|
||||
* @return A Histogream decoded from the string
|
||||
* @throws DataFormatException on error parsing/decompressing the input
|
||||
*/
|
||||
public static ArrayHistogram fromString(final String base64CompressedHistogramString)
|
||||
throws DataFormatException {
|
||||
// 这还有个base64字符串的解析
|
||||
return decodeFromCompressedByteBuffer(
|
||||
ByteBuffer.wrap(Base64Helper.parseBase64Binary(base64CompressedHistogramString)),
|
||||
0);
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<Percentile> percentileList(int percentileTicksPerHalfDistance) {
|
||||
List<Percentile> percentiles = new ArrayList<>();
|
||||
for (HistogramIterationValue percentile : this.percentiles(percentileTicksPerHalfDistance)) {
|
||||
if(percentile.getCountAddedInThisIterationStep() > 0){
|
||||
percentiles.add(new Percentile(percentile.getValueIteratedTo(), percentile.getCountAddedInThisIterationStep(), percentile.getPercentile()));
|
||||
}
|
||||
}
|
||||
return percentiles;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Histogramer resetHistogram() {
|
||||
if(isAutoResize()){
|
||||
return new ArrayHistogram(this.numberOfSignificantValueDigits);
|
||||
}else{
|
||||
this.reset();
|
||||
return this;
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public Histogramer merge(Histogramer histogram) {
|
||||
if(histogram instanceof AbstractHistogram){
|
||||
this.add((AbstractHistogram)histogram);
|
||||
return this;
|
||||
}else if(histogram instanceof DirectMapHistogram){
|
||||
try {
|
||||
((DirectMapHistogram)histogram).mergeInto(this);
|
||||
return this;
|
||||
} catch (Exception e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
}else{
|
||||
throw new UnsupportedOperationException("unsupported method");
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public byte[] toBytes() {
|
||||
ByteBuffer byteBuffer = ByteBuffer.allocate(this.getNeededByteBufferCapacity());
|
||||
this.encodeIntoByteBuffer(byteBuffer);
|
||||
return byteBuffer2Bytes(byteBuffer);
|
||||
}
|
||||
|
||||
public static ArrayHistogram fromBytes(byte[] bytes) {
|
||||
ByteBuffer byteBuffer = ByteBuffer.wrap(bytes);
|
||||
return fromByteBuffer(byteBuffer);
|
||||
}
|
||||
|
||||
public static ArrayHistogram fromByteBuffer(ByteBuffer byteBuffer) {
|
||||
int initPosition = byteBuffer.position();
|
||||
int cookie = byteBuffer.getInt(initPosition);
|
||||
if(DirectMapHistogram.getCookieBase(cookie) == DirectMapHistogram.V2CompressedEncodingCookieBase){
|
||||
try {
|
||||
return ArrayHistogram.decodeFromCompressedByteBuffer(byteBuffer, 2);
|
||||
} catch (DataFormatException e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
}else if(DirectMapHistogram.getCookieBase(cookie) == DirectMapHistogram.V2EncodingCookieBase){
|
||||
return ArrayHistogram.decodeFromByteBuffer(byteBuffer, 2);
|
||||
}
|
||||
throw new UnsupportedOperationException("unsupported method");
|
||||
}
|
||||
}
|
||||
package org.HdrHistogram; /**
|
||||
* Written by Gil Tene of Azul Systems, and released to the public domain,
|
||||
* as explained at http://creativecommons.org/publicdomain/zero/1.0/
|
||||
*
|
||||
* @author Gil Tene
|
||||
*/
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.ObjectInputStream;
|
||||
import java.nio.ByteBuffer;
|
||||
import java.util.*;
|
||||
import java.util.zip.DataFormatException;
|
||||
|
||||
/**
|
||||
* <h3>A High Dynamic Range (HDR) Histogram</h3>
|
||||
* <p>
|
||||
* {@link ArrayHistogram} supports the recording and analyzing sampled data value counts across a configurable integer value
|
||||
* range with configurable value precision within the range. Value precision is expressed as the number of significant
|
||||
* digits in the value recording, and provides control over value quantization behavior across the value range and the
|
||||
* subsequent value resolution at any given level.
|
||||
* <p>
|
||||
* For example, a Histogram could be configured to track the counts of observed integer values between 0 and
|
||||
* 3,600,000,000 while maintaining a value precision of 3 significant digits across that range. Value quantization
|
||||
* within the range will thus be no larger than 1/1,000th (or 0.1%) of any value. This example Histogram could
|
||||
* be used to track and analyze the counts of observed response times ranging between 1 microsecond and 1 hour
|
||||
* in magnitude, while maintaining a value resolution of 1 microsecond up to 1 millisecond, a resolution of
|
||||
* 1 millisecond (or better) up to one second, and a resolution of 1 second (or better) up to 1,000 seconds. At its
|
||||
* maximum tracked value (1 hour), it would still maintain a resolution of 3.6 seconds (or better).
|
||||
* <p>
|
||||
* Histogram tracks value counts in <b><code>long</code></b> fields. Smaller field types are available in the
|
||||
* {@link IntCountsHistogram} and {@link ShortCountsHistogram} implementations of
|
||||
* {@link AbstractHistogram}.
|
||||
* <p>
|
||||
* Auto-resizing: When constructed with no specified value range range (or when auto-resize is turned on with {@link
|
||||
* ArrayHistogram#setAutoResize}) a {@link ArrayHistogram} will auto-resize its dynamic range to include recorded values as
|
||||
* they are encountered. Note that recording calls that cause auto-resizing may take longer to execute, as resizing
|
||||
* incurs allocation and copying of internal data structures.
|
||||
* <p>
|
||||
* See package description for {@link org.HdrHistogram} for details.
|
||||
*/
|
||||
|
||||
public class ArrayHistogram extends AbstractHistogram implements Histogramer{
|
||||
long totalCount;
|
||||
long[] counts;
|
||||
int normalizingIndexOffset;
|
||||
|
||||
@Override
|
||||
long getCountAtIndex(final int index) {
|
||||
return counts[normalizeIndex(index, normalizingIndexOffset, countsArrayLength)];
|
||||
}
|
||||
|
||||
@Override
|
||||
long getCountAtNormalizedIndex(final int index) {
|
||||
return counts[index];
|
||||
}
|
||||
|
||||
@Override
|
||||
void incrementCountAtIndex(final int index) {
|
||||
counts[normalizeIndex(index, normalizingIndexOffset, countsArrayLength)]++;
|
||||
}
|
||||
|
||||
@Override
|
||||
void addToCountAtIndex(final int index, final long value) {
|
||||
// 正常情况下normalizingIndexOffset = 0, index不用偏移
|
||||
counts[normalizeIndex(index, normalizingIndexOffset, countsArrayLength)] += value;
|
||||
}
|
||||
|
||||
@Override
|
||||
void setCountAtIndex(int index, long value) {
|
||||
counts[normalizeIndex(index, normalizingIndexOffset, countsArrayLength)] = value;
|
||||
}
|
||||
|
||||
@Override
|
||||
void setCountAtNormalizedIndex(int index, long value) {
|
||||
counts[index] = value;
|
||||
}
|
||||
|
||||
@Override
|
||||
int getNormalizingIndexOffset() {
|
||||
return normalizingIndexOffset;
|
||||
}
|
||||
|
||||
@Override
|
||||
void setNormalizingIndexOffset(int normalizingIndexOffset) {
|
||||
this.normalizingIndexOffset = normalizingIndexOffset;
|
||||
}
|
||||
|
||||
@Override
|
||||
void setIntegerToDoubleValueConversionRatio(double integerToDoubleValueConversionRatio) {
|
||||
nonConcurrentSetIntegerToDoubleValueConversionRatio(integerToDoubleValueConversionRatio);
|
||||
}
|
||||
|
||||
@Override
|
||||
void shiftNormalizingIndexByOffset(int offsetToAdd,
|
||||
boolean lowestHalfBucketPopulated,
|
||||
double newIntegerToDoubleValueConversionRatio) {
|
||||
nonConcurrentNormalizingIndexShift(offsetToAdd, lowestHalfBucketPopulated);
|
||||
}
|
||||
|
||||
@Override
|
||||
void clearCounts() {
|
||||
Arrays.fill(counts, 0);
|
||||
totalCount = 0;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Histogramer makeCopy() {
|
||||
return miniCopy();
|
||||
}
|
||||
|
||||
@Override
|
||||
public ArrayHistogram copy() {
|
||||
ArrayHistogram copy = new ArrayHistogram(this);
|
||||
copy.add(this);
|
||||
return copy;
|
||||
}
|
||||
|
||||
public ArrayHistogram miniCopy() {
|
||||
ArrayHistogram copy = new ArrayHistogram(lowestDiscernibleValue, maxValue < highestTrackableValue ? Math.max(maxValue, lowestDiscernibleValue * 2) : highestTrackableValue, numberOfSignificantValueDigits);
|
||||
copy.add(this);
|
||||
return copy;
|
||||
}
|
||||
|
||||
@Override
|
||||
public ArrayHistogram copyCorrectedForCoordinatedOmission(final long expectedIntervalBetweenValueSamples) {
|
||||
ArrayHistogram copy = new ArrayHistogram(this);
|
||||
copy.addWhileCorrectingForCoordinatedOmission(this, expectedIntervalBetweenValueSamples);
|
||||
return copy;
|
||||
}
|
||||
|
||||
@Override
|
||||
public long getTotalCount() {
|
||||
return totalCount;
|
||||
}
|
||||
|
||||
@Override
|
||||
void setTotalCount(final long totalCount) {
|
||||
this.totalCount = totalCount;
|
||||
}
|
||||
|
||||
@Override
|
||||
void incrementTotalCount() {
|
||||
totalCount++;
|
||||
}
|
||||
|
||||
@Override
|
||||
void addToTotalCount(final long value) {
|
||||
totalCount += value;
|
||||
}
|
||||
|
||||
@Override
|
||||
int _getEstimatedFootprintInBytes() {
|
||||
return (512 + (8 * counts.length));
|
||||
}
|
||||
|
||||
@Override
|
||||
void resize(long newHighestTrackableValue) {
|
||||
int oldNormalizedZeroIndex = normalizeIndex(0, normalizingIndexOffset, countsArrayLength);
|
||||
|
||||
establishSize(newHighestTrackableValue);
|
||||
|
||||
int countsDelta = countsArrayLength - counts.length;
|
||||
|
||||
counts = Arrays.copyOf(counts, countsArrayLength);
|
||||
|
||||
if (oldNormalizedZeroIndex != 0) {
|
||||
// We need to shift the stuff from the zero index and up to the end of the array:
|
||||
int newNormalizedZeroIndex = oldNormalizedZeroIndex + countsDelta;
|
||||
int lengthToCopy = (countsArrayLength - countsDelta) - oldNormalizedZeroIndex;
|
||||
System.arraycopy(counts, oldNormalizedZeroIndex, counts, newNormalizedZeroIndex, lengthToCopy);
|
||||
Arrays.fill(counts, oldNormalizedZeroIndex, newNormalizedZeroIndex, 0);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Construct an auto-resizing histogram with a lowest discernible value of 1 and an auto-adjusting
|
||||
* highestTrackableValue. Can auto-resize up to track values up to (Long.MAX_VALUE / 2).
|
||||
*
|
||||
* @param numberOfSignificantValueDigits Specifies the precision to use. This is the number of significant
|
||||
* decimal digits to which the histogram will maintain value resolution
|
||||
* and separation. Must be a non-negative integer between 0 and 5.
|
||||
*/
|
||||
public ArrayHistogram(final int numberOfSignificantValueDigits) {
|
||||
this(1, 2, numberOfSignificantValueDigits);
|
||||
setAutoResize(true);
|
||||
}
|
||||
|
||||
/**
|
||||
* Construct a Histogram given the Highest value to be tracked and a number of significant decimal digits. The
|
||||
* histogram will be constructed to implicitly track (distinguish from 0) values as low as 1.
|
||||
*
|
||||
* @param highestTrackableValue The highest value to be tracked by the histogram. Must be a positive
|
||||
* integer that is {@literal >=} 2.
|
||||
* @param numberOfSignificantValueDigits Specifies the precision to use. This is the number of significant
|
||||
* decimal digits to which the histogram will maintain value resolution
|
||||
* and separation. Must be a non-negative integer between 0 and 5.
|
||||
*/
|
||||
public ArrayHistogram(final long highestTrackableValue, final int numberOfSignificantValueDigits) {
|
||||
this(1, highestTrackableValue, numberOfSignificantValueDigits);
|
||||
}
|
||||
|
||||
/**
|
||||
* Construct a Histogram given the Lowest and Highest values to be tracked and a number of significant
|
||||
* decimal digits. Providing a lowestDiscernibleValue is useful is situations where the units used
|
||||
* for the histogram's values are much smaller that the minimal accuracy required. E.g. when tracking
|
||||
* time values stated in nanosecond units, where the minimal accuracy required is a microsecond, the
|
||||
* proper value for lowestDiscernibleValue would be 1000.
|
||||
*
|
||||
* @param lowestDiscernibleValue The lowest value that can be discerned (distinguished from 0) by the
|
||||
* histogram. Must be a positive integer that is {@literal >=} 1. May be
|
||||
* internally rounded down to nearest power of 2.
|
||||
* @param highestTrackableValue The highest value to be tracked by the histogram. Must be a positive
|
||||
* integer that is {@literal >=} (2 * lowestDiscernibleValue).
|
||||
* @param numberOfSignificantValueDigits Specifies the precision to use. This is the number of significant
|
||||
* decimal digits to which the histogram will maintain value resolution
|
||||
* and separation. Must be a non-negative integer between 0 and 5.
|
||||
*/
|
||||
public ArrayHistogram(final long lowestDiscernibleValue, final long highestTrackableValue,
|
||||
final int numberOfSignificantValueDigits) {
|
||||
this(lowestDiscernibleValue, highestTrackableValue, numberOfSignificantValueDigits, true);
|
||||
}
|
||||
|
||||
/**
|
||||
* Construct a histogram with the same range settings as a given source histogram,
|
||||
* duplicating the source's start/end timestamps (but NOT its contents)
|
||||
* @param source The source histogram to duplicate
|
||||
*/
|
||||
public ArrayHistogram(final AbstractHistogram source) {
|
||||
this(source, true);
|
||||
}
|
||||
|
||||
ArrayHistogram(final AbstractHistogram source, boolean allocateCountsArray) {
|
||||
super(source);
|
||||
if (allocateCountsArray) {
|
||||
counts = new long[countsArrayLength];
|
||||
}
|
||||
wordSizeInBytes = 8;
|
||||
}
|
||||
|
||||
ArrayHistogram(final long lowestDiscernibleValue, final long highestTrackableValue,
|
||||
final int numberOfSignificantValueDigits, boolean allocateCountsArray) {
|
||||
super(lowestDiscernibleValue, highestTrackableValue, numberOfSignificantValueDigits);
|
||||
if (allocateCountsArray) {
|
||||
counts = new long[countsArrayLength];
|
||||
}
|
||||
// 写死 = 8
|
||||
wordSizeInBytes = 8;
|
||||
}
|
||||
|
||||
/**
|
||||
* Construct a new histogram by decoding it from a ByteBuffer.
|
||||
* @param buffer The buffer to decode from
|
||||
* @param minBarForHighestTrackableValue Force highestTrackableValue to be set at least this high
|
||||
* @return The newly constructed histogram
|
||||
*/
|
||||
public static ArrayHistogram decodeFromByteBuffer(final ByteBuffer buffer,
|
||||
final long minBarForHighestTrackableValue) {
|
||||
return decodeFromByteBuffer(buffer, ArrayHistogram.class, minBarForHighestTrackableValue);
|
||||
}
|
||||
|
||||
/**
|
||||
* Construct a new histogram by decoding it from a compressed form in a ByteBuffer.
|
||||
* @param buffer The buffer to decode from
|
||||
* @param minBarForHighestTrackableValue Force highestTrackableValue to be set at least this high
|
||||
* @return The newly constructed histogram
|
||||
* @throws DataFormatException on error parsing/decompressing the buffer
|
||||
*/
|
||||
public static ArrayHistogram decodeFromCompressedByteBuffer(final ByteBuffer buffer,
|
||||
final long minBarForHighestTrackableValue)
|
||||
throws DataFormatException {
|
||||
return decodeFromCompressedByteBuffer(buffer, ArrayHistogram.class, minBarForHighestTrackableValue);
|
||||
}
|
||||
|
||||
private void readObject(final ObjectInputStream o)
|
||||
throws IOException, ClassNotFoundException {
|
||||
o.defaultReadObject();
|
||||
}
|
||||
|
||||
/**
|
||||
* Construct a new Histogram by decoding it from a String containing a base64 encoded
|
||||
* compressed histogram representation.
|
||||
*
|
||||
* @param base64CompressedHistogramString A string containing a base64 encoding of a compressed histogram
|
||||
* @return A Histogream decoded from the string
|
||||
* @throws DataFormatException on error parsing/decompressing the input
|
||||
*/
|
||||
public static ArrayHistogram fromString(final String base64CompressedHistogramString)
|
||||
throws DataFormatException {
|
||||
// 这还有个base64字符串的解析
|
||||
return decodeFromCompressedByteBuffer(
|
||||
ByteBuffer.wrap(Base64Helper.parseBase64Binary(base64CompressedHistogramString)),
|
||||
0);
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<Percentile> percentileList(int percentileTicksPerHalfDistance) {
|
||||
List<Percentile> percentiles = new ArrayList<>();
|
||||
for (HistogramIterationValue percentile : this.percentiles(percentileTicksPerHalfDistance)) {
|
||||
if(percentile.getCountAddedInThisIterationStep() > 0){
|
||||
percentiles.add(new Percentile(percentile.getValueIteratedTo(), percentile.getCountAddedInThisIterationStep(), percentile.getPercentile()));
|
||||
}
|
||||
}
|
||||
return percentiles;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Map<String, Object> describe() {
|
||||
long min = getMinValue();
|
||||
long max = getMaxValue(); // max = this.maxValue;
|
||||
long count = getTotalCount();
|
||||
double mean = getMean();
|
||||
long sum = (long) (mean * count);
|
||||
mean = Math.round(mean * 100.0) / 100.0;
|
||||
long p25 = getValueAtPercentile(25);
|
||||
long p50 = getValueAtPercentile(50);
|
||||
long p75 = getValueAtPercentile(75);
|
||||
long p90 = getValueAtPercentile(90);
|
||||
long p95 = getValueAtPercentile(95);
|
||||
long p99 = getValueAtPercentile(99);
|
||||
Map<String, Object> rst = new LinkedHashMap<>();
|
||||
rst.put("count", count);
|
||||
rst.put("mean", mean);
|
||||
rst.put("sum", sum);
|
||||
rst.put("min", min);
|
||||
rst.put("p25", p25);
|
||||
rst.put("p50", p50);
|
||||
rst.put("p75", p75);
|
||||
rst.put("p90", p90);
|
||||
rst.put("p95", p95);
|
||||
rst.put("p99", p99);
|
||||
rst.put("max", max);
|
||||
return rst;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Histogramer resetHistogram() {
|
||||
if(isAutoResize()){
|
||||
return new ArrayHistogram(this.numberOfSignificantValueDigits);
|
||||
}else{
|
||||
this.reset();
|
||||
return this;
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public Histogramer merge(Histogramer histogram) {
|
||||
if(histogram instanceof AbstractHistogram){
|
||||
this.add((AbstractHistogram)histogram);
|
||||
return this;
|
||||
}else if(histogram instanceof DirectMapHistogram){
|
||||
try {
|
||||
((DirectMapHistogram)histogram).mergeInto(this);
|
||||
return this;
|
||||
} catch (Exception e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
}else{
|
||||
throw new UnsupportedOperationException("unsupported method");
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public byte[] toBytes() {
|
||||
ByteBuffer byteBuffer = ByteBuffer.allocate(this.getNeededByteBufferCapacity());
|
||||
this.encodeIntoByteBuffer(byteBuffer);
|
||||
return byteBuffer2Bytes(byteBuffer);
|
||||
}
|
||||
|
||||
public static ArrayHistogram fromBytes(byte[] bytes) {
|
||||
ByteBuffer byteBuffer = ByteBuffer.wrap(bytes);
|
||||
return fromByteBuffer(byteBuffer);
|
||||
}
|
||||
|
||||
public static ArrayHistogram fromByteBuffer(ByteBuffer byteBuffer) {
|
||||
int initPosition = byteBuffer.position();
|
||||
int cookie = byteBuffer.getInt(initPosition);
|
||||
if(DirectMapHistogram.getCookieBase(cookie) == DirectMapHistogram.V2CompressedEncodingCookieBase){
|
||||
try {
|
||||
return ArrayHistogram.decodeFromCompressedByteBuffer(byteBuffer, 2);
|
||||
} catch (DataFormatException e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
}else if(DirectMapHistogram.getCookieBase(cookie) == DirectMapHistogram.V2EncodingCookieBase){
|
||||
return ArrayHistogram.decodeFromByteBuffer(byteBuffer, 2);
|
||||
}
|
||||
throw new UnsupportedOperationException("unsupported method");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,203 +1,234 @@
|
||||
package org.HdrHistogram;
|
||||
|
||||
import java.nio.ByteBuffer;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
public class DirectArrayHistogram extends AbstractHistogram implements Histogramer{
|
||||
long totalCount;
|
||||
int normalizingIndexOffset;
|
||||
private ByteBuffer byteBuffer;
|
||||
private int initPosition;
|
||||
|
||||
public DirectArrayHistogram(final long lowestDiscernibleValue, final long highestTrackableValue,
|
||||
final int numberOfSignificantValueDigits, ByteBuffer byteBuffer) {
|
||||
super(lowestDiscernibleValue, highestTrackableValue, numberOfSignificantValueDigits);
|
||||
this.byteBuffer = byteBuffer;
|
||||
this.initPosition = byteBuffer.position();
|
||||
wordSizeInBytes = 8;
|
||||
}
|
||||
|
||||
// druid内部使用
|
||||
public void resetByteBuffer(ByteBuffer byteBuffer){
|
||||
this.byteBuffer = byteBuffer;
|
||||
this.initPosition = byteBuffer.position();
|
||||
}
|
||||
|
||||
@Override
|
||||
long getCountAtIndex(int index) {
|
||||
int i = normalizeIndex(index, normalizingIndexOffset, countsArrayLength);
|
||||
return byteBuffer.getLong(initPosition + i * 8);
|
||||
}
|
||||
|
||||
@Override
|
||||
long getCountAtNormalizedIndex(int index) {
|
||||
return byteBuffer.getLong(initPosition + index * 8);
|
||||
}
|
||||
|
||||
@Override
|
||||
void incrementCountAtIndex(int index) {
|
||||
int i = normalizeIndex(index, normalizingIndexOffset, countsArrayLength);
|
||||
int pos = initPosition + i * 8;
|
||||
long val = byteBuffer.getLong(pos);
|
||||
byteBuffer.putLong(pos, val + 1);
|
||||
}
|
||||
|
||||
@Override
|
||||
void addToCountAtIndex(int index, long value) {
|
||||
int i = normalizeIndex(index, normalizingIndexOffset, countsArrayLength);
|
||||
int pos = initPosition + i * 8;
|
||||
long val = byteBuffer.getLong(pos);
|
||||
byteBuffer.putLong(pos, val + value);
|
||||
}
|
||||
|
||||
@Override
|
||||
void setCountAtIndex(int index, long value) {
|
||||
int i = normalizeIndex(index, normalizingIndexOffset, countsArrayLength);
|
||||
int pos = initPosition + i * 8;
|
||||
byteBuffer.putLong(pos, value);
|
||||
}
|
||||
|
||||
@Override
|
||||
void setCountAtNormalizedIndex(int index, long value) {
|
||||
int pos = initPosition + index * 8;
|
||||
byteBuffer.putLong(pos, value);
|
||||
}
|
||||
|
||||
@Override
|
||||
int getNormalizingIndexOffset() {
|
||||
return normalizingIndexOffset;
|
||||
}
|
||||
|
||||
@Override
|
||||
void setNormalizingIndexOffset(int normalizingIndexOffset) {
|
||||
if(normalizingIndexOffset == 0){
|
||||
this.normalizingIndexOffset = normalizingIndexOffset;
|
||||
}else{
|
||||
throw new RuntimeException("cant not setNormalizingIndexOffset");
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
void setIntegerToDoubleValueConversionRatio(double integerToDoubleValueConversionRatio) {
|
||||
nonConcurrentSetIntegerToDoubleValueConversionRatio(integerToDoubleValueConversionRatio);
|
||||
}
|
||||
|
||||
@Override
|
||||
void shiftNormalizingIndexByOffset(int offsetToAdd, boolean lowestHalfBucketPopulated, double newIntegerToDoubleValueConversionRatio) {
|
||||
nonConcurrentNormalizingIndexShift(offsetToAdd, lowestHalfBucketPopulated);
|
||||
}
|
||||
|
||||
@Override
|
||||
void clearCounts() {
|
||||
for (int i = 0; i < countsArrayLength; i++) {
|
||||
byteBuffer.putLong(initPosition + i * 8, 0L);
|
||||
}
|
||||
totalCount = 0;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Histogramer makeCopy() {
|
||||
return miniCopy();
|
||||
}
|
||||
|
||||
@Override
|
||||
public ArrayHistogram copy() {
|
||||
ArrayHistogram copy = new ArrayHistogram(this);
|
||||
copy.add(this);
|
||||
return copy;
|
||||
}
|
||||
|
||||
public ArrayHistogram miniCopy() {
|
||||
ArrayHistogram copy = new ArrayHistogram(lowestDiscernibleValue, maxValue < highestTrackableValue ? Math.max(maxValue, lowestDiscernibleValue * 2) : highestTrackableValue, numberOfSignificantValueDigits);
|
||||
copy.add(this);
|
||||
return copy;
|
||||
}
|
||||
|
||||
@Override
|
||||
public AbstractHistogram copyCorrectedForCoordinatedOmission(long expectedIntervalBetweenValueSamples) {
|
||||
Histogram copy = new Histogram(this);
|
||||
copy.addWhileCorrectingForCoordinatedOmission(this, expectedIntervalBetweenValueSamples);
|
||||
return copy;
|
||||
}
|
||||
|
||||
@Override
|
||||
public long getTotalCount() {
|
||||
return totalCount;
|
||||
}
|
||||
|
||||
@Override
|
||||
void setTotalCount(final long totalCount) {
|
||||
this.totalCount = totalCount;
|
||||
}
|
||||
|
||||
@Override
|
||||
void incrementTotalCount() {
|
||||
totalCount++;
|
||||
}
|
||||
|
||||
@Override
|
||||
void addToTotalCount(long value) {
|
||||
totalCount += value;
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
int _getEstimatedFootprintInBytes() {
|
||||
return (512 + (8 * countsArrayLength));
|
||||
}
|
||||
|
||||
@Override
|
||||
void resize(long newHighestTrackableValue) {
|
||||
throw new RuntimeException("cant not resize");
|
||||
}
|
||||
|
||||
public static int getCountsArrayLength(long lowestDiscernibleValue, long highestTrackableValue, int numberOfSignificantValueDigits){
|
||||
Histogram his = new Histogram(lowestDiscernibleValue, highestTrackableValue, numberOfSignificantValueDigits, false);
|
||||
return his.countsArrayLength;
|
||||
}
|
||||
|
||||
public static final int getUpdatableSerializationBytes(long lowestDiscernibleValue, long highestTrackableValue, int numberOfSignificantValueDigits){
|
||||
return getCountsArrayLength(lowestDiscernibleValue, highestTrackableValue, numberOfSignificantValueDigits) * 8;
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<Percentile> percentileList(int percentileTicksPerHalfDistance) {
|
||||
List<Percentile> percentiles = new ArrayList<>();
|
||||
for (HistogramIterationValue percentile : this.percentiles(percentileTicksPerHalfDistance)) {
|
||||
if(percentile.getCountAddedInThisIterationStep() > 0){
|
||||
percentiles.add(new Percentile(percentile.getValueIteratedTo(), percentile.getCountAddedInThisIterationStep(), percentile.getPercentile()));
|
||||
}
|
||||
}
|
||||
return percentiles;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Histogramer resetHistogram() {
|
||||
throw new UnsupportedOperationException("unsupported method");
|
||||
}
|
||||
|
||||
@Override
|
||||
public Histogramer merge(Histogramer histogram) {
|
||||
if(histogram instanceof AbstractHistogram){
|
||||
this.add((AbstractHistogram)histogram);
|
||||
return this;
|
||||
}else if(histogram instanceof DirectMapHistogram){
|
||||
try {
|
||||
((DirectMapHistogram)histogram).mergeInto(this);
|
||||
return this;
|
||||
} catch (Exception e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
}else{
|
||||
throw new UnsupportedOperationException("unsupported method");
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public byte[] toBytes() {
|
||||
ByteBuffer byteBuffer = ByteBuffer.allocate(this.getNeededByteBufferCapacity());
|
||||
this.encodeIntoByteBuffer(byteBuffer);
|
||||
return byteBuffer2Bytes(byteBuffer);
|
||||
}
|
||||
}
|
||||
package org.HdrHistogram;
|
||||
|
||||
import java.nio.ByteBuffer;
|
||||
import java.util.ArrayList;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
public class DirectArrayHistogram extends AbstractHistogram implements Histogramer{
|
||||
long totalCount;
|
||||
int normalizingIndexOffset;
|
||||
private ByteBuffer byteBuffer;
|
||||
private int initPosition;
|
||||
|
||||
public DirectArrayHistogram(final long lowestDiscernibleValue, final long highestTrackableValue,
|
||||
final int numberOfSignificantValueDigits, ByteBuffer byteBuffer) {
|
||||
super(lowestDiscernibleValue, highestTrackableValue, numberOfSignificantValueDigits);
|
||||
this.byteBuffer = byteBuffer;
|
||||
this.initPosition = byteBuffer.position();
|
||||
wordSizeInBytes = 8;
|
||||
}
|
||||
|
||||
// druid内部使用
|
||||
public void resetByteBuffer(ByteBuffer byteBuffer){
|
||||
this.byteBuffer = byteBuffer;
|
||||
this.initPosition = byteBuffer.position();
|
||||
}
|
||||
|
||||
@Override
|
||||
long getCountAtIndex(int index) {
|
||||
int i = normalizeIndex(index, normalizingIndexOffset, countsArrayLength);
|
||||
return byteBuffer.getLong(initPosition + i * 8);
|
||||
}
|
||||
|
||||
@Override
|
||||
long getCountAtNormalizedIndex(int index) {
|
||||
return byteBuffer.getLong(initPosition + index * 8);
|
||||
}
|
||||
|
||||
@Override
|
||||
void incrementCountAtIndex(int index) {
|
||||
int i = normalizeIndex(index, normalizingIndexOffset, countsArrayLength);
|
||||
int pos = initPosition + i * 8;
|
||||
long val = byteBuffer.getLong(pos);
|
||||
byteBuffer.putLong(pos, val + 1);
|
||||
}
|
||||
|
||||
@Override
|
||||
void addToCountAtIndex(int index, long value) {
|
||||
int i = normalizeIndex(index, normalizingIndexOffset, countsArrayLength);
|
||||
int pos = initPosition + i * 8;
|
||||
long val = byteBuffer.getLong(pos);
|
||||
byteBuffer.putLong(pos, val + value);
|
||||
}
|
||||
|
||||
@Override
|
||||
void setCountAtIndex(int index, long value) {
|
||||
int i = normalizeIndex(index, normalizingIndexOffset, countsArrayLength);
|
||||
int pos = initPosition + i * 8;
|
||||
byteBuffer.putLong(pos, value);
|
||||
}
|
||||
|
||||
@Override
|
||||
void setCountAtNormalizedIndex(int index, long value) {
|
||||
int pos = initPosition + index * 8;
|
||||
byteBuffer.putLong(pos, value);
|
||||
}
|
||||
|
||||
@Override
|
||||
int getNormalizingIndexOffset() {
|
||||
return normalizingIndexOffset;
|
||||
}
|
||||
|
||||
@Override
|
||||
void setNormalizingIndexOffset(int normalizingIndexOffset) {
|
||||
if(normalizingIndexOffset == 0){
|
||||
this.normalizingIndexOffset = normalizingIndexOffset;
|
||||
}else{
|
||||
throw new RuntimeException("cant not setNormalizingIndexOffset");
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
void setIntegerToDoubleValueConversionRatio(double integerToDoubleValueConversionRatio) {
|
||||
nonConcurrentSetIntegerToDoubleValueConversionRatio(integerToDoubleValueConversionRatio);
|
||||
}
|
||||
|
||||
@Override
|
||||
void shiftNormalizingIndexByOffset(int offsetToAdd, boolean lowestHalfBucketPopulated, double newIntegerToDoubleValueConversionRatio) {
|
||||
nonConcurrentNormalizingIndexShift(offsetToAdd, lowestHalfBucketPopulated);
|
||||
}
|
||||
|
||||
@Override
|
||||
void clearCounts() {
|
||||
for (int i = 0; i < countsArrayLength; i++) {
|
||||
byteBuffer.putLong(initPosition + i * 8, 0L);
|
||||
}
|
||||
totalCount = 0;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Histogramer makeCopy() {
|
||||
return miniCopy();
|
||||
}
|
||||
|
||||
@Override
|
||||
public ArrayHistogram copy() {
|
||||
ArrayHistogram copy = new ArrayHistogram(this);
|
||||
copy.add(this);
|
||||
return copy;
|
||||
}
|
||||
|
||||
public ArrayHistogram miniCopy() {
|
||||
ArrayHistogram copy = new ArrayHistogram(lowestDiscernibleValue, maxValue < highestTrackableValue ? Math.max(maxValue, lowestDiscernibleValue * 2) : highestTrackableValue, numberOfSignificantValueDigits);
|
||||
copy.add(this);
|
||||
return copy;
|
||||
}
|
||||
|
||||
@Override
|
||||
public AbstractHistogram copyCorrectedForCoordinatedOmission(long expectedIntervalBetweenValueSamples) {
|
||||
Histogram copy = new Histogram(this);
|
||||
copy.addWhileCorrectingForCoordinatedOmission(this, expectedIntervalBetweenValueSamples);
|
||||
return copy;
|
||||
}
|
||||
|
||||
@Override
|
||||
public long getTotalCount() {
|
||||
return totalCount;
|
||||
}
|
||||
|
||||
@Override
|
||||
void setTotalCount(final long totalCount) {
|
||||
this.totalCount = totalCount;
|
||||
}
|
||||
|
||||
@Override
|
||||
void incrementTotalCount() {
|
||||
totalCount++;
|
||||
}
|
||||
|
||||
@Override
|
||||
void addToTotalCount(long value) {
|
||||
totalCount += value;
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
int _getEstimatedFootprintInBytes() {
|
||||
return (512 + (8 * countsArrayLength));
|
||||
}
|
||||
|
||||
@Override
|
||||
void resize(long newHighestTrackableValue) {
|
||||
throw new RuntimeException("cant not resize");
|
||||
}
|
||||
|
||||
public static int getCountsArrayLength(long lowestDiscernibleValue, long highestTrackableValue, int numberOfSignificantValueDigits){
|
||||
Histogram his = new Histogram(lowestDiscernibleValue, highestTrackableValue, numberOfSignificantValueDigits, false);
|
||||
return his.countsArrayLength;
|
||||
}
|
||||
|
||||
public static final int getUpdatableSerializationBytes(long lowestDiscernibleValue, long highestTrackableValue, int numberOfSignificantValueDigits){
|
||||
return getCountsArrayLength(lowestDiscernibleValue, highestTrackableValue, numberOfSignificantValueDigits) * 8;
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<Percentile> percentileList(int percentileTicksPerHalfDistance) {
|
||||
List<Percentile> percentiles = new ArrayList<>();
|
||||
for (HistogramIterationValue percentile : this.percentiles(percentileTicksPerHalfDistance)) {
|
||||
if(percentile.getCountAddedInThisIterationStep() > 0){
|
||||
percentiles.add(new Percentile(percentile.getValueIteratedTo(), percentile.getCountAddedInThisIterationStep(), percentile.getPercentile()));
|
||||
}
|
||||
}
|
||||
return percentiles;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Map<String, Object> describe() {
|
||||
long min = getMinValue();
|
||||
long max = getMaxValue(); // max = this.maxValue;
|
||||
long count = getTotalCount();
|
||||
double mean = getMean();
|
||||
long sum = (long) (mean * count);
|
||||
mean = Math.round(mean * 100.0) / 100.0;
|
||||
long p25 = getValueAtPercentile(25);
|
||||
long p50 = getValueAtPercentile(50);
|
||||
long p75 = getValueAtPercentile(75);
|
||||
long p90 = getValueAtPercentile(90);
|
||||
long p95 = getValueAtPercentile(95);
|
||||
long p99 = getValueAtPercentile(99);
|
||||
Map<String, Object> rst = new LinkedHashMap<>();
|
||||
rst.put("count", count);
|
||||
rst.put("mean", mean);
|
||||
rst.put("sum", sum);
|
||||
rst.put("min", min);
|
||||
rst.put("p25", p25);
|
||||
rst.put("p50", p50);
|
||||
rst.put("p75", p75);
|
||||
rst.put("p90", p90);
|
||||
rst.put("p95", p95);
|
||||
rst.put("p99", p99);
|
||||
rst.put("max", max);
|
||||
return rst;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Histogramer resetHistogram() {
|
||||
throw new UnsupportedOperationException("unsupported method");
|
||||
}
|
||||
|
||||
@Override
|
||||
public Histogramer merge(Histogramer histogram) {
|
||||
if(histogram instanceof AbstractHistogram){
|
||||
this.add((AbstractHistogram)histogram);
|
||||
return this;
|
||||
}else if(histogram instanceof DirectMapHistogram){
|
||||
try {
|
||||
((DirectMapHistogram)histogram).mergeInto(this);
|
||||
return this;
|
||||
} catch (Exception e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
}else{
|
||||
throw new UnsupportedOperationException("unsupported method");
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public byte[] toBytes() {
|
||||
ByteBuffer byteBuffer = ByteBuffer.allocate(this.getNeededByteBufferCapacity());
|
||||
this.encodeIntoByteBuffer(byteBuffer);
|
||||
return byteBuffer2Bytes(byteBuffer);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,486 +1,492 @@
|
||||
package org.HdrHistogram;
|
||||
|
||||
import java.nio.ByteBuffer;
|
||||
import java.nio.ByteOrder;
|
||||
import java.util.List;
|
||||
import java.util.zip.DataFormatException;
|
||||
import java.util.zip.Inflater;
|
||||
|
||||
import static java.nio.ByteOrder.BIG_ENDIAN;
|
||||
|
||||
/**
|
||||
* 直接映射字节数组到Histogram，只读的Histogram，用于druid查询，减少gc减少计算，序列化后的是稀疏数组的形式
|
||||
*/
|
||||
public class DirectMapHistogram implements Histogramer{
|
||||
static final int V2maxWordSizeInBytes = 9; // LEB128-64b9B + ZigZag require up to 9 bytes per word
|
||||
static final int V2EncodingCookieBase = 0x1c849303;
|
||||
static final int V2CompressedEncodingCookieBase = 0x1c849304;
|
||||
|
||||
final ByteBuffer byteBuffer;
|
||||
final int initPosition;
|
||||
long totalCount;
|
||||
|
||||
private DirectMapHistogram(ByteBuffer byteBuffer) {
|
||||
int initPosition = byteBuffer.position();
|
||||
this.byteBuffer = byteBuffer;
|
||||
this.initPosition = initPosition;
|
||||
this.totalCount = -1;
|
||||
}
|
||||
|
||||
public static boolean byteBufferCanToDirectMapHistogram(ByteBuffer byteBuffer) {
|
||||
int initPosition = byteBuffer.position();
|
||||
int cookie = byteBuffer.getInt(initPosition);
|
||||
return getCookieBase(cookie) == V2EncodingCookieBase || getCookieBase(cookie) == V2CompressedEncodingCookieBase;
|
||||
}
|
||||
|
||||
public static DirectMapHistogram wrapBytes(byte[] bytes) {
|
||||
ByteBuffer byteBuffer = ByteBuffer.wrap(bytes);
|
||||
return wrapByteBuffer(byteBuffer);
|
||||
}
|
||||
|
||||
public static DirectMapHistogram wrapByteBuffer(ByteBuffer byteBuffer) {
|
||||
if(byteBufferCanToDirectMapHistogram(byteBuffer)){
|
||||
DirectMapHistogram hll = new DirectMapHistogram(byteBuffer);
|
||||
return hll;
|
||||
}
|
||||
throw new RuntimeException("can not wrapByteBuffer");
|
||||
}
|
||||
|
||||
public void mergeInto(AbstractHistogram histogram) throws Exception{
|
||||
int cookie = byteBuffer.getInt(initPosition);
|
||||
if(getCookieBase(cookie) == V2CompressedEncodingCookieBase){
|
||||
final int lengthOfCompressedContents = byteBuffer.getInt(initPosition + 4);
|
||||
final Inflater decompressor = new Inflater();
|
||||
|
||||
if (byteBuffer.hasArray()) {
|
||||
decompressor.setInput(byteBuffer.array(), initPosition + 8, lengthOfCompressedContents);
|
||||
} else {
|
||||
byte[] compressedContents = new byte[lengthOfCompressedContents];
|
||||
byteBuffer.position(initPosition + 8);
|
||||
try {
|
||||
byteBuffer.get(compressedContents);
|
||||
decompressor.setInput(compressedContents);
|
||||
}finally {
|
||||
byteBuffer.position(initPosition);
|
||||
}
|
||||
}
|
||||
final int headerSize = 40;
|
||||
final ByteBuffer headerBuffer = ByteBuffer.allocate(headerSize).order(BIG_ENDIAN);
|
||||
decompressor.inflate(headerBuffer.array());
|
||||
|
||||
cookie = headerBuffer.getInt();
|
||||
final int payloadLengthInBytes;
|
||||
final int normalizingIndexOffset;
|
||||
final int numberOfSignificantValueDigits;
|
||||
final long lowestTrackableUnitValue;
|
||||
long highestTrackableValue;
|
||||
final double integerToDoubleValueConversionRatio;
|
||||
|
||||
assert getCookieBase(cookie) == V2EncodingCookieBase;
|
||||
|
||||
payloadLengthInBytes = headerBuffer.getInt(4);
|
||||
normalizingIndexOffset = headerBuffer.getInt(8);
|
||||
numberOfSignificantValueDigits = headerBuffer.getInt( 12);
|
||||
lowestTrackableUnitValue = headerBuffer.getLong(16);
|
||||
highestTrackableValue = headerBuffer.getLong(24);
|
||||
integerToDoubleValueConversionRatio = headerBuffer.getDouble(32);
|
||||
|
||||
highestTrackableValue = Math.max(highestTrackableValue, 2);
|
||||
|
||||
final long largestValueWithSingleUnitResolution = 2 * (long) Math.pow(10, numberOfSignificantValueDigits);
|
||||
final int unitMagnitude = (int) (Math.log(lowestTrackableUnitValue)/Math.log(2));
|
||||
final long unitMagnitudeMask = (1 << unitMagnitude) - 1;
|
||||
int subBucketCountMagnitude = (int) Math.ceil(Math.log(largestValueWithSingleUnitResolution)/Math.log(2));
|
||||
final int subBucketHalfCountMagnitude = subBucketCountMagnitude - 1;
|
||||
final int subBucketCount = 1 << subBucketCountMagnitude;
|
||||
final int subBucketHalfCount = subBucketCount / 2;
|
||||
final long subBucketMask = ((long)subBucketCount - 1) << unitMagnitude;
|
||||
if (subBucketCountMagnitude + unitMagnitude > 62) {
|
||||
// subBucketCount entries can't be represented, with unitMagnitude applied, in a positive long.
|
||||
// Technically it still sort of works if their sum is 63: you can represent all but the last number
|
||||
// in the shifted subBucketCount. However, the utility of such a histogram vs ones whose magnitude here
|
||||
// fits in 62 bits is debatable, and it makes it harder to work through the logic.
|
||||
// Sums larger than 64 are totally broken as leadingZeroCountBase would go negative.
|
||||
throw new IllegalArgumentException("Cannot represent numberOfSignificantValueDigits worth of values " +
|
||||
"beyond lowestDiscernibleValue");
|
||||
}
|
||||
|
||||
final int expectedCapacity = payloadLengthInBytes;
|
||||
|
||||
ByteBuffer sourceBuffer = ByteBuffer.allocate(expectedCapacity).order(BIG_ENDIAN);
|
||||
int decompressedByteCount = decompressor.inflate(sourceBuffer.array());
|
||||
decompressor.end(); // 必须手动调用，否则快速调用可能内存溢出(堆外内存)
|
||||
if ((payloadLengthInBytes != Integer.MAX_VALUE) && (decompressedByteCount < payloadLengthInBytes)) {
|
||||
throw new IllegalArgumentException("The buffer does not contain the indicated payload amount");
|
||||
}
|
||||
assert decompressedByteCount == expectedCapacity;
|
||||
|
||||
int dstIndex = 0;
|
||||
int endPosition = sourceBuffer.position() + expectedCapacity; //期望的结束读取的索引
|
||||
while (sourceBuffer.position() < endPosition) {
|
||||
long count;
|
||||
int zerosCount = 0;
|
||||
// V2 encoding format uses a long encoded in a ZigZag LEB128 format (up to V2maxWordSizeInBytes):
|
||||
count = ZigZagEncoding.getLong(sourceBuffer);
|
||||
if (count < 0) {
|
||||
long zc = -count; // 0值的连续个数
|
||||
if (zc > Integer.MAX_VALUE) {
|
||||
throw new IllegalArgumentException(
|
||||
"An encoded zero count of > Integer.MAX_VALUE was encountered in the source");
|
||||
}
|
||||
zerosCount = (int) zc;
|
||||
}
|
||||
if (zerosCount > 0) {
|
||||
dstIndex += zerosCount; // No need to set zeros in array. Just skip them.
|
||||
} else {
|
||||
// 单个非连续的0也会被输出
|
||||
if(count > 0){
|
||||
long value = valueFromIndex(dstIndex, subBucketHalfCountMagnitude, subBucketHalfCount, unitMagnitude);
|
||||
histogram.recordValueWithCount(value, count);
|
||||
}
|
||||
dstIndex++;
|
||||
}
|
||||
}
|
||||
|
||||
}else if(getCookieBase(cookie) == V2EncodingCookieBase){
|
||||
final int payloadLengthInBytes;
|
||||
final int normalizingIndexOffset;
|
||||
final int numberOfSignificantValueDigits;
|
||||
final long lowestTrackableUnitValue;
|
||||
long highestTrackableValue;
|
||||
final double integerToDoubleValueConversionRatio;
|
||||
|
||||
payloadLengthInBytes = byteBuffer.getInt(initPosition + 4);
|
||||
normalizingIndexOffset = byteBuffer.getInt(initPosition + 8);
|
||||
numberOfSignificantValueDigits = byteBuffer.getInt(initPosition + 12);
|
||||
lowestTrackableUnitValue = byteBuffer.getLong(initPosition + 16);
|
||||
highestTrackableValue = byteBuffer.getLong(initPosition + 24);
|
||||
integerToDoubleValueConversionRatio = byteBuffer.getDouble(initPosition + 32);
|
||||
|
||||
highestTrackableValue = Math.max(highestTrackableValue, 2);
|
||||
|
||||
final long largestValueWithSingleUnitResolution = 2 * (long) Math.pow(10, numberOfSignificantValueDigits);
|
||||
final int unitMagnitude = (int) (Math.log(lowestTrackableUnitValue)/Math.log(2));
|
||||
final long unitMagnitudeMask = (1 << unitMagnitude) - 1;
|
||||
int subBucketCountMagnitude = (int) Math.ceil(Math.log(largestValueWithSingleUnitResolution)/Math.log(2));
|
||||
final int subBucketHalfCountMagnitude = subBucketCountMagnitude - 1;
|
||||
final int subBucketCount = 1 << subBucketCountMagnitude;
|
||||
final int subBucketHalfCount = subBucketCount / 2;
|
||||
final long subBucketMask = ((long)subBucketCount - 1) << unitMagnitude;
|
||||
if (subBucketCountMagnitude + unitMagnitude > 62) {
|
||||
// subBucketCount entries can't be represented, with unitMagnitude applied, in a positive long.
|
||||
// Technically it still sort of works if their sum is 63: you can represent all but the last number
|
||||
// in the shifted subBucketCount. However, the utility of such a histogram vs ones whose magnitude here
|
||||
// fits in 62 bits is debatable, and it makes it harder to work through the logic.
|
||||
// Sums larger than 64 are totally broken as leadingZeroCountBase would go negative.
|
||||
throw new IllegalArgumentException("Cannot represent numberOfSignificantValueDigits worth of values " +
|
||||
"beyond lowestDiscernibleValue");
|
||||
}
|
||||
|
||||
final int expectedCapacity =payloadLengthInBytes;
|
||||
assert expectedCapacity == payloadLengthInBytes;
|
||||
if(expectedCapacity > byteBuffer.limit() - 40){
|
||||
throw new IllegalArgumentException("The buffer does not contain the full Histogram payload");
|
||||
}
|
||||
final int position = initPosition + 40;
|
||||
final int lengthInBytes = expectedCapacity;
|
||||
final int wordSizeInBytes = V2maxWordSizeInBytes;
|
||||
// fillCountsArrayFromSourceBuffer
|
||||
|
||||
ByteBuffer sourceBuffer = byteBuffer.duplicate();
|
||||
sourceBuffer.position(position);
|
||||
final long maxAllowableCountInHistigram = Long.MAX_VALUE;
|
||||
int dstIndex = 0;
|
||||
int endPosition = sourceBuffer.position() + lengthInBytes; //期望的结束读取的索引
|
||||
while (sourceBuffer.position() < endPosition) {
|
||||
long count;
|
||||
int zerosCount = 0;
|
||||
// V2 encoding format uses a long encoded in a ZigZag LEB128 format (up to V2maxWordSizeInBytes):
|
||||
count = ZigZagEncoding.getLong(sourceBuffer);
|
||||
if (count < 0) {
|
||||
long zc = -count; // 0值的连续个数
|
||||
if (zc > Integer.MAX_VALUE) {
|
||||
throw new IllegalArgumentException(
|
||||
"An encoded zero count of > Integer.MAX_VALUE was encountered in the source");
|
||||
}
|
||||
zerosCount = (int) zc;
|
||||
}
|
||||
if (zerosCount > 0) {
|
||||
dstIndex += zerosCount; // No need to set zeros in array. Just skip them.
|
||||
} else {
|
||||
// 单个非连续的0也会被输出
|
||||
if(count > 0){
|
||||
long value = valueFromIndex(dstIndex, subBucketHalfCountMagnitude, subBucketHalfCount, unitMagnitude);
|
||||
histogram.recordValueWithCount(value, count);
|
||||
}
|
||||
dstIndex++;
|
||||
}
|
||||
}
|
||||
}else{
|
||||
throw new RuntimeException("can not wrapByteBuffer");
|
||||
}
|
||||
}
|
||||
|
||||
final long valueFromIndex(final int index, int subBucketHalfCountMagnitude, int subBucketHalfCount, int unitMagnitude) {
|
||||
int bucketIndex = (index >> subBucketHalfCountMagnitude) - 1;
|
||||
int subBucketIndex = (index & (subBucketHalfCount - 1)) + subBucketHalfCount;
|
||||
if (bucketIndex < 0) {
|
||||
subBucketIndex -= subBucketHalfCount;
|
||||
bucketIndex = 0;
|
||||
}
|
||||
return valueFromIndex(bucketIndex, subBucketIndex, unitMagnitude);
|
||||
}
|
||||
|
||||
private long valueFromIndex(final int bucketIndex, final int subBucketIndex, int unitMagnitude) {
|
||||
return ((long) subBucketIndex) << (bucketIndex + unitMagnitude);
|
||||
}
|
||||
|
||||
static int getCookieBase(final int cookie) {
|
||||
return (cookie & ~0xf0);
|
||||
}
|
||||
|
||||
@Override
|
||||
public long getTotalCount() {
|
||||
if(totalCount >= 0){
|
||||
return totalCount;
|
||||
}
|
||||
try {
|
||||
totalCount = 0;
|
||||
int cookie = byteBuffer.getInt(initPosition);
|
||||
if(getCookieBase(cookie) == V2CompressedEncodingCookieBase){
|
||||
final int lengthOfCompressedContents = byteBuffer.getInt(initPosition + 4);
|
||||
final Inflater decompressor = new Inflater();
|
||||
|
||||
if (byteBuffer.hasArray()) {
|
||||
decompressor.setInput(byteBuffer.array(), initPosition + 8, lengthOfCompressedContents);
|
||||
} else {
|
||||
byte[] compressedContents = new byte[lengthOfCompressedContents];
|
||||
byteBuffer.position(initPosition + 8);
|
||||
try {
|
||||
byteBuffer.get(compressedContents);
|
||||
decompressor.setInput(compressedContents);
|
||||
}finally {
|
||||
byteBuffer.position(initPosition);
|
||||
}
|
||||
}
|
||||
final int headerSize = 40;
|
||||
final ByteBuffer headerBuffer = ByteBuffer.allocate(headerSize).order(BIG_ENDIAN);
|
||||
decompressor.inflate(headerBuffer.array());
|
||||
|
||||
cookie = headerBuffer.getInt();
|
||||
final int payloadLengthInBytes;
|
||||
final int normalizingIndexOffset;
|
||||
final int numberOfSignificantValueDigits;
|
||||
final long lowestTrackableUnitValue;
|
||||
long highestTrackableValue;
|
||||
final double integerToDoubleValueConversionRatio;
|
||||
|
||||
assert getCookieBase(cookie) == V2EncodingCookieBase;
|
||||
|
||||
payloadLengthInBytes = headerBuffer.getInt(4);
|
||||
normalizingIndexOffset = headerBuffer.getInt(8);
|
||||
numberOfSignificantValueDigits = headerBuffer.getInt( 12);
|
||||
lowestTrackableUnitValue = headerBuffer.getLong(16);
|
||||
highestTrackableValue = headerBuffer.getLong(24);
|
||||
integerToDoubleValueConversionRatio = headerBuffer.getDouble(32);
|
||||
|
||||
highestTrackableValue = Math.max(highestTrackableValue, 2);
|
||||
|
||||
final long largestValueWithSingleUnitResolution = 2 * (long) Math.pow(10, numberOfSignificantValueDigits);
|
||||
final int unitMagnitude = (int) (Math.log(lowestTrackableUnitValue)/Math.log(2));
|
||||
final long unitMagnitudeMask = (1 << unitMagnitude) - 1;
|
||||
int subBucketCountMagnitude = (int) Math.ceil(Math.log(largestValueWithSingleUnitResolution)/Math.log(2));
|
||||
final int subBucketHalfCountMagnitude = subBucketCountMagnitude - 1;
|
||||
final int subBucketCount = 1 << subBucketCountMagnitude;
|
||||
final int subBucketHalfCount = subBucketCount / 2;
|
||||
final long subBucketMask = ((long)subBucketCount - 1) << unitMagnitude;
|
||||
if (subBucketCountMagnitude + unitMagnitude > 62) {
|
||||
// subBucketCount entries can't be represented, with unitMagnitude applied, in a positive long.
|
||||
// Technically it still sort of works if their sum is 63: you can represent all but the last number
|
||||
// in the shifted subBucketCount. However, the utility of such a histogram vs ones whose magnitude here
|
||||
// fits in 62 bits is debatable, and it makes it harder to work through the logic.
|
||||
// Sums larger than 64 are totally broken as leadingZeroCountBase would go negative.
|
||||
throw new IllegalArgumentException("Cannot represent numberOfSignificantValueDigits worth of values " +
|
||||
"beyond lowestDiscernibleValue");
|
||||
}
|
||||
|
||||
final int expectedCapacity = payloadLengthInBytes;
|
||||
|
||||
ByteBuffer sourceBuffer = ByteBuffer.allocate(expectedCapacity).order(BIG_ENDIAN);
|
||||
int decompressedByteCount = decompressor.inflate(sourceBuffer.array());
|
||||
decompressor.end(); // 必须手动调用，否则快速调用可能内存溢出(堆外内存)
|
||||
if ((payloadLengthInBytes != Integer.MAX_VALUE) && (decompressedByteCount < payloadLengthInBytes)) {
|
||||
throw new IllegalArgumentException("The buffer does not contain the indicated payload amount");
|
||||
}
|
||||
assert decompressedByteCount == expectedCapacity;
|
||||
|
||||
int dstIndex = 0;
|
||||
int endPosition = sourceBuffer.position() + expectedCapacity; //期望的结束读取的索引
|
||||
while (sourceBuffer.position() < endPosition) {
|
||||
long count;
|
||||
int zerosCount = 0;
|
||||
// V2 encoding format uses a long encoded in a ZigZag LEB128 format (up to V2maxWordSizeInBytes):
|
||||
count = ZigZagEncoding.getLong(sourceBuffer);
|
||||
if (count < 0) {
|
||||
long zc = -count; // 0值的连续个数
|
||||
if (zc > Integer.MAX_VALUE) {
|
||||
throw new IllegalArgumentException(
|
||||
"An encoded zero count of > Integer.MAX_VALUE was encountered in the source");
|
||||
}
|
||||
zerosCount = (int) zc;
|
||||
}
|
||||
if (zerosCount > 0) {
|
||||
dstIndex += zerosCount; // No need to set zeros in array. Just skip them.
|
||||
} else {
|
||||
// 单个非连续的0也会被输出
|
||||
if(count > 0){
|
||||
//long value = valueFromIndex(dstIndex, subBucketHalfCountMagnitude, subBucketHalfCount, unitMagnitude);
|
||||
//histogram.recordValueWithCount(value, count);
|
||||
totalCount += count;
|
||||
}
|
||||
dstIndex++;
|
||||
}
|
||||
}
|
||||
return totalCount;
|
||||
}else if(getCookieBase(cookie) == V2EncodingCookieBase){
|
||||
final int payloadLengthInBytes;
|
||||
final int normalizingIndexOffset;
|
||||
final int numberOfSignificantValueDigits;
|
||||
final long lowestTrackableUnitValue;
|
||||
long highestTrackableValue;
|
||||
final double integerToDoubleValueConversionRatio;
|
||||
|
||||
payloadLengthInBytes = byteBuffer.getInt(initPosition + 4);
|
||||
normalizingIndexOffset = byteBuffer.getInt(initPosition + 8);
|
||||
numberOfSignificantValueDigits = byteBuffer.getInt(initPosition + 12);
|
||||
lowestTrackableUnitValue = byteBuffer.getLong(initPosition + 16);
|
||||
highestTrackableValue = byteBuffer.getLong(initPosition + 24);
|
||||
integerToDoubleValueConversionRatio = byteBuffer.getDouble(initPosition + 32);
|
||||
|
||||
highestTrackableValue = Math.max(highestTrackableValue, 2);
|
||||
|
||||
final long largestValueWithSingleUnitResolution = 2 * (long) Math.pow(10, numberOfSignificantValueDigits);
|
||||
final int unitMagnitude = (int) (Math.log(lowestTrackableUnitValue)/Math.log(2));
|
||||
final long unitMagnitudeMask = (1 << unitMagnitude) - 1;
|
||||
int subBucketCountMagnitude = (int) Math.ceil(Math.log(largestValueWithSingleUnitResolution)/Math.log(2));
|
||||
final int subBucketHalfCountMagnitude = subBucketCountMagnitude - 1;
|
||||
final int subBucketCount = 1 << subBucketCountMagnitude;
|
||||
final int subBucketHalfCount = subBucketCount / 2;
|
||||
final long subBucketMask = ((long)subBucketCount - 1) << unitMagnitude;
|
||||
if (subBucketCountMagnitude + unitMagnitude > 62) {
|
||||
// subBucketCount entries can't be represented, with unitMagnitude applied, in a positive long.
|
||||
// Technically it still sort of works if their sum is 63: you can represent all but the last number
|
||||
// in the shifted subBucketCount. However, the utility of such a histogram vs ones whose magnitude here
|
||||
// fits in 62 bits is debatable, and it makes it harder to work through the logic.
|
||||
// Sums larger than 64 are totally broken as leadingZeroCountBase would go negative.
|
||||
throw new IllegalArgumentException("Cannot represent numberOfSignificantValueDigits worth of values " +
|
||||
"beyond lowestDiscernibleValue");
|
||||
}
|
||||
|
||||
final int expectedCapacity =payloadLengthInBytes;
|
||||
assert expectedCapacity == payloadLengthInBytes;
|
||||
if(expectedCapacity > byteBuffer.limit() - 40){
|
||||
throw new IllegalArgumentException("The buffer does not contain the full Histogram payload");
|
||||
}
|
||||
final int position = initPosition + 40;
|
||||
final int lengthInBytes = expectedCapacity;
|
||||
final int wordSizeInBytes = V2maxWordSizeInBytes;
|
||||
// fillCountsArrayFromSourceBuffer
|
||||
|
||||
ByteBuffer sourceBuffer = byteBuffer.duplicate();
|
||||
sourceBuffer.position(position);
|
||||
final long maxAllowableCountInHistigram = Long.MAX_VALUE;
|
||||
int dstIndex = 0;
|
||||
int endPosition = sourceBuffer.position() + lengthInBytes; //期望的结束读取的索引
|
||||
while (sourceBuffer.position() < endPosition) {
|
||||
long count;
|
||||
int zerosCount = 0;
|
||||
// V2 encoding format uses a long encoded in a ZigZag LEB128 format (up to V2maxWordSizeInBytes):
|
||||
count = ZigZagEncoding.getLong(sourceBuffer);
|
||||
if (count < 0) {
|
||||
long zc = -count; // 0值的连续个数
|
||||
if (zc > Integer.MAX_VALUE) {
|
||||
throw new IllegalArgumentException(
|
||||
"An encoded zero count of > Integer.MAX_VALUE was encountered in the source");
|
||||
}
|
||||
zerosCount = (int) zc;
|
||||
}
|
||||
if (zerosCount > 0) {
|
||||
dstIndex += zerosCount; // No need to set zeros in array. Just skip them.
|
||||
} else {
|
||||
// 单个非连续的0也会被输出
|
||||
if(count > 0){
|
||||
//long value = valueFromIndex(dstIndex, subBucketHalfCountMagnitude, subBucketHalfCount, unitMagnitude);
|
||||
//histogram.recordValueWithCount(value, count);
|
||||
totalCount += count;
|
||||
}
|
||||
dstIndex++;
|
||||
}
|
||||
}
|
||||
return totalCount;
|
||||
}else{
|
||||
throw new UnsupportedOperationException("unsupported method");
|
||||
}
|
||||
} catch (DataFormatException e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void recordValue(long value) throws RuntimeException {
|
||||
throw new UnsupportedOperationException("unsupported method");
|
||||
}
|
||||
|
||||
@Override
|
||||
public void recordValueWithCount(long value, long count) throws RuntimeException {
|
||||
throw new UnsupportedOperationException("unsupported method");
|
||||
}
|
||||
|
||||
@Override
|
||||
public long getValueAtPercentile(double percentile) {
|
||||
throw new UnsupportedOperationException("unsupported method");
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<Percentile> percentileList(int percentileTicksPerHalfDistance) {
|
||||
throw new UnsupportedOperationException("unsupported method");
|
||||
}
|
||||
|
||||
@Override
|
||||
public Histogramer resetHistogram() {
|
||||
throw new UnsupportedOperationException("unsupported method");
|
||||
}
|
||||
|
||||
@Override
|
||||
public Histogramer merge(Histogramer histogram) {
|
||||
throw new UnsupportedOperationException("unsupported method");
|
||||
}
|
||||
|
||||
@Override
|
||||
public Histogramer makeCopy() throws RuntimeException{
|
||||
int cookie = byteBuffer.getInt(initPosition);
|
||||
if(getCookieBase(cookie) == V2CompressedEncodingCookieBase){
|
||||
try {
|
||||
return ArrayHistogram.decodeFromCompressedByteBuffer(byteBuffer, 2);
|
||||
} catch (DataFormatException e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
}else if(getCookieBase(cookie) == V2EncodingCookieBase){
|
||||
return ArrayHistogram.decodeFromByteBuffer(byteBuffer, 2);
|
||||
}
|
||||
throw new UnsupportedOperationException("unsupported method");
|
||||
}
|
||||
|
||||
@Override
|
||||
public byte[] toBytes() {
|
||||
int size = byteBuffer.limit() - initPosition;
|
||||
byte[] bytes = new byte[size];
|
||||
assert byteBuffer.order() == ByteOrder.BIG_ENDIAN;
|
||||
int oldPosition = byteBuffer.position();
|
||||
byteBuffer.position(initPosition);
|
||||
byteBuffer.get(bytes, 0, size);
|
||||
byteBuffer.position(oldPosition);
|
||||
return bytes;
|
||||
}
|
||||
}
|
||||
|
||||
package org.HdrHistogram;
|
||||
|
||||
import java.nio.ByteBuffer;
|
||||
import java.nio.ByteOrder;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.zip.DataFormatException;
|
||||
import java.util.zip.Inflater;
|
||||
|
||||
import static java.nio.ByteOrder.BIG_ENDIAN;
|
||||
|
||||
/**
|
||||
* 直接映射字节数组到Histogram，只读的Histogram，用于druid查询，减少gc减少计算，序列化后的是稀疏数组的形式
|
||||
*/
|
||||
public class DirectMapHistogram implements Histogramer{
|
||||
static final int V2maxWordSizeInBytes = 9; // LEB128-64b9B + ZigZag require up to 9 bytes per word
|
||||
static final int V2EncodingCookieBase = 0x1c849303;
|
||||
static final int V2CompressedEncodingCookieBase = 0x1c849304;
|
||||
|
||||
final ByteBuffer byteBuffer;
|
||||
final int initPosition;
|
||||
long totalCount;
|
||||
|
||||
private DirectMapHistogram(ByteBuffer byteBuffer) {
|
||||
int initPosition = byteBuffer.position();
|
||||
this.byteBuffer = byteBuffer;
|
||||
this.initPosition = initPosition;
|
||||
this.totalCount = -1;
|
||||
}
|
||||
|
||||
public static boolean byteBufferCanToDirectMapHistogram(ByteBuffer byteBuffer) {
|
||||
int initPosition = byteBuffer.position();
|
||||
int cookie = byteBuffer.getInt(initPosition);
|
||||
return getCookieBase(cookie) == V2EncodingCookieBase || getCookieBase(cookie) == V2CompressedEncodingCookieBase;
|
||||
}
|
||||
|
||||
public static DirectMapHistogram wrapBytes(byte[] bytes) {
|
||||
ByteBuffer byteBuffer = ByteBuffer.wrap(bytes);
|
||||
return wrapByteBuffer(byteBuffer);
|
||||
}
|
||||
|
||||
public static DirectMapHistogram wrapByteBuffer(ByteBuffer byteBuffer) {
|
||||
if(byteBufferCanToDirectMapHistogram(byteBuffer)){
|
||||
DirectMapHistogram hll = new DirectMapHistogram(byteBuffer);
|
||||
return hll;
|
||||
}
|
||||
throw new RuntimeException("can not wrapByteBuffer");
|
||||
}
|
||||
|
||||
public void mergeInto(AbstractHistogram histogram) throws Exception{
|
||||
int cookie = byteBuffer.getInt(initPosition);
|
||||
if(getCookieBase(cookie) == V2CompressedEncodingCookieBase){
|
||||
final int lengthOfCompressedContents = byteBuffer.getInt(initPosition + 4);
|
||||
final Inflater decompressor = new Inflater();
|
||||
|
||||
if (byteBuffer.hasArray()) {
|
||||
decompressor.setInput(byteBuffer.array(), initPosition + 8, lengthOfCompressedContents);
|
||||
} else {
|
||||
byte[] compressedContents = new byte[lengthOfCompressedContents];
|
||||
byteBuffer.position(initPosition + 8);
|
||||
try {
|
||||
byteBuffer.get(compressedContents);
|
||||
decompressor.setInput(compressedContents);
|
||||
}finally {
|
||||
byteBuffer.position(initPosition);
|
||||
}
|
||||
}
|
||||
final int headerSize = 40;
|
||||
final ByteBuffer headerBuffer = ByteBuffer.allocate(headerSize).order(BIG_ENDIAN);
|
||||
decompressor.inflate(headerBuffer.array());
|
||||
|
||||
cookie = headerBuffer.getInt();
|
||||
final int payloadLengthInBytes;
|
||||
final int normalizingIndexOffset;
|
||||
final int numberOfSignificantValueDigits;
|
||||
final long lowestTrackableUnitValue;
|
||||
long highestTrackableValue;
|
||||
final double integerToDoubleValueConversionRatio;
|
||||
|
||||
assert getCookieBase(cookie) == V2EncodingCookieBase;
|
||||
|
||||
payloadLengthInBytes = headerBuffer.getInt(4);
|
||||
normalizingIndexOffset = headerBuffer.getInt(8);
|
||||
numberOfSignificantValueDigits = headerBuffer.getInt( 12);
|
||||
lowestTrackableUnitValue = headerBuffer.getLong(16);
|
||||
highestTrackableValue = headerBuffer.getLong(24);
|
||||
integerToDoubleValueConversionRatio = headerBuffer.getDouble(32);
|
||||
|
||||
highestTrackableValue = Math.max(highestTrackableValue, 2);
|
||||
|
||||
final long largestValueWithSingleUnitResolution = 2 * (long) Math.pow(10, numberOfSignificantValueDigits);
|
||||
final int unitMagnitude = (int) (Math.log(lowestTrackableUnitValue)/Math.log(2));
|
||||
final long unitMagnitudeMask = (1 << unitMagnitude) - 1;
|
||||
int subBucketCountMagnitude = (int) Math.ceil(Math.log(largestValueWithSingleUnitResolution)/Math.log(2));
|
||||
final int subBucketHalfCountMagnitude = subBucketCountMagnitude - 1;
|
||||
final int subBucketCount = 1 << subBucketCountMagnitude;
|
||||
final int subBucketHalfCount = subBucketCount / 2;
|
||||
final long subBucketMask = ((long)subBucketCount - 1) << unitMagnitude;
|
||||
if (subBucketCountMagnitude + unitMagnitude > 62) {
|
||||
// subBucketCount entries can't be represented, with unitMagnitude applied, in a positive long.
|
||||
// Technically it still sort of works if their sum is 63: you can represent all but the last number
|
||||
// in the shifted subBucketCount. However, the utility of such a histogram vs ones whose magnitude here
|
||||
// fits in 62 bits is debatable, and it makes it harder to work through the logic.
|
||||
// Sums larger than 64 are totally broken as leadingZeroCountBase would go negative.
|
||||
throw new IllegalArgumentException("Cannot represent numberOfSignificantValueDigits worth of values " +
|
||||
"beyond lowestDiscernibleValue");
|
||||
}
|
||||
|
||||
final int expectedCapacity = payloadLengthInBytes;
|
||||
|
||||
ByteBuffer sourceBuffer = ByteBuffer.allocate(expectedCapacity).order(BIG_ENDIAN);
|
||||
int decompressedByteCount = decompressor.inflate(sourceBuffer.array());
|
||||
decompressor.end(); // 必须手动调用，否则快速调用可能内存溢出(堆外内存)
|
||||
if ((payloadLengthInBytes != Integer.MAX_VALUE) && (decompressedByteCount < payloadLengthInBytes)) {
|
||||
throw new IllegalArgumentException("The buffer does not contain the indicated payload amount");
|
||||
}
|
||||
assert decompressedByteCount == expectedCapacity;
|
||||
|
||||
int dstIndex = 0;
|
||||
int endPosition = sourceBuffer.position() + expectedCapacity; //期望的结束读取的索引
|
||||
while (sourceBuffer.position() < endPosition) {
|
||||
long count;
|
||||
int zerosCount = 0;
|
||||
// V2 encoding format uses a long encoded in a ZigZag LEB128 format (up to V2maxWordSizeInBytes):
|
||||
count = ZigZagEncoding.getLong(sourceBuffer);
|
||||
if (count < 0) {
|
||||
long zc = -count; // 0值的连续个数
|
||||
if (zc > Integer.MAX_VALUE) {
|
||||
throw new IllegalArgumentException(
|
||||
"An encoded zero count of > Integer.MAX_VALUE was encountered in the source");
|
||||
}
|
||||
zerosCount = (int) zc;
|
||||
}
|
||||
if (zerosCount > 0) {
|
||||
dstIndex += zerosCount; // No need to set zeros in array. Just skip them.
|
||||
} else {
|
||||
// 单个非连续的0也会被输出
|
||||
if(count > 0){
|
||||
long value = valueFromIndex(dstIndex, subBucketHalfCountMagnitude, subBucketHalfCount, unitMagnitude);
|
||||
histogram.recordValueWithCount(value, count);
|
||||
}
|
||||
dstIndex++;
|
||||
}
|
||||
}
|
||||
|
||||
}else if(getCookieBase(cookie) == V2EncodingCookieBase){
|
||||
final int payloadLengthInBytes;
|
||||
final int normalizingIndexOffset;
|
||||
final int numberOfSignificantValueDigits;
|
||||
final long lowestTrackableUnitValue;
|
||||
long highestTrackableValue;
|
||||
final double integerToDoubleValueConversionRatio;
|
||||
|
||||
payloadLengthInBytes = byteBuffer.getInt(initPosition + 4);
|
||||
normalizingIndexOffset = byteBuffer.getInt(initPosition + 8);
|
||||
numberOfSignificantValueDigits = byteBuffer.getInt(initPosition + 12);
|
||||
lowestTrackableUnitValue = byteBuffer.getLong(initPosition + 16);
|
||||
highestTrackableValue = byteBuffer.getLong(initPosition + 24);
|
||||
integerToDoubleValueConversionRatio = byteBuffer.getDouble(initPosition + 32);
|
||||
|
||||
highestTrackableValue = Math.max(highestTrackableValue, 2);
|
||||
|
||||
final long largestValueWithSingleUnitResolution = 2 * (long) Math.pow(10, numberOfSignificantValueDigits);
|
||||
final int unitMagnitude = (int) (Math.log(lowestTrackableUnitValue)/Math.log(2));
|
||||
final long unitMagnitudeMask = (1 << unitMagnitude) - 1;
|
||||
int subBucketCountMagnitude = (int) Math.ceil(Math.log(largestValueWithSingleUnitResolution)/Math.log(2));
|
||||
final int subBucketHalfCountMagnitude = subBucketCountMagnitude - 1;
|
||||
final int subBucketCount = 1 << subBucketCountMagnitude;
|
||||
final int subBucketHalfCount = subBucketCount / 2;
|
||||
final long subBucketMask = ((long)subBucketCount - 1) << unitMagnitude;
|
||||
if (subBucketCountMagnitude + unitMagnitude > 62) {
|
||||
// subBucketCount entries can't be represented, with unitMagnitude applied, in a positive long.
|
||||
// Technically it still sort of works if their sum is 63: you can represent all but the last number
|
||||
// in the shifted subBucketCount. However, the utility of such a histogram vs ones whose magnitude here
|
||||
// fits in 62 bits is debatable, and it makes it harder to work through the logic.
|
||||
// Sums larger than 64 are totally broken as leadingZeroCountBase would go negative.
|
||||
throw new IllegalArgumentException("Cannot represent numberOfSignificantValueDigits worth of values " +
|
||||
"beyond lowestDiscernibleValue");
|
||||
}
|
||||
|
||||
final int expectedCapacity =payloadLengthInBytes;
|
||||
assert expectedCapacity == payloadLengthInBytes;
|
||||
if(expectedCapacity > byteBuffer.limit() - 40){
|
||||
throw new IllegalArgumentException("The buffer does not contain the full Histogram payload");
|
||||
}
|
||||
final int position = initPosition + 40;
|
||||
final int lengthInBytes = expectedCapacity;
|
||||
final int wordSizeInBytes = V2maxWordSizeInBytes;
|
||||
// fillCountsArrayFromSourceBuffer
|
||||
|
||||
ByteBuffer sourceBuffer = byteBuffer.duplicate();
|
||||
sourceBuffer.position(position);
|
||||
final long maxAllowableCountInHistigram = Long.MAX_VALUE;
|
||||
int dstIndex = 0;
|
||||
int endPosition = sourceBuffer.position() + lengthInBytes; //期望的结束读取的索引
|
||||
while (sourceBuffer.position() < endPosition) {
|
||||
long count;
|
||||
int zerosCount = 0;
|
||||
// V2 encoding format uses a long encoded in a ZigZag LEB128 format (up to V2maxWordSizeInBytes):
|
||||
count = ZigZagEncoding.getLong(sourceBuffer);
|
||||
if (count < 0) {
|
||||
long zc = -count; // 0值的连续个数
|
||||
if (zc > Integer.MAX_VALUE) {
|
||||
throw new IllegalArgumentException(
|
||||
"An encoded zero count of > Integer.MAX_VALUE was encountered in the source");
|
||||
}
|
||||
zerosCount = (int) zc;
|
||||
}
|
||||
if (zerosCount > 0) {
|
||||
dstIndex += zerosCount; // No need to set zeros in array. Just skip them.
|
||||
} else {
|
||||
// 单个非连续的0也会被输出
|
||||
if(count > 0){
|
||||
long value = valueFromIndex(dstIndex, subBucketHalfCountMagnitude, subBucketHalfCount, unitMagnitude);
|
||||
histogram.recordValueWithCount(value, count);
|
||||
}
|
||||
dstIndex++;
|
||||
}
|
||||
}
|
||||
}else{
|
||||
throw new RuntimeException("can not wrapByteBuffer");
|
||||
}
|
||||
}
|
||||
|
||||
final long valueFromIndex(final int index, int subBucketHalfCountMagnitude, int subBucketHalfCount, int unitMagnitude) {
|
||||
int bucketIndex = (index >> subBucketHalfCountMagnitude) - 1;
|
||||
int subBucketIndex = (index & (subBucketHalfCount - 1)) + subBucketHalfCount;
|
||||
if (bucketIndex < 0) {
|
||||
subBucketIndex -= subBucketHalfCount;
|
||||
bucketIndex = 0;
|
||||
}
|
||||
return valueFromIndex(bucketIndex, subBucketIndex, unitMagnitude);
|
||||
}
|
||||
|
||||
private long valueFromIndex(final int bucketIndex, final int subBucketIndex, int unitMagnitude) {
|
||||
return ((long) subBucketIndex) << (bucketIndex + unitMagnitude);
|
||||
}
|
||||
|
||||
static int getCookieBase(final int cookie) {
|
||||
return (cookie & ~0xf0);
|
||||
}
|
||||
|
||||
@Override
|
||||
public long getTotalCount() {
|
||||
if(totalCount >= 0){
|
||||
return totalCount;
|
||||
}
|
||||
try {
|
||||
totalCount = 0;
|
||||
int cookie = byteBuffer.getInt(initPosition);
|
||||
if(getCookieBase(cookie) == V2CompressedEncodingCookieBase){
|
||||
final int lengthOfCompressedContents = byteBuffer.getInt(initPosition + 4);
|
||||
final Inflater decompressor = new Inflater();
|
||||
|
||||
if (byteBuffer.hasArray()) {
|
||||
decompressor.setInput(byteBuffer.array(), initPosition + 8, lengthOfCompressedContents);
|
||||
} else {
|
||||
byte[] compressedContents = new byte[lengthOfCompressedContents];
|
||||
byteBuffer.position(initPosition + 8);
|
||||
try {
|
||||
byteBuffer.get(compressedContents);
|
||||
decompressor.setInput(compressedContents);
|
||||
}finally {
|
||||
byteBuffer.position(initPosition);
|
||||
}
|
||||
}
|
||||
final int headerSize = 40;
|
||||
final ByteBuffer headerBuffer = ByteBuffer.allocate(headerSize).order(BIG_ENDIAN);
|
||||
decompressor.inflate(headerBuffer.array());
|
||||
|
||||
cookie = headerBuffer.getInt();
|
||||
final int payloadLengthInBytes;
|
||||
final int normalizingIndexOffset;
|
||||
final int numberOfSignificantValueDigits;
|
||||
final long lowestTrackableUnitValue;
|
||||
long highestTrackableValue;
|
||||
final double integerToDoubleValueConversionRatio;
|
||||
|
||||
assert getCookieBase(cookie) == V2EncodingCookieBase;
|
||||
|
||||
payloadLengthInBytes = headerBuffer.getInt(4);
|
||||
normalizingIndexOffset = headerBuffer.getInt(8);
|
||||
numberOfSignificantValueDigits = headerBuffer.getInt( 12);
|
||||
lowestTrackableUnitValue = headerBuffer.getLong(16);
|
||||
highestTrackableValue = headerBuffer.getLong(24);
|
||||
integerToDoubleValueConversionRatio = headerBuffer.getDouble(32);
|
||||
|
||||
highestTrackableValue = Math.max(highestTrackableValue, 2);
|
||||
|
||||
final long largestValueWithSingleUnitResolution = 2 * (long) Math.pow(10, numberOfSignificantValueDigits);
|
||||
final int unitMagnitude = (int) (Math.log(lowestTrackableUnitValue)/Math.log(2));
|
||||
final long unitMagnitudeMask = (1 << unitMagnitude) - 1;
|
||||
int subBucketCountMagnitude = (int) Math.ceil(Math.log(largestValueWithSingleUnitResolution)/Math.log(2));
|
||||
final int subBucketHalfCountMagnitude = subBucketCountMagnitude - 1;
|
||||
final int subBucketCount = 1 << subBucketCountMagnitude;
|
||||
final int subBucketHalfCount = subBucketCount / 2;
|
||||
final long subBucketMask = ((long)subBucketCount - 1) << unitMagnitude;
|
||||
if (subBucketCountMagnitude + unitMagnitude > 62) {
|
||||
// subBucketCount entries can't be represented, with unitMagnitude applied, in a positive long.
|
||||
// Technically it still sort of works if their sum is 63: you can represent all but the last number
|
||||
// in the shifted subBucketCount. However, the utility of such a histogram vs ones whose magnitude here
|
||||
// fits in 62 bits is debatable, and it makes it harder to work through the logic.
|
||||
// Sums larger than 64 are totally broken as leadingZeroCountBase would go negative.
|
||||
throw new IllegalArgumentException("Cannot represent numberOfSignificantValueDigits worth of values " +
|
||||
"beyond lowestDiscernibleValue");
|
||||
}
|
||||
|
||||
final int expectedCapacity = payloadLengthInBytes;
|
||||
|
||||
ByteBuffer sourceBuffer = ByteBuffer.allocate(expectedCapacity).order(BIG_ENDIAN);
|
||||
int decompressedByteCount = decompressor.inflate(sourceBuffer.array());
|
||||
decompressor.end(); // 必须手动调用，否则快速调用可能内存溢出(堆外内存)
|
||||
if ((payloadLengthInBytes != Integer.MAX_VALUE) && (decompressedByteCount < payloadLengthInBytes)) {
|
||||
throw new IllegalArgumentException("The buffer does not contain the indicated payload amount");
|
||||
}
|
||||
assert decompressedByteCount == expectedCapacity;
|
||||
|
||||
int dstIndex = 0;
|
||||
int endPosition = sourceBuffer.position() + expectedCapacity; //期望的结束读取的索引
|
||||
while (sourceBuffer.position() < endPosition) {
|
||||
long count;
|
||||
int zerosCount = 0;
|
||||
// V2 encoding format uses a long encoded in a ZigZag LEB128 format (up to V2maxWordSizeInBytes):
|
||||
count = ZigZagEncoding.getLong(sourceBuffer);
|
||||
if (count < 0) {
|
||||
long zc = -count; // 0值的连续个数
|
||||
if (zc > Integer.MAX_VALUE) {
|
||||
throw new IllegalArgumentException(
|
||||
"An encoded zero count of > Integer.MAX_VALUE was encountered in the source");
|
||||
}
|
||||
zerosCount = (int) zc;
|
||||
}
|
||||
if (zerosCount > 0) {
|
||||
dstIndex += zerosCount; // No need to set zeros in array. Just skip them.
|
||||
} else {
|
||||
// 单个非连续的0也会被输出
|
||||
if(count > 0){
|
||||
//long value = valueFromIndex(dstIndex, subBucketHalfCountMagnitude, subBucketHalfCount, unitMagnitude);
|
||||
//histogram.recordValueWithCount(value, count);
|
||||
totalCount += count;
|
||||
}
|
||||
dstIndex++;
|
||||
}
|
||||
}
|
||||
return totalCount;
|
||||
}else if(getCookieBase(cookie) == V2EncodingCookieBase){
|
||||
final int payloadLengthInBytes;
|
||||
final int normalizingIndexOffset;
|
||||
final int numberOfSignificantValueDigits;
|
||||
final long lowestTrackableUnitValue;
|
||||
long highestTrackableValue;
|
||||
final double integerToDoubleValueConversionRatio;
|
||||
|
||||
payloadLengthInBytes = byteBuffer.getInt(initPosition + 4);
|
||||
normalizingIndexOffset = byteBuffer.getInt(initPosition + 8);
|
||||
numberOfSignificantValueDigits = byteBuffer.getInt(initPosition + 12);
|
||||
lowestTrackableUnitValue = byteBuffer.getLong(initPosition + 16);
|
||||
highestTrackableValue = byteBuffer.getLong(initPosition + 24);
|
||||
integerToDoubleValueConversionRatio = byteBuffer.getDouble(initPosition + 32);
|
||||
|
||||
highestTrackableValue = Math.max(highestTrackableValue, 2);
|
||||
|
||||
final long largestValueWithSingleUnitResolution = 2 * (long) Math.pow(10, numberOfSignificantValueDigits);
|
||||
final int unitMagnitude = (int) (Math.log(lowestTrackableUnitValue)/Math.log(2));
|
||||
final long unitMagnitudeMask = (1 << unitMagnitude) - 1;
|
||||
int subBucketCountMagnitude = (int) Math.ceil(Math.log(largestValueWithSingleUnitResolution)/Math.log(2));
|
||||
final int subBucketHalfCountMagnitude = subBucketCountMagnitude - 1;
|
||||
final int subBucketCount = 1 << subBucketCountMagnitude;
|
||||
final int subBucketHalfCount = subBucketCount / 2;
|
||||
final long subBucketMask = ((long)subBucketCount - 1) << unitMagnitude;
|
||||
if (subBucketCountMagnitude + unitMagnitude > 62) {
|
||||
// subBucketCount entries can't be represented, with unitMagnitude applied, in a positive long.
|
||||
// Technically it still sort of works if their sum is 63: you can represent all but the last number
|
||||
// in the shifted subBucketCount. However, the utility of such a histogram vs ones whose magnitude here
|
||||
// fits in 62 bits is debatable, and it makes it harder to work through the logic.
|
||||
// Sums larger than 64 are totally broken as leadingZeroCountBase would go negative.
|
||||
throw new IllegalArgumentException("Cannot represent numberOfSignificantValueDigits worth of values " +
|
||||
"beyond lowestDiscernibleValue");
|
||||
}
|
||||
|
||||
final int expectedCapacity =payloadLengthInBytes;
|
||||
assert expectedCapacity == payloadLengthInBytes;
|
||||
if(expectedCapacity > byteBuffer.limit() - 40){
|
||||
throw new IllegalArgumentException("The buffer does not contain the full Histogram payload");
|
||||
}
|
||||
final int position = initPosition + 40;
|
||||
final int lengthInBytes = expectedCapacity;
|
||||
final int wordSizeInBytes = V2maxWordSizeInBytes;
|
||||
// fillCountsArrayFromSourceBuffer
|
||||
|
||||
ByteBuffer sourceBuffer = byteBuffer.duplicate();
|
||||
sourceBuffer.position(position);
|
||||
final long maxAllowableCountInHistigram = Long.MAX_VALUE;
|
||||
int dstIndex = 0;
|
||||
int endPosition = sourceBuffer.position() + lengthInBytes; //期望的结束读取的索引
|
||||
while (sourceBuffer.position() < endPosition) {
|
||||
long count;
|
||||
int zerosCount = 0;
|
||||
// V2 encoding format uses a long encoded in a ZigZag LEB128 format (up to V2maxWordSizeInBytes):
|
||||
count = ZigZagEncoding.getLong(sourceBuffer);
|
||||
if (count < 0) {
|
||||
long zc = -count; // 0值的连续个数
|
||||
if (zc > Integer.MAX_VALUE) {
|
||||
throw new IllegalArgumentException(
|
||||
"An encoded zero count of > Integer.MAX_VALUE was encountered in the source");
|
||||
}
|
||||
zerosCount = (int) zc;
|
||||
}
|
||||
if (zerosCount > 0) {
|
||||
dstIndex += zerosCount; // No need to set zeros in array. Just skip them.
|
||||
} else {
|
||||
// 单个非连续的0也会被输出
|
||||
if(count > 0){
|
||||
//long value = valueFromIndex(dstIndex, subBucketHalfCountMagnitude, subBucketHalfCount, unitMagnitude);
|
||||
//histogram.recordValueWithCount(value, count);
|
||||
totalCount += count;
|
||||
}
|
||||
dstIndex++;
|
||||
}
|
||||
}
|
||||
return totalCount;
|
||||
}else{
|
||||
throw new UnsupportedOperationException("unsupported method");
|
||||
}
|
||||
} catch (DataFormatException e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void recordValue(long value) throws RuntimeException {
|
||||
throw new UnsupportedOperationException("unsupported method");
|
||||
}
|
||||
|
||||
@Override
|
||||
public void recordValueWithCount(long value, long count) throws RuntimeException {
|
||||
throw new UnsupportedOperationException("unsupported method");
|
||||
}
|
||||
|
||||
@Override
|
||||
public long getValueAtPercentile(double percentile) {
|
||||
throw new UnsupportedOperationException("unsupported method");
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<Percentile> percentileList(int percentileTicksPerHalfDistance) {
|
||||
throw new UnsupportedOperationException("unsupported method");
|
||||
}
|
||||
|
||||
@Override
|
||||
public Map<String, Object> describe() {
|
||||
throw new UnsupportedOperationException("unsupported method");
|
||||
}
|
||||
|
||||
@Override
|
||||
public Histogramer resetHistogram() {
|
||||
throw new UnsupportedOperationException("unsupported method");
|
||||
}
|
||||
|
||||
@Override
|
||||
public Histogramer merge(Histogramer histogram) {
|
||||
throw new UnsupportedOperationException("unsupported method");
|
||||
}
|
||||
|
||||
@Override
|
||||
public Histogramer makeCopy() throws RuntimeException{
|
||||
int cookie = byteBuffer.getInt(initPosition);
|
||||
if(getCookieBase(cookie) == V2CompressedEncodingCookieBase){
|
||||
try {
|
||||
return ArrayHistogram.decodeFromCompressedByteBuffer(byteBuffer, 2);
|
||||
} catch (DataFormatException e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
}else if(getCookieBase(cookie) == V2EncodingCookieBase){
|
||||
return ArrayHistogram.decodeFromByteBuffer(byteBuffer, 2);
|
||||
}
|
||||
throw new UnsupportedOperationException("unsupported method");
|
||||
}
|
||||
|
||||
@Override
|
||||
public byte[] toBytes() {
|
||||
int size = byteBuffer.limit() - initPosition;
|
||||
byte[] bytes = new byte[size];
|
||||
assert byteBuffer.order() == ByteOrder.BIG_ENDIAN;
|
||||
int oldPosition = byteBuffer.position();
|
||||
byteBuffer.position(initPosition);
|
||||
byteBuffer.get(bytes, 0, size);
|
||||
byteBuffer.position(oldPosition);
|
||||
return bytes;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -1,85 +1,90 @@
|
||||
package org.HdrHistogram;
|
||||
|
||||
import java.nio.ByteBuffer;
|
||||
import java.util.List;
|
||||
|
||||
public class HistogramSketch {
|
||||
public Histogramer hisImpl = null;
|
||||
|
||||
public HistogramSketch(final int numberOfSignificantValueDigits){
|
||||
hisImpl = new ArrayHistogram(numberOfSignificantValueDigits);
|
||||
}
|
||||
|
||||
public HistogramSketch(final long lowestDiscernibleValue, final long highestTrackableValue,
|
||||
final int numberOfSignificantValueDigits, final boolean autoResize){
|
||||
ArrayHistogram histogram = new ArrayHistogram(lowestDiscernibleValue, highestTrackableValue, numberOfSignificantValueDigits);
|
||||
histogram.setAutoResize(autoResize);
|
||||
hisImpl = histogram;
|
||||
}
|
||||
|
||||
public HistogramSketch(final Histogramer that) {
|
||||
hisImpl = that;
|
||||
}
|
||||
|
||||
/**
|
||||
* Copy constructor used by copy().
|
||||
*/
|
||||
HistogramSketch(final HistogramSketch that) {
|
||||
hisImpl = that.hisImpl.makeCopy();
|
||||
}
|
||||
|
||||
/**
|
||||
* 复制hisImpl到堆内存实例hisImpl
|
||||
*/
|
||||
public HistogramSketch copy() {
|
||||
return new HistogramSketch(this);
|
||||
}
|
||||
|
||||
public void reset() {
|
||||
hisImpl = hisImpl.resetHistogram();
|
||||
}
|
||||
|
||||
public long getTotalCount(){
|
||||
return hisImpl.getTotalCount();
|
||||
}
|
||||
|
||||
public void recordValue(long value){
|
||||
hisImpl.recordValue(value);
|
||||
}
|
||||
|
||||
public void recordValueWithCount(long value, long count){
|
||||
hisImpl.recordValueWithCount(value, count);
|
||||
}
|
||||
|
||||
public long getValueAtPercentile(double percentile){
|
||||
return hisImpl.getValueAtPercentile(percentile);
|
||||
}
|
||||
|
||||
public List<Percentile> percentileList(int percentileTicksPerHalfDistance){
|
||||
return hisImpl.percentileList(percentileTicksPerHalfDistance);
|
||||
}
|
||||
|
||||
public static final int getUpdatableSerializationBytes(long lowestDiscernibleValue, long highestTrackableValue, int numberOfSignificantValueDigits){
|
||||
return DirectArrayHistogram.getUpdatableSerializationBytes(lowestDiscernibleValue, highestTrackableValue, numberOfSignificantValueDigits);
|
||||
}
|
||||
|
||||
public byte[] toBytes() {
|
||||
return hisImpl.toBytes();
|
||||
}
|
||||
|
||||
public static HistogramSketch fromBytes(byte[] bytes) {
|
||||
return new HistogramSketch(ArrayHistogram.fromBytes(bytes));
|
||||
}
|
||||
|
||||
public static HistogramSketch fromByteBuffer(ByteBuffer byteBuffer) {
|
||||
return new HistogramSketch(ArrayHistogram.fromByteBuffer(byteBuffer));
|
||||
}
|
||||
|
||||
public static HistogramSketch wrapBytes(byte[] bytes) {
|
||||
return new HistogramSketch(DirectMapHistogram.wrapBytes(bytes));
|
||||
}
|
||||
|
||||
public static HistogramSketch wrapByteBuffer(ByteBuffer byteBuffer) {
|
||||
return new HistogramSketch(DirectMapHistogram.wrapByteBuffer(byteBuffer));
|
||||
}
|
||||
}
|
||||
package org.HdrHistogram;
|
||||
|
||||
import java.nio.ByteBuffer;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
public class HistogramSketch {
|
||||
public Histogramer hisImpl = null;
|
||||
|
||||
public HistogramSketch(final int numberOfSignificantValueDigits){
|
||||
hisImpl = new ArrayHistogram(numberOfSignificantValueDigits);
|
||||
}
|
||||
|
||||
public HistogramSketch(final long lowestDiscernibleValue, final long highestTrackableValue,
|
||||
final int numberOfSignificantValueDigits, final boolean autoResize){
|
||||
ArrayHistogram histogram = new ArrayHistogram(lowestDiscernibleValue, highestTrackableValue, numberOfSignificantValueDigits);
|
||||
histogram.setAutoResize(autoResize);
|
||||
hisImpl = histogram;
|
||||
}
|
||||
|
||||
public HistogramSketch(final Histogramer that) {
|
||||
hisImpl = that;
|
||||
}
|
||||
|
||||
/**
|
||||
* Copy constructor used by copy().
|
||||
*/
|
||||
HistogramSketch(final HistogramSketch that) {
|
||||
hisImpl = that.hisImpl.makeCopy();
|
||||
}
|
||||
|
||||
/**
|
||||
* 复制hisImpl到堆内存实例hisImpl
|
||||
*/
|
||||
public HistogramSketch copy() {
|
||||
return new HistogramSketch(this);
|
||||
}
|
||||
|
||||
public void reset() {
|
||||
hisImpl = hisImpl.resetHistogram();
|
||||
}
|
||||
|
||||
public long getTotalCount(){
|
||||
return hisImpl.getTotalCount();
|
||||
}
|
||||
|
||||
public void recordValue(long value){
|
||||
hisImpl.recordValue(value);
|
||||
}
|
||||
|
||||
public void recordValueWithCount(long value, long count){
|
||||
hisImpl.recordValueWithCount(value, count);
|
||||
}
|
||||
|
||||
public long getValueAtPercentile(double percentile){
|
||||
return hisImpl.getValueAtPercentile(percentile);
|
||||
}
|
||||
|
||||
public List<Percentile> percentileList(int percentileTicksPerHalfDistance){
|
||||
return hisImpl.percentileList(percentileTicksPerHalfDistance);
|
||||
}
|
||||
|
||||
public Map<String, Object> describe(){
|
||||
return hisImpl.describe();
|
||||
}
|
||||
|
||||
public static final int getUpdatableSerializationBytes(long lowestDiscernibleValue, long highestTrackableValue, int numberOfSignificantValueDigits){
|
||||
return DirectArrayHistogram.getUpdatableSerializationBytes(lowestDiscernibleValue, highestTrackableValue, numberOfSignificantValueDigits);
|
||||
}
|
||||
|
||||
public byte[] toBytes() {
|
||||
return hisImpl.toBytes();
|
||||
}
|
||||
|
||||
public static HistogramSketch fromBytes(byte[] bytes) {
|
||||
return new HistogramSketch(ArrayHistogram.fromBytes(bytes));
|
||||
}
|
||||
|
||||
public static HistogramSketch fromByteBuffer(ByteBuffer byteBuffer) {
|
||||
return new HistogramSketch(ArrayHistogram.fromByteBuffer(byteBuffer));
|
||||
}
|
||||
|
||||
public static HistogramSketch wrapBytes(byte[] bytes) {
|
||||
return new HistogramSketch(DirectMapHistogram.wrapBytes(bytes));
|
||||
}
|
||||
|
||||
public static HistogramSketch wrapByteBuffer(ByteBuffer byteBuffer) {
|
||||
return new HistogramSketch(DirectMapHistogram.wrapByteBuffer(byteBuffer));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,34 +1,37 @@
|
||||
package org.HdrHistogram;
|
||||
|
||||
import java.nio.ByteBuffer;
|
||||
import java.util.List;
|
||||
|
||||
public interface Histogramer {
|
||||
long getTotalCount();
|
||||
|
||||
void recordValue(long value) throws RuntimeException;
|
||||
|
||||
void recordValueWithCount(long value, long count) throws RuntimeException;
|
||||
|
||||
long getValueAtPercentile(double percentile);
|
||||
|
||||
List<Percentile> percentileList(int percentileTicksPerHalfDistance);
|
||||
|
||||
Histogramer resetHistogram();
|
||||
|
||||
Histogramer merge(Histogramer histogram);
|
||||
|
||||
// 复制到堆内存实例ArrayHistogram
|
||||
Histogramer makeCopy();
|
||||
|
||||
byte[] toBytes();
|
||||
|
||||
default byte[] byteBuffer2Bytes(ByteBuffer byteBuffer){
|
||||
//必须调用完后flip()才可以调用此方法
|
||||
byteBuffer.flip();
|
||||
int len = byteBuffer.limit() - byteBuffer.position();
|
||||
byte[] bytes = new byte[len];
|
||||
byteBuffer.get(bytes);
|
||||
return bytes;
|
||||
}
|
||||
}
|
||||
package org.HdrHistogram;
|
||||
|
||||
import java.nio.ByteBuffer;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
public interface Histogramer {
|
||||
long getTotalCount();
|
||||
|
||||
void recordValue(long value) throws RuntimeException;
|
||||
|
||||
void recordValueWithCount(long value, long count) throws RuntimeException;
|
||||
|
||||
long getValueAtPercentile(double percentile);
|
||||
|
||||
List<Percentile> percentileList(int percentileTicksPerHalfDistance);
|
||||
|
||||
Map<String, Object> describe();
|
||||
|
||||
Histogramer resetHistogram();
|
||||
|
||||
Histogramer merge(Histogramer histogram);
|
||||
|
||||
// 复制到堆内存实例ArrayHistogram
|
||||
Histogramer makeCopy();
|
||||
|
||||
byte[] toBytes();
|
||||
|
||||
default byte[] byteBuffer2Bytes(ByteBuffer byteBuffer){
|
||||
//必须调用完后flip()才可以调用此方法
|
||||
byteBuffer.flip();
|
||||
int len = byteBuffer.limit() - byteBuffer.position();
|
||||
byte[] bytes = new byte[len];
|
||||
byteBuffer.get(bytes);
|
||||
return bytes;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,41 +1,50 @@
|
||||
package org.HdrHistogram;
|
||||
|
||||
public class Percentile {
|
||||
public long value;
|
||||
public long count;
|
||||
public double percentile;
|
||||
|
||||
public Percentile() {
|
||||
|
||||
}
|
||||
|
||||
public Percentile(long value, long count, double percentile) {
|
||||
this.value = value;
|
||||
this.count = count;
|
||||
this.percentile = percentile;
|
||||
}
|
||||
|
||||
public long getValue() {
|
||||
return value;
|
||||
}
|
||||
|
||||
public void setValue(long value) {
|
||||
this.value = value;
|
||||
}
|
||||
|
||||
public long getCount() {
|
||||
return count;
|
||||
}
|
||||
|
||||
public void setCount(long count) {
|
||||
this.count = count;
|
||||
}
|
||||
|
||||
public double getPercentile() {
|
||||
return percentile;
|
||||
}
|
||||
|
||||
public void setPercentile(double percentile) {
|
||||
this.percentile = percentile;
|
||||
}
|
||||
}
|
||||
package org.HdrHistogram;
|
||||
|
||||
public class Percentile {
|
||||
public long value;
|
||||
public long count;
|
||||
public double percentile;
|
||||
|
||||
public Percentile() {
|
||||
|
||||
}
|
||||
|
||||
public Percentile(long value, long count, double percentile) {
|
||||
this.value = value;
|
||||
this.count = count;
|
||||
this.percentile = percentile;
|
||||
}
|
||||
|
||||
public long getValue() {
|
||||
return value;
|
||||
}
|
||||
|
||||
public void setValue(long value) {
|
||||
this.value = value;
|
||||
}
|
||||
|
||||
public long getCount() {
|
||||
return count;
|
||||
}
|
||||
|
||||
public void setCount(long count) {
|
||||
this.count = count;
|
||||
}
|
||||
|
||||
public double getPercentile() {
|
||||
return percentile;
|
||||
}
|
||||
|
||||
public void setPercentile(double percentile) {
|
||||
this.percentile = percentile;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return "Percentile{" +
|
||||
"value=" + value +
|
||||
", count=" + count +
|
||||
", percentile=" + percentile +
|
||||
'}';
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,321 +1,348 @@
|
||||
package org.apache.druid.query.aggregation.sketch.HdrHistogram;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||
import org.HdrHistogram.DirectHistogram;
|
||||
import org.HdrHistogram.Histogram;
|
||||
import org.HdrHistogram.HistogramSketch;
|
||||
import org.HdrHistogram.HistogramUnion;
|
||||
import org.apache.druid.java.util.common.IAE;
|
||||
import org.apache.druid.query.aggregation.*;
|
||||
import org.apache.druid.query.cache.CacheKeyBuilder;
|
||||
import org.apache.druid.segment.ColumnSelectorFactory;
|
||||
import org.apache.druid.segment.ColumnValueSelector;
|
||||
|
||||
import javax.annotation.Nullable;
|
||||
import java.util.Collections;
|
||||
import java.util.Comparator;
|
||||
import java.util.List;
|
||||
import java.util.Objects;
|
||||
|
||||
public class HdrHistogramAggregatorFactory extends AggregatorFactory {
|
||||
public static final long DEFAULT_LOWEST = 1;
|
||||
public static final long DEFAULT_HIGHEST = 2;
|
||||
public static final int DEFAULT_SIGNIFICANT = 3;
|
||||
public static final boolean DEFAULT_AUTO_RESIZE = true;
|
||||
public static final long BUFFER_AUTO_RESIZE_HIGHEST = 100000000L * 1000000L;
|
||||
public static final Comparator<HistogramSketch> COMPARATOR =
|
||||
Comparator.nullsFirst(Comparator.comparingLong(HistogramSketch::getTotalCount));
|
||||
|
||||
protected final String name;
|
||||
protected final String fieldName;
|
||||
protected final long lowestDiscernibleValue;
|
||||
protected final long highestTrackableValue;
|
||||
protected final int numberOfSignificantValueDigits;
|
||||
protected final boolean autoResize; //默认是false
|
||||
|
||||
public HdrHistogramAggregatorFactory(
|
||||
@JsonProperty("name") String name,
|
||||
@JsonProperty("fieldName") String fieldName,
|
||||
@JsonProperty("lowestDiscernibleValue") @Nullable Long lowestDiscernibleValue,
|
||||
@JsonProperty("highestTrackableValue") @Nullable Long highestTrackableValue,
|
||||
@JsonProperty("numberOfSignificantValueDigits") @Nullable Integer numberOfSignificantValueDigits,
|
||||
@JsonProperty("autoResize") @Nullable Boolean autoResize
|
||||
) {
|
||||
if (name == null) {
|
||||
throw new IAE("Must have a valid, non-null aggregator name");
|
||||
}
|
||||
if (fieldName == null) {
|
||||
throw new IAE("Parameter fieldName must be specified");
|
||||
}
|
||||
|
||||
if(lowestDiscernibleValue == null){
|
||||
lowestDiscernibleValue = DEFAULT_LOWEST;
|
||||
}
|
||||
// Verify argument validity
|
||||
if (lowestDiscernibleValue < 1) {
|
||||
throw new IAE("lowestDiscernibleValue must be >= 1");
|
||||
}
|
||||
if (lowestDiscernibleValue > Long.MAX_VALUE / 2) {
|
||||
// prevent subsequent multiplication by 2 for highestTrackableValue check from overflowing
|
||||
throw new IAE("lowestDiscernibleValue must be <= Long.MAX_VALUE / 2");
|
||||
}
|
||||
if(highestTrackableValue == null){
|
||||
highestTrackableValue = DEFAULT_HIGHEST;
|
||||
}
|
||||
if (highestTrackableValue < 2L * lowestDiscernibleValue) {
|
||||
throw new IAE("highestTrackableValue must be >= 2 * lowestDiscernibleValue");
|
||||
}
|
||||
if(numberOfSignificantValueDigits == null){
|
||||
numberOfSignificantValueDigits = DEFAULT_SIGNIFICANT;
|
||||
}
|
||||
if ((numberOfSignificantValueDigits < 0) || (numberOfSignificantValueDigits > 5)) {
|
||||
throw new IAE("numberOfSignificantValueDigits must be between 0 and 5");
|
||||
}
|
||||
if(autoResize == null){
|
||||
autoResize = DEFAULT_AUTO_RESIZE;
|
||||
}
|
||||
|
||||
this.name = name;
|
||||
this.fieldName = fieldName;
|
||||
this.lowestDiscernibleValue = lowestDiscernibleValue;
|
||||
this.highestTrackableValue = highestTrackableValue;
|
||||
this.numberOfSignificantValueDigits = numberOfSignificantValueDigits;
|
||||
this.autoResize = autoResize;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Aggregator factorize(ColumnSelectorFactory metricFactory) {
|
||||
return new HdrHistogramAggregator(
|
||||
metricFactory.makeColumnValueSelector(fieldName),
|
||||
lowestDiscernibleValue,
|
||||
highestTrackableValue,
|
||||
numberOfSignificantValueDigits,
|
||||
autoResize
|
||||
);
|
||||
}
|
||||
|
||||
@Override
|
||||
public BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory) {
|
||||
return new HdrHistogramBufferAggregator(
|
||||
metricFactory.makeColumnValueSelector(fieldName),
|
||||
lowestDiscernibleValue,
|
||||
highestTrackableValue,
|
||||
numberOfSignificantValueDigits,
|
||||
autoResize,
|
||||
getMaxIntermediateSize()
|
||||
);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Comparator getComparator() {
|
||||
return COMPARATOR;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Object combine(Object lhs, Object rhs) {
|
||||
if(lhs == null){
|
||||
return rhs;
|
||||
}else if(rhs == null){
|
||||
return lhs;
|
||||
}else{
|
||||
final HistogramUnion union = new HistogramUnion(lowestDiscernibleValue,highestTrackableValue,numberOfSignificantValueDigits,autoResize);
|
||||
union.update((HistogramSketch) lhs);
|
||||
union.update((HistogramSketch) rhs);
|
||||
HistogramSketch result = union.getResult();
|
||||
return result;
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public AggregateCombiner makeAggregateCombiner() {
|
||||
return new ObjectAggregateCombiner<HistogramSketch>() {
|
||||
private HistogramUnion union = null;
|
||||
|
||||
@Override
|
||||
public void reset(ColumnValueSelector selector) {
|
||||
//union.reset();
|
||||
union = null;
|
||||
fold(selector);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void fold(ColumnValueSelector selector) {
|
||||
HistogramSketch h = (HistogramSketch) selector.getObject();
|
||||
if(h != null){
|
||||
if(union == null){
|
||||
union = new HistogramUnion(lowestDiscernibleValue,highestTrackableValue,numberOfSignificantValueDigits,autoResize);
|
||||
}
|
||||
union.update(h);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public Class<HistogramSketch> classOfObject() {
|
||||
return HistogramSketch.class;
|
||||
}
|
||||
|
||||
@Nullable
|
||||
@Override
|
||||
public HistogramSketch getObject() {
|
||||
if(union == null){
|
||||
return null;
|
||||
}else{
|
||||
HistogramSketch result = union.getResult();
|
||||
/*if(result.getTotalCount() == 0){
|
||||
return null;
|
||||
}*/
|
||||
return result;
|
||||
}
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
/*public Histogram geneHistogram() {
|
||||
Histogram histogram = new Histogram(lowestDiscernibleValue, highestTrackableValue, numberOfSignificantValueDigits);
|
||||
histogram.setAutoResize(autoResize);
|
||||
return histogram;
|
||||
}*/
|
||||
|
||||
@Override
|
||||
public AggregatorFactory getCombiningFactory() {
|
||||
return new HdrHistogramMergeAggregatorFactory(name, name, lowestDiscernibleValue, highestTrackableValue, numberOfSignificantValueDigits, autoResize);
|
||||
}
|
||||
|
||||
@Override
|
||||
public AggregatorFactory getMergingFactory(AggregatorFactory other) throws AggregatorFactoryNotMergeableException {
|
||||
if (other.getName().equals(this.getName()) && other instanceof HdrHistogramAggregatorFactory) {
|
||||
HdrHistogramAggregatorFactory castedOther = (HdrHistogramAggregatorFactory) other;
|
||||
|
||||
return new HdrHistogramMergeAggregatorFactory(name, name,
|
||||
Math.min(lowestDiscernibleValue, castedOther.lowestDiscernibleValue),
|
||||
Math.max(highestTrackableValue, castedOther.highestTrackableValue),
|
||||
Math.max(numberOfSignificantValueDigits, castedOther.numberOfSignificantValueDigits),
|
||||
autoResize || castedOther.autoResize
|
||||
);
|
||||
} else {
|
||||
throw new AggregatorFactoryNotMergeableException(this, other);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<AggregatorFactory> getRequiredColumns() {
|
||||
return Collections.singletonList(
|
||||
new HdrHistogramAggregatorFactory(
|
||||
fieldName,
|
||||
fieldName,
|
||||
lowestDiscernibleValue, highestTrackableValue, numberOfSignificantValueDigits, autoResize
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Object deserialize(Object object) {
|
||||
return HistogramUtils.deserializeHistogram(object);
|
||||
}
|
||||
|
||||
@Nullable
|
||||
@Override
|
||||
public Object finalizeComputation(@Nullable Object object) {
|
||||
return object == null ? null : ((HistogramSketch) object).getTotalCount();
|
||||
}
|
||||
|
||||
@Override
|
||||
@JsonProperty
|
||||
public String getName() {
|
||||
return name;
|
||||
}
|
||||
|
||||
@JsonProperty
|
||||
public String getFieldName() {
|
||||
return fieldName;
|
||||
}
|
||||
|
||||
@JsonProperty
|
||||
public long getLowestDiscernibleValue() {
|
||||
return lowestDiscernibleValue;
|
||||
}
|
||||
|
||||
@JsonProperty
|
||||
public long getHighestTrackableValue() {
|
||||
return highestTrackableValue;
|
||||
}
|
||||
|
||||
@JsonProperty
|
||||
public int getNumberOfSignificantValueDigits() {
|
||||
return numberOfSignificantValueDigits;
|
||||
}
|
||||
|
||||
@JsonProperty
|
||||
public boolean isAutoResize() {
|
||||
return autoResize;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getTypeName() {
|
||||
return HdrHistogramModule.HDRHISTOGRAM_TYPE_NAME;
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<String> requiredFields() {
|
||||
return Collections.singletonList(fieldName);
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public int getMaxIntermediateSize() {
|
||||
if(!autoResize){
|
||||
/*Histogram histogram = new Histogram(lowestDiscernibleValue, highestTrackableValue, numberOfSignificantValueDigits);
|
||||
histogram.setAutoResize(autoResize);
|
||||
return histogram.getNeededByteBufferCapacity();*/
|
||||
return HistogramSketch.getUpdatableSerializationBytes(lowestDiscernibleValue, highestTrackableValue, numberOfSignificantValueDigits);
|
||||
}else{
|
||||
//return (1 << 10) * 512;
|
||||
return HistogramSketch.getUpdatableSerializationBytes(lowestDiscernibleValue, BUFFER_AUTO_RESIZE_HIGHEST, numberOfSignificantValueDigits);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public byte[] getCacheKey() {
|
||||
return new CacheKeyBuilder(HdrHistogramModule.CACHE_TYPE_ID_OFFSET).appendByte(HdrHistogramModule.QUANTILES_HDRHISTOGRAM_BUILD_CACHE_TYPE_ID)
|
||||
.appendString(name).appendString(fieldName)
|
||||
.appendDouble(lowestDiscernibleValue).appendDouble(highestTrackableValue)
|
||||
.appendInt(numberOfSignificantValueDigits).appendBoolean(autoResize)
|
||||
.build();
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean equals(final Object o){
|
||||
if (this == o) {
|
||||
return true;
|
||||
}
|
||||
if (o == null || !getClass().equals(o.getClass())) {
|
||||
return false;
|
||||
}
|
||||
|
||||
HdrHistogramAggregatorFactory that = (HdrHistogramAggregatorFactory) o;
|
||||
return name.equals(that.name) && fieldName.equals(that.fieldName) &&
|
||||
lowestDiscernibleValue == that.lowestDiscernibleValue &&
|
||||
highestTrackableValue == that.highestTrackableValue &&
|
||||
numberOfSignificantValueDigits == that.numberOfSignificantValueDigits &&
|
||||
autoResize == that.autoResize
|
||||
;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int hashCode(){
|
||||
return Objects.hash(name, fieldName, lowestDiscernibleValue, highestTrackableValue, numberOfSignificantValueDigits, autoResize);
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return getClass().getSimpleName() + "{" +
|
||||
"name='" + name + '\'' +
|
||||
", fieldName='" + fieldName + '\'' +
|
||||
", lowestDiscernibleValue=" + lowestDiscernibleValue +
|
||||
", highestTrackableValue=" + highestTrackableValue +
|
||||
", numberOfSignificantValueDigits=" + numberOfSignificantValueDigits +
|
||||
", autoResize=" + autoResize +
|
||||
'}';
|
||||
}
|
||||
}
|
||||
package org.apache.druid.query.aggregation.sketch.HdrHistogram;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||
import org.HdrHistogram.HistogramSketch;
|
||||
import org.HdrHistogram.HistogramUnion;
|
||||
import org.apache.druid.java.util.common.IAE;
|
||||
import org.apache.druid.query.aggregation.*;
|
||||
import org.apache.druid.query.cache.CacheKeyBuilder;
|
||||
import org.apache.druid.segment.ColumnSelectorFactory;
|
||||
import org.apache.druid.segment.ColumnValueSelector;
|
||||
import org.apache.druid.segment.column.ColumnType;
|
||||
|
||||
import javax.annotation.Nullable;
|
||||
import java.util.Collections;
|
||||
import java.util.Comparator;
|
||||
import java.util.List;
|
||||
import java.util.Objects;
|
||||
|
||||
public class HdrHistogramAggregatorFactory extends AggregatorFactory {
|
||||
public static final long DEFAULT_LOWEST = 1;
|
||||
public static final long DEFAULT_HIGHEST = 2;
|
||||
public static final int DEFAULT_SIGNIFICANT = 1;
|
||||
public static final boolean DEFAULT_AUTO_RESIZE = true;
|
||||
public static final long BUFFER_AUTO_RESIZE_HIGHEST = 100000000L * 100L;
|
||||
public static final Comparator<HistogramSketch> COMPARATOR =
|
||||
Comparator.nullsFirst(Comparator.comparingLong(HistogramSketch::getTotalCount));
|
||||
|
||||
protected final String name;
|
||||
protected final String fieldName;
|
||||
protected final long lowestDiscernibleValue;
|
||||
protected final long highestTrackableValue;
|
||||
protected final int numberOfSignificantValueDigits;
|
||||
protected final boolean autoResize; //默认是false
|
||||
protected final int updatableSerializationBytes;
|
||||
|
||||
public HdrHistogramAggregatorFactory(
|
||||
@JsonProperty("name") String name,
|
||||
@JsonProperty("fieldName") String fieldName,
|
||||
@JsonProperty("lowestDiscernibleValue") @Nullable Long lowestDiscernibleValue,
|
||||
@JsonProperty("highestTrackableValue") @Nullable Long highestTrackableValue,
|
||||
@JsonProperty("numberOfSignificantValueDigits") @Nullable Integer numberOfSignificantValueDigits,
|
||||
@JsonProperty("autoResize") @Nullable Boolean autoResize
|
||||
) {
|
||||
if (name == null) {
|
||||
throw new IAE("Must have a valid, non-null aggregator name");
|
||||
}
|
||||
if (fieldName == null) {
|
||||
throw new IAE("Parameter fieldName must be specified");
|
||||
}
|
||||
|
||||
if(lowestDiscernibleValue == null){
|
||||
lowestDiscernibleValue = DEFAULT_LOWEST;
|
||||
}
|
||||
// Verify argument validity
|
||||
if (lowestDiscernibleValue < 1) {
|
||||
throw new IAE("lowestDiscernibleValue must be >= 1");
|
||||
}
|
||||
if (lowestDiscernibleValue > Long.MAX_VALUE / 2) {
|
||||
// prevent subsequent multiplication by 2 for highestTrackableValue check from overflowing
|
||||
throw new IAE("lowestDiscernibleValue must be <= Long.MAX_VALUE / 2");
|
||||
}
|
||||
if(highestTrackableValue == null){
|
||||
highestTrackableValue = DEFAULT_HIGHEST;
|
||||
}
|
||||
if (highestTrackableValue < 2L * lowestDiscernibleValue) {
|
||||
throw new IAE("highestTrackableValue must be >= 2 * lowestDiscernibleValue");
|
||||
}
|
||||
if(numberOfSignificantValueDigits == null){
|
||||
numberOfSignificantValueDigits = DEFAULT_SIGNIFICANT;
|
||||
}
|
||||
if ((numberOfSignificantValueDigits < 0) || (numberOfSignificantValueDigits > 5)) {
|
||||
throw new IAE("numberOfSignificantValueDigits must be between 0 and 5");
|
||||
}
|
||||
if(autoResize == null){
|
||||
autoResize = DEFAULT_AUTO_RESIZE;
|
||||
}
|
||||
|
||||
this.name = name;
|
||||
this.fieldName = fieldName;
|
||||
this.lowestDiscernibleValue = lowestDiscernibleValue;
|
||||
this.highestTrackableValue = highestTrackableValue;
|
||||
this.numberOfSignificantValueDigits = numberOfSignificantValueDigits;
|
||||
this.autoResize = autoResize;
|
||||
this.updatableSerializationBytes = getUpdatableSerializationBytes();
|
||||
}
|
||||
|
||||
@Override
|
||||
public Aggregator factorize(ColumnSelectorFactory metricFactory) {
|
||||
return new HdrHistogramAggregator(
|
||||
metricFactory.makeColumnValueSelector(fieldName),
|
||||
lowestDiscernibleValue,
|
||||
highestTrackableValue,
|
||||
numberOfSignificantValueDigits,
|
||||
autoResize
|
||||
);
|
||||
}
|
||||
|
||||
@Override
|
||||
public BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory) {
|
||||
return new HdrHistogramBufferAggregator(
|
||||
metricFactory.makeColumnValueSelector(fieldName),
|
||||
lowestDiscernibleValue,
|
||||
highestTrackableValue,
|
||||
numberOfSignificantValueDigits,
|
||||
autoResize,
|
||||
getMaxIntermediateSize()
|
||||
);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Comparator getComparator() {
|
||||
return COMPARATOR;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Object combine(Object lhs, Object rhs) {
|
||||
if(lhs == null){
|
||||
return rhs;
|
||||
}else if(rhs == null){
|
||||
return lhs;
|
||||
}else{
|
||||
final HistogramUnion union = new HistogramUnion(lowestDiscernibleValue,highestTrackableValue,numberOfSignificantValueDigits,autoResize);
|
||||
union.update((HistogramSketch) lhs);
|
||||
union.update((HistogramSketch) rhs);
|
||||
HistogramSketch result = union.getResult();
|
||||
return result;
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public AggregateCombiner makeAggregateCombiner() {
|
||||
return new ObjectAggregateCombiner<HistogramSketch>() {
|
||||
private HistogramUnion union = null;
|
||||
|
||||
@Override
|
||||
public void reset(ColumnValueSelector selector) {
|
||||
//union.reset();
|
||||
union = null;
|
||||
fold(selector);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void fold(ColumnValueSelector selector) {
|
||||
HistogramSketch h = (HistogramSketch) selector.getObject();
|
||||
if(h != null){
|
||||
if(union == null){
|
||||
union = new HistogramUnion(lowestDiscernibleValue,highestTrackableValue,numberOfSignificantValueDigits,autoResize);
|
||||
}
|
||||
union.update(h);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public Class<HistogramSketch> classOfObject() {
|
||||
return HistogramSketch.class;
|
||||
}
|
||||
|
||||
@Nullable
|
||||
@Override
|
||||
public HistogramSketch getObject() {
|
||||
if(union == null){
|
||||
return null;
|
||||
}else{
|
||||
HistogramSketch result = union.getResult();
|
||||
/*if(result.getTotalCount() == 0){
|
||||
return null;
|
||||
}*/
|
||||
return result;
|
||||
}
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
/*public Histogram geneHistogram() {
|
||||
Histogram histogram = new Histogram(lowestDiscernibleValue, highestTrackableValue, numberOfSignificantValueDigits);
|
||||
histogram.setAutoResize(autoResize);
|
||||
return histogram;
|
||||
}*/
|
||||
|
||||
@Override
|
||||
public AggregatorFactory getCombiningFactory() {
|
||||
return new HdrHistogramMergeAggregatorFactory(name, name, lowestDiscernibleValue, highestTrackableValue, numberOfSignificantValueDigits, autoResize);
|
||||
}
|
||||
|
||||
@Override
|
||||
public AggregatorFactory getMergingFactory(AggregatorFactory other) throws AggregatorFactoryNotMergeableException {
|
||||
if (other.getName().equals(this.getName()) && other instanceof HdrHistogramAggregatorFactory) {
|
||||
HdrHistogramAggregatorFactory castedOther = (HdrHistogramAggregatorFactory) other;
|
||||
|
||||
return new HdrHistogramMergeAggregatorFactory(name, name,
|
||||
Math.min(lowestDiscernibleValue, castedOther.lowestDiscernibleValue),
|
||||
Math.max(highestTrackableValue, castedOther.highestTrackableValue),
|
||||
Math.max(numberOfSignificantValueDigits, castedOther.numberOfSignificantValueDigits),
|
||||
autoResize || castedOther.autoResize
|
||||
);
|
||||
} else {
|
||||
throw new AggregatorFactoryNotMergeableException(this, other);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<AggregatorFactory> getRequiredColumns() {
|
||||
return Collections.singletonList(
|
||||
new HdrHistogramAggregatorFactory(
|
||||
fieldName,
|
||||
fieldName,
|
||||
lowestDiscernibleValue, highestTrackableValue, numberOfSignificantValueDigits, autoResize
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
@Override
|
||||
public AggregatorFactory withName(String newName) {
|
||||
return new HdrHistogramAggregatorFactory(newName, fieldName, lowestDiscernibleValue, highestTrackableValue, numberOfSignificantValueDigits, autoResize);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Object deserialize(Object object) {
|
||||
if (object == null) {
|
||||
return null;
|
||||
}
|
||||
return HistogramUtils.deserializeHistogram(object);
|
||||
}
|
||||
|
||||
@Override
|
||||
public ColumnType getResultType() {
|
||||
//return ColumnType.LONG;
|
||||
return getIntermediateType();
|
||||
}
|
||||
|
||||
@Nullable
|
||||
@Override
|
||||
public Object finalizeComputation(@Nullable Object object) {
|
||||
//return object == null ? null : ((HistogramSketch) object).getTotalCount();
|
||||
return object;
|
||||
}
|
||||
|
||||
@Override
|
||||
@JsonProperty
|
||||
public String getName() {
|
||||
return name;
|
||||
}
|
||||
|
||||
@JsonProperty
|
||||
public String getFieldName() {
|
||||
return fieldName;
|
||||
}
|
||||
|
||||
@JsonProperty
|
||||
public long getLowestDiscernibleValue() {
|
||||
return lowestDiscernibleValue;
|
||||
}
|
||||
|
||||
@JsonProperty
|
||||
public long getHighestTrackableValue() {
|
||||
return highestTrackableValue;
|
||||
}
|
||||
|
||||
@JsonProperty
|
||||
public int getNumberOfSignificantValueDigits() {
|
||||
return numberOfSignificantValueDigits;
|
||||
}
|
||||
|
||||
@JsonProperty
|
||||
public boolean isAutoResize() {
|
||||
return autoResize;
|
||||
}
|
||||
|
||||
/*
|
||||
没这个方法了, 新版本需要实现getIntermediateType方法
|
||||
@Override
|
||||
public String getTypeName() {
|
||||
return HdrHistogramModule.HDRHISTOGRAM_TYPE_NAME;
|
||||
}*/
|
||||
|
||||
@Override
|
||||
public ColumnType getIntermediateType() {
|
||||
return HdrHistogramModule.TYPE;
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<String> requiredFields() {
|
||||
return Collections.singletonList(fieldName);
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public int getMaxIntermediateSize() {
|
||||
return updatableSerializationBytes == 0? getUpdatableSerializationBytes():updatableSerializationBytes;
|
||||
}
|
||||
|
||||
private int getUpdatableSerializationBytes(){
|
||||
if(!autoResize){
|
||||
/*Histogram histogram = new Histogram(lowestDiscernibleValue, highestTrackableValue, numberOfSignificantValueDigits);
|
||||
histogram.setAutoResize(autoResize);
|
||||
return histogram.getNeededByteBufferCapacity();*/
|
||||
return HistogramSketch.getUpdatableSerializationBytes(lowestDiscernibleValue, highestTrackableValue, numberOfSignificantValueDigits);
|
||||
}else{
|
||||
//return (1 << 10) * 512;
|
||||
return HistogramSketch.getUpdatableSerializationBytes(lowestDiscernibleValue, BUFFER_AUTO_RESIZE_HIGHEST, numberOfSignificantValueDigits);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public byte[] getCacheKey() {
|
||||
return new CacheKeyBuilder(HdrHistogramModule.CACHE_TYPE_ID_OFFSET).appendByte(HdrHistogramModule.QUANTILES_HDRHISTOGRAM_BUILD_CACHE_TYPE_ID)
|
||||
.appendString(name).appendString(fieldName)
|
||||
.appendDouble(lowestDiscernibleValue).appendDouble(highestTrackableValue)
|
||||
.appendInt(numberOfSignificantValueDigits).appendBoolean(autoResize)
|
||||
.build();
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean equals(final Object o){
|
||||
if (this == o) {
|
||||
return true;
|
||||
}
|
||||
if (o == null || !getClass().equals(o.getClass())) {
|
||||
return false;
|
||||
}
|
||||
|
||||
HdrHistogramAggregatorFactory that = (HdrHistogramAggregatorFactory) o;
|
||||
return name.equals(that.name) && fieldName.equals(that.fieldName) &&
|
||||
lowestDiscernibleValue == that.lowestDiscernibleValue &&
|
||||
highestTrackableValue == that.highestTrackableValue &&
|
||||
numberOfSignificantValueDigits == that.numberOfSignificantValueDigits &&
|
||||
autoResize == that.autoResize
|
||||
;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int hashCode(){
|
||||
return Objects.hash(name, fieldName, lowestDiscernibleValue, highestTrackableValue, numberOfSignificantValueDigits, autoResize);
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return getClass().getSimpleName() + "{" +
|
||||
"name='" + name + '\'' +
|
||||
", fieldName='" + fieldName + '\'' +
|
||||
", lowestDiscernibleValue=" + lowestDiscernibleValue +
|
||||
", highestTrackableValue=" + highestTrackableValue +
|
||||
", numberOfSignificantValueDigits=" + numberOfSignificantValueDigits +
|
||||
", autoResize=" + autoResize +
|
||||
'}';
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,9 +1,9 @@
|
||||
package org.apache.druid.query.aggregation.sketch.HdrHistogram;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||
import org.HdrHistogram.Histogram;
|
||||
import org.HdrHistogram.HistogramSketch;
|
||||
import org.apache.druid.query.aggregation.Aggregator;
|
||||
import org.apache.druid.query.aggregation.AggregatorFactory;
|
||||
import org.apache.druid.query.aggregation.BufferAggregator;
|
||||
import org.apache.druid.query.cache.CacheKeyBuilder;
|
||||
import org.apache.druid.segment.ColumnSelectorFactory;
|
||||
@@ -48,6 +48,11 @@ public class HdrHistogramMergeAggregatorFactory extends HdrHistogramAggregatorFa
|
||||
);
|
||||
}
|
||||
|
||||
@Override
|
||||
public AggregatorFactory withName(String newName) {
|
||||
return new HdrHistogramMergeAggregatorFactory(newName, fieldName, lowestDiscernibleValue, highestTrackableValue, numberOfSignificantValueDigits, autoResize);
|
||||
}
|
||||
|
||||
@Override
|
||||
public byte[] getCacheKey() {
|
||||
return new CacheKeyBuilder(HdrHistogramModule.CACHE_TYPE_ID_OFFSET).appendByte(HdrHistogramModule.QUANTILES_HDRHISTOGRAM_MERGE_CACHE_TYPE_ID)
|
||||
|
||||
@@ -1,134 +1,134 @@
|
||||
package org.apache.druid.query.aggregation.sketch.HdrHistogram;
|
||||
|
||||
import it.unimi.dsi.fastutil.ints.Int2ObjectMap;
|
||||
import it.unimi.dsi.fastutil.ints.Int2ObjectOpenHashMap;
|
||||
import org.HdrHistogram.*;
|
||||
import org.apache.druid.java.util.common.logger.Logger;
|
||||
import org.apache.druid.query.aggregation.BufferAggregator;
|
||||
import org.apache.druid.query.monomorphicprocessing.RuntimeShapeInspector;
|
||||
import org.apache.druid.segment.BaseObjectColumnValueSelector;
|
||||
|
||||
import javax.annotation.Nullable;
|
||||
import java.nio.ByteBuffer;
|
||||
import java.util.IdentityHashMap;
|
||||
|
||||
public class HdrHistogramMergeBufferAggregator implements BufferAggregator {
|
||||
private static final Logger LOG = new Logger(HdrHistogramAggregator.class);
|
||||
private long lastTs = 0L;
|
||||
private final BaseObjectColumnValueSelector<HistogramSketch> selector;
|
||||
private final long lowestDiscernibleValue;
|
||||
private final long highestTrackableValue;
|
||||
private final int numberOfSignificantValueDigits;
|
||||
private final boolean autoResize;
|
||||
private final int size;
|
||||
private final IdentityHashMap<ByteBuffer, Int2ObjectMap<HistogramUnion>> histograms = new IdentityHashMap<>();
|
||||
|
||||
public HdrHistogramMergeBufferAggregator(
|
||||
BaseObjectColumnValueSelector<HistogramSketch> selector,
|
||||
long lowestDiscernibleValue,
|
||||
long highestTrackableValue,
|
||||
int numberOfSignificantValueDigits,
|
||||
boolean autoResize,
|
||||
int size
|
||||
) {
|
||||
this.selector = selector;
|
||||
this.lowestDiscernibleValue = lowestDiscernibleValue;
|
||||
this.highestTrackableValue = highestTrackableValue;
|
||||
this.numberOfSignificantValueDigits = numberOfSignificantValueDigits;
|
||||
this.autoResize = autoResize;
|
||||
this.size = size;
|
||||
LOG.error("HdrHistogramMergeBufferAggregator gene:" + Thread.currentThread().getName() + "-" + Thread.currentThread().getId());
|
||||
}
|
||||
|
||||
@Override
|
||||
public synchronized void init(ByteBuffer buf, int position) {
|
||||
final int oldPosition = buf.position();
|
||||
try {
|
||||
buf.position(position);
|
||||
|
||||
long highest = autoResize?HdrHistogramAggregatorFactory.BUFFER_AUTO_RESIZE_HIGHEST: highestTrackableValue;
|
||||
final DirectArrayHistogram histogram = new DirectArrayHistogram(lowestDiscernibleValue, highest, numberOfSignificantValueDigits, buf);
|
||||
histogram.reset();
|
||||
HistogramUnion union = new HistogramUnion(new HistogramSketch(histogram));
|
||||
putUnion(buf, position, union);
|
||||
}finally {
|
||||
buf.position(oldPosition);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public synchronized void aggregate(ByteBuffer buf, int position) {
|
||||
/*long ts = System.currentTimeMillis();
|
||||
if(ts - lastTs > 2000){
|
||||
//LOG.warn("HdrHistogramMergeBufferAggregator call");
|
||||
LOG.error("HdrHistogramMergeBufferAggregator call");
|
||||
lastTs = ts;
|
||||
}*/
|
||||
HistogramSketch h = selector.getObject();
|
||||
if (h == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
final int oldPosition = buf.position();
|
||||
try {
|
||||
buf.position(position);
|
||||
|
||||
HistogramUnion union = histograms.get(buf).get(position);
|
||||
union.update(h);
|
||||
}finally{
|
||||
buf.position(oldPosition);
|
||||
}
|
||||
}
|
||||
|
||||
@Nullable
|
||||
@Override
|
||||
public synchronized HistogramSketch get(ByteBuffer buf, int position) {
|
||||
LOG.error("HdrHistogramMergeBufferAggregator get:" + 0 + "-" + Thread.currentThread().getId() + "-" + this);
|
||||
HistogramUnion union = histograms.get(buf).get(position);
|
||||
//return histogram.copy();
|
||||
return union.getResult().copy();
|
||||
}
|
||||
|
||||
@Override
|
||||
public synchronized void relocate(int oldPosition, int newPosition, ByteBuffer oldBuffer, ByteBuffer newBuffer) {
|
||||
HistogramUnion union = histograms.get(oldBuffer).get(oldPosition);
|
||||
|
||||
Int2ObjectMap<HistogramUnion> map = histograms.get(oldBuffer);
|
||||
map.remove(oldPosition);
|
||||
if (map.isEmpty()) {
|
||||
histograms.remove(oldBuffer);
|
||||
}
|
||||
|
||||
try {
|
||||
newBuffer.position(newPosition);
|
||||
union.resetByteBuffer(newBuffer);
|
||||
putUnion(newBuffer, newPosition, union);
|
||||
}finally {
|
||||
newBuffer.position(newPosition);
|
||||
}
|
||||
}
|
||||
|
||||
private void putUnion(final ByteBuffer buffer, final int position, final HistogramUnion union) {
|
||||
Int2ObjectMap<HistogramUnion> map = histograms.computeIfAbsent(buffer, buf -> new Int2ObjectOpenHashMap<>());
|
||||
map.put(position, union);
|
||||
}
|
||||
@Override
|
||||
public float getFloat(ByteBuffer buf, int position) {
|
||||
throw new UnsupportedOperationException("Not implemented");
|
||||
}
|
||||
|
||||
@Override
|
||||
public long getLong(ByteBuffer buf, int position) {
|
||||
throw new UnsupportedOperationException("Not implemented");
|
||||
}
|
||||
|
||||
@Override
|
||||
public void close() {
|
||||
|
||||
}
|
||||
|
||||
@Override
|
||||
public void inspectRuntimeShape(RuntimeShapeInspector inspector){
|
||||
inspector.visit("selector", selector);
|
||||
}
|
||||
}
|
||||
package org.apache.druid.query.aggregation.sketch.HdrHistogram;
|
||||
|
||||
import it.unimi.dsi.fastutil.ints.Int2ObjectMap;
|
||||
import it.unimi.dsi.fastutil.ints.Int2ObjectOpenHashMap;
|
||||
import org.HdrHistogram.*;
|
||||
import org.apache.druid.java.util.common.logger.Logger;
|
||||
import org.apache.druid.query.aggregation.BufferAggregator;
|
||||
import org.apache.druid.query.monomorphicprocessing.RuntimeShapeInspector;
|
||||
import org.apache.druid.segment.BaseObjectColumnValueSelector;
|
||||
|
||||
import javax.annotation.Nullable;
|
||||
import java.nio.ByteBuffer;
|
||||
import java.util.IdentityHashMap;
|
||||
|
||||
public class HdrHistogramMergeBufferAggregator implements BufferAggregator {
|
||||
private static final Logger LOG = new Logger(HdrHistogramAggregator.class);
|
||||
private long lastTs = 0L;
|
||||
private final BaseObjectColumnValueSelector<HistogramSketch> selector;
|
||||
private final long lowestDiscernibleValue;
|
||||
private final long highestTrackableValue;
|
||||
private final int numberOfSignificantValueDigits;
|
||||
private final boolean autoResize;
|
||||
private final int size;
|
||||
private final IdentityHashMap<ByteBuffer, Int2ObjectMap<HistogramUnion>> histograms = new IdentityHashMap<>();
|
||||
|
||||
public HdrHistogramMergeBufferAggregator(
|
||||
BaseObjectColumnValueSelector<HistogramSketch> selector,
|
||||
long lowestDiscernibleValue,
|
||||
long highestTrackableValue,
|
||||
int numberOfSignificantValueDigits,
|
||||
boolean autoResize,
|
||||
int size
|
||||
) {
|
||||
this.selector = selector;
|
||||
this.lowestDiscernibleValue = lowestDiscernibleValue;
|
||||
this.highestTrackableValue = highestTrackableValue;
|
||||
this.numberOfSignificantValueDigits = numberOfSignificantValueDigits;
|
||||
this.autoResize = autoResize;
|
||||
this.size = size;
|
||||
//LOG.error("HdrHistogramMergeBufferAggregator gene:" + Thread.currentThread().getName() + "-" + Thread.currentThread().getId());
|
||||
}
|
||||
|
||||
@Override
|
||||
public synchronized void init(ByteBuffer buf, int position) {
|
||||
final int oldPosition = buf.position();
|
||||
try {
|
||||
buf.position(position);
|
||||
|
||||
long highest = autoResize?HdrHistogramAggregatorFactory.BUFFER_AUTO_RESIZE_HIGHEST: highestTrackableValue;
|
||||
final DirectArrayHistogram histogram = new DirectArrayHistogram(lowestDiscernibleValue, highest, numberOfSignificantValueDigits, buf);
|
||||
histogram.reset();
|
||||
HistogramUnion union = new HistogramUnion(new HistogramSketch(histogram));
|
||||
putUnion(buf, position, union);
|
||||
}finally {
|
||||
buf.position(oldPosition);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public synchronized void aggregate(ByteBuffer buf, int position) {
|
||||
/*long ts = System.currentTimeMillis();
|
||||
if(ts - lastTs > 2000){
|
||||
//LOG.warn("HdrHistogramMergeBufferAggregator call");
|
||||
LOG.error("HdrHistogramMergeBufferAggregator call");
|
||||
lastTs = ts;
|
||||
}*/
|
||||
HistogramSketch h = selector.getObject();
|
||||
if (h == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
final int oldPosition = buf.position();
|
||||
try {
|
||||
buf.position(position);
|
||||
|
||||
HistogramUnion union = histograms.get(buf).get(position);
|
||||
union.update(h);
|
||||
}finally{
|
||||
buf.position(oldPosition);
|
||||
}
|
||||
}
|
||||
|
||||
@Nullable
|
||||
@Override
|
||||
public synchronized HistogramSketch get(ByteBuffer buf, int position) {
|
||||
//LOG.error("HdrHistogramMergeBufferAggregator get:" + 0 + "-" + Thread.currentThread().getId() + "-" + this);
|
||||
HistogramUnion union = histograms.get(buf).get(position);
|
||||
//return histogram.copy();
|
||||
return union.getResult().copy();
|
||||
}
|
||||
|
||||
@Override
|
||||
public synchronized void relocate(int oldPosition, int newPosition, ByteBuffer oldBuffer, ByteBuffer newBuffer) {
|
||||
HistogramUnion union = histograms.get(oldBuffer).get(oldPosition);
|
||||
|
||||
Int2ObjectMap<HistogramUnion> map = histograms.get(oldBuffer);
|
||||
map.remove(oldPosition);
|
||||
if (map.isEmpty()) {
|
||||
histograms.remove(oldBuffer);
|
||||
}
|
||||
|
||||
try {
|
||||
newBuffer.position(newPosition);
|
||||
union.resetByteBuffer(newBuffer);
|
||||
putUnion(newBuffer, newPosition, union);
|
||||
}finally {
|
||||
newBuffer.position(newPosition);
|
||||
}
|
||||
}
|
||||
|
||||
private void putUnion(final ByteBuffer buffer, final int position, final HistogramUnion union) {
|
||||
Int2ObjectMap<HistogramUnion> map = histograms.computeIfAbsent(buffer, buf -> new Int2ObjectOpenHashMap<>());
|
||||
map.put(position, union);
|
||||
}
|
||||
@Override
|
||||
public float getFloat(ByteBuffer buf, int position) {
|
||||
throw new UnsupportedOperationException("Not implemented");
|
||||
}
|
||||
|
||||
@Override
|
||||
public long getLong(ByteBuffer buf, int position) {
|
||||
throw new UnsupportedOperationException("Not implemented");
|
||||
}
|
||||
|
||||
@Override
|
||||
public void close() {
|
||||
|
||||
}
|
||||
|
||||
@Override
|
||||
public void inspectRuntimeShape(RuntimeShapeInspector inspector){
|
||||
inspector.visit("selector", selector);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -7,13 +7,10 @@ import com.fasterxml.jackson.databind.jsontype.NamedType;
|
||||
import com.fasterxml.jackson.databind.module.SimpleModule;
|
||||
import com.google.common.annotations.VisibleForTesting;
|
||||
import com.google.inject.Binder;
|
||||
import org.HdrHistogram.Histogram;
|
||||
import org.HdrHistogram.HistogramSketch;
|
||||
import org.apache.druid.initialization.DruidModule;
|
||||
import org.apache.druid.query.aggregation.sketch.HdrHistogram.sql.HdrHistogramObjectSqlAggregator;
|
||||
import org.apache.druid.query.aggregation.sketch.HdrHistogram.sql.HdrHistogramPercentilesOperatorConversion;
|
||||
import org.apache.druid.query.aggregation.sketch.HdrHistogram.sql.HdrHistogramQuantileSqlAggregator;
|
||||
import org.apache.druid.query.aggregation.sketch.HdrHistogram.sql.HdrHistogramQuantilesOperatorConversion;
|
||||
import org.apache.druid.query.aggregation.sketch.HdrHistogram.sql.*;
|
||||
import org.apache.druid.segment.column.ColumnType;
|
||||
import org.apache.druid.segment.serde.ComplexMetrics;
|
||||
import org.apache.druid.sql.guice.SqlBindings;
|
||||
|
||||
@@ -27,8 +24,11 @@ public class HdrHistogramModule implements DruidModule {
|
||||
public static final byte QUANTILES_HDRHISTOGRAM_TO_QUANTILE_CACHE_TYPE_ID = 0x03;
|
||||
public static final byte QUANTILES_HDRHISTOGRAM_TO_QUANTILES_CACHE_TYPE_ID = 0x04;
|
||||
public static final byte QUANTILES_HDRHISTOGRAM_TO_PERCENTILES_CACHE_TYPE_ID = 0x05;
|
||||
public static final byte QUANTILES_HDRHISTOGRAM_TO_DESCRIBE_CACHE_TYPE_ID = 0x06;
|
||||
public static final byte QUANTILES_HDRHISTOGRAM_TO_PERCENTILES_DESCRIBE_CACHE_TYPE_ID = 0x07;
|
||||
|
||||
public static final String HDRHISTOGRAM_TYPE_NAME = "HdrHistogramSketch";
|
||||
public static final ColumnType TYPE = ColumnType.ofComplex(HDRHISTOGRAM_TYPE_NAME);
|
||||
|
||||
public static final ObjectMapper objectMapper = new ObjectMapper();
|
||||
|
||||
@@ -49,6 +49,8 @@ public class HdrHistogramModule implements DruidModule {
|
||||
|
||||
SqlBindings.addOperatorConversion(binder, HdrHistogramQuantilesOperatorConversion.class);
|
||||
SqlBindings.addOperatorConversion(binder, HdrHistogramPercentilesOperatorConversion.class);
|
||||
SqlBindings.addOperatorConversion(binder, HdrHistogramDescribeOperatorConversion.class);
|
||||
SqlBindings.addOperatorConversion(binder, HdrHistogramPercentilesDescribeOperatorConversion.class);
|
||||
}
|
||||
|
||||
@Override
|
||||
@@ -60,7 +62,9 @@ public class HdrHistogramModule implements DruidModule {
|
||||
new NamedType(HdrHistogramMergeAggregatorFactory.class, "HdrHistogramSketchMerge"),
|
||||
new NamedType(HdrHistogramToQuantilePostAggregator.class, "HdrHistogramSketchToQuantile"),
|
||||
new NamedType(HdrHistogramToQuantilesPostAggregator.class, "HdrHistogramSketchToQuantiles"),
|
||||
new NamedType(HdrHistogramToPercentilesPostAggregator.class, "HdrHistogramSketchToPercentiles")
|
||||
new NamedType(HdrHistogramToPercentilesPostAggregator.class, "HdrHistogramSketchToPercentiles"),
|
||||
new NamedType(HdrHistogramToDescribePostAggregator.class, "HdrHistogramSketchToDescribe"),
|
||||
new NamedType(HdrHistogramToPercentilesDescribePostAggregator.class, "HdrHistogramSketchToPercentilesDescription")
|
||||
).addSerializer(HistogramSketch.class, new HistogramJsonSerializer())
|
||||
);
|
||||
}
|
||||
|
||||
@@ -0,0 +1,108 @@
|
||||
package org.apache.druid.query.aggregation.sketch.HdrHistogram;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonCreator;
|
||||
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||
import com.google.common.collect.Sets;
|
||||
import org.HdrHistogram.HistogramSketch;
|
||||
import org.apache.druid.java.util.common.IAE;
|
||||
import org.apache.druid.query.aggregation.AggregatorFactory;
|
||||
import org.apache.druid.query.aggregation.PostAggregator;
|
||||
import org.apache.druid.query.cache.CacheKeyBuilder;
|
||||
import org.apache.druid.segment.ColumnInspector;
|
||||
import org.apache.druid.segment.column.ColumnType;
|
||||
|
||||
import javax.annotation.Nullable;
|
||||
import java.util.*;
|
||||
|
||||
public class HdrHistogramToDescribePostAggregator implements PostAggregator {
|
||||
private final String name;
|
||||
private final String fieldName;
|
||||
|
||||
@JsonCreator
|
||||
public HdrHistogramToDescribePostAggregator(
|
||||
@JsonProperty("name") String name,
|
||||
@JsonProperty("fieldName") String fieldName
|
||||
){
|
||||
this.name = name;
|
||||
this.fieldName = fieldName;
|
||||
}
|
||||
|
||||
@Override
|
||||
public ColumnType getType(ColumnInspector signature){
|
||||
return ColumnType.STRING;
|
||||
}
|
||||
|
||||
@Override
|
||||
@JsonProperty
|
||||
public String getName() {
|
||||
return name;
|
||||
}
|
||||
|
||||
@JsonProperty
|
||||
public String getFieldName() {
|
||||
return fieldName;
|
||||
}
|
||||
|
||||
@Nullable
|
||||
@Override
|
||||
public Object compute(Map<String, Object> values) {
|
||||
HistogramSketch histogram = (HistogramSketch) values.get(fieldName);
|
||||
if(histogram == null){
|
||||
return "{}"; //"[]"
|
||||
}
|
||||
return HdrHistogramModule.toJson(histogram.describe());
|
||||
}
|
||||
|
||||
@Override
|
||||
public Comparator<double[]> getComparator()
|
||||
{
|
||||
throw new IAE("Comparing arrays of quantiles is not supported");
|
||||
}
|
||||
|
||||
@Override
|
||||
public Set<String> getDependentFields()
|
||||
{
|
||||
return Sets.newHashSet(fieldName);
|
||||
}
|
||||
|
||||
@Override
|
||||
public PostAggregator decorate(Map<String, AggregatorFactory> aggregators) {
|
||||
return this;
|
||||
}
|
||||
|
||||
@Override
|
||||
public byte[] getCacheKey() {
|
||||
CacheKeyBuilder builder = new CacheKeyBuilder(HdrHistogramModule.CACHE_TYPE_ID_OFFSET).appendByte(HdrHistogramModule.QUANTILES_HDRHISTOGRAM_TO_DESCRIBE_CACHE_TYPE_ID)
|
||||
.appendString(fieldName);
|
||||
return builder.build();
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean equals(Object o) {
|
||||
if (this == o) {
|
||||
return true;
|
||||
}
|
||||
if (o == null || getClass() != o.getClass()) {
|
||||
return false;
|
||||
}
|
||||
HdrHistogramToDescribePostAggregator that = (HdrHistogramToDescribePostAggregator) o;
|
||||
|
||||
return name.equals(that.name) &&
|
||||
fieldName.equals(that.fieldName);
|
||||
}
|
||||
|
||||
@Override
|
||||
public int hashCode() {
|
||||
return Objects.hash(name, fieldName);
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return "HdrHistogramToDescribePostAggregator{" +
|
||||
"name='" + name + '\'' +
|
||||
", fieldName='" + fieldName + '\'' +
|
||||
'}';
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
@@ -0,0 +1,125 @@
|
||||
package org.apache.druid.query.aggregation.sketch.HdrHistogram;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonCreator;
|
||||
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||
import com.google.common.collect.Sets;
|
||||
import org.HdrHistogram.HistogramSketch;
|
||||
import org.HdrHistogram.Percentile;
|
||||
import org.apache.druid.java.util.common.IAE;
|
||||
import org.apache.druid.query.aggregation.AggregatorFactory;
|
||||
import org.apache.druid.query.aggregation.PostAggregator;
|
||||
import org.apache.druid.query.cache.CacheKeyBuilder;
|
||||
import org.apache.druid.segment.ColumnInspector;
|
||||
import org.apache.druid.segment.column.ColumnType;
|
||||
|
||||
import javax.annotation.Nullable;
|
||||
import java.util.*;
|
||||
|
||||
public class HdrHistogramToPercentilesDescribePostAggregator implements PostAggregator {
|
||||
private final String name;
|
||||
private final String fieldName;
|
||||
private final int percentileTicksPerHalfDistance;
|
||||
|
||||
@JsonCreator
|
||||
public HdrHistogramToPercentilesDescribePostAggregator(
|
||||
@JsonProperty("name") String name,
|
||||
@JsonProperty("fieldName") String fieldName,
|
||||
@JsonProperty("percentileTicksPerHalfDistance") int percentileTicksPerHalfDistance
|
||||
){
|
||||
this.name = name;
|
||||
this.fieldName = fieldName;
|
||||
this.percentileTicksPerHalfDistance = percentileTicksPerHalfDistance;
|
||||
}
|
||||
|
||||
@Override
|
||||
public ColumnType getType(ColumnInspector signature){
|
||||
return ColumnType.STRING;
|
||||
}
|
||||
|
||||
@Override
|
||||
@JsonProperty
|
||||
public String getName() {
|
||||
return name;
|
||||
}
|
||||
|
||||
@JsonProperty
|
||||
public String getFieldName() {
|
||||
return fieldName;
|
||||
}
|
||||
|
||||
@JsonProperty
|
||||
public int getPercentileTicksPerHalfDistance() {
|
||||
return percentileTicksPerHalfDistance;
|
||||
}
|
||||
|
||||
@Nullable
|
||||
@Override
|
||||
public Object compute(Map<String, Object> values) {
|
||||
HistogramSketch histogram = (HistogramSketch) values.get(fieldName);
|
||||
if(histogram == null){
|
||||
return "{\"percentiles\":[],\"describe\":{}}";
|
||||
}
|
||||
List<Percentile> percentiles = histogram.percentileList(percentileTicksPerHalfDistance);
|
||||
Map<String, Object> describe = histogram.describe();
|
||||
Map<String, Object> rst = new LinkedHashMap<>();
|
||||
rst.put("percentiles", percentiles);
|
||||
rst.put("description", describe);
|
||||
return HdrHistogramModule.toJson(rst);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Comparator<double[]> getComparator()
|
||||
{
|
||||
throw new IAE("Comparing object is not supported");
|
||||
}
|
||||
|
||||
@Override
|
||||
public Set<String> getDependentFields()
|
||||
{
|
||||
return Sets.newHashSet(fieldName);
|
||||
}
|
||||
|
||||
@Override
|
||||
public PostAggregator decorate(Map<String, AggregatorFactory> aggregators) {
|
||||
return this;
|
||||
}
|
||||
|
||||
@Override
|
||||
public byte[] getCacheKey() {
|
||||
CacheKeyBuilder builder = new CacheKeyBuilder(HdrHistogramModule.CACHE_TYPE_ID_OFFSET).appendByte(HdrHistogramModule.QUANTILES_HDRHISTOGRAM_TO_PERCENTILES_DESCRIBE_CACHE_TYPE_ID)
|
||||
.appendString(fieldName);
|
||||
builder.appendInt(percentileTicksPerHalfDistance);
|
||||
return builder.build();
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean equals(Object o) {
|
||||
if (this == o) {
|
||||
return true;
|
||||
}
|
||||
if (o == null || getClass() != o.getClass()) {
|
||||
return false;
|
||||
}
|
||||
HdrHistogramToPercentilesDescribePostAggregator that = (HdrHistogramToPercentilesDescribePostAggregator) o;
|
||||
|
||||
return percentileTicksPerHalfDistance == that.percentileTicksPerHalfDistance &&
|
||||
name.equals(that.name) &&
|
||||
fieldName.equals(that.fieldName);
|
||||
}
|
||||
|
||||
@Override
|
||||
public int hashCode() {
|
||||
return Objects.hash(name, fieldName, percentileTicksPerHalfDistance);
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return "HdrHistogramToPercentilesDescribePostAggregator{" +
|
||||
"name='" + name + '\'' +
|
||||
", fieldName='" + fieldName + '\'' +
|
||||
", probabilitys=" + percentileTicksPerHalfDistance +
|
||||
'}';
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
@@ -1,111 +1,121 @@
|
||||
package org.apache.druid.query.aggregation.sketch.HdrHistogram;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonCreator;
|
||||
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||
import com.google.common.collect.Sets;
|
||||
import org.HdrHistogram.HistogramSketch;
|
||||
import org.HdrHistogram.Percentile;
|
||||
import org.apache.druid.java.util.common.IAE;
|
||||
import org.apache.druid.query.aggregation.AggregatorFactory;
|
||||
import org.apache.druid.query.aggregation.PostAggregator;
|
||||
import org.apache.druid.query.cache.CacheKeyBuilder;
|
||||
|
||||
import javax.annotation.Nullable;
|
||||
import java.util.*;
|
||||
|
||||
public class HdrHistogramToPercentilesPostAggregator implements PostAggregator {
|
||||
private final String name;
|
||||
private final String fieldName;
|
||||
private final int percentileTicksPerHalfDistance;
|
||||
|
||||
@JsonCreator
|
||||
public HdrHistogramToPercentilesPostAggregator(
|
||||
@JsonProperty("name") String name,
|
||||
@JsonProperty("fieldName") String fieldName,
|
||||
@JsonProperty("percentileTicksPerHalfDistance") int percentileTicksPerHalfDistance
|
||||
){
|
||||
this.name = name;
|
||||
this.fieldName = fieldName;
|
||||
this.percentileTicksPerHalfDistance = percentileTicksPerHalfDistance;
|
||||
}
|
||||
|
||||
@Override
|
||||
@JsonProperty
|
||||
public String getName() {
|
||||
return name;
|
||||
}
|
||||
|
||||
@JsonProperty
|
||||
public String getFieldName() {
|
||||
return fieldName;
|
||||
}
|
||||
|
||||
@JsonProperty
|
||||
public int getPercentileTicksPerHalfDistance() {
|
||||
return percentileTicksPerHalfDistance;
|
||||
}
|
||||
|
||||
@Nullable
|
||||
@Override
|
||||
public Object compute(Map<String, Object> values) {
|
||||
HistogramSketch histogram = (HistogramSketch) values.get(fieldName);
|
||||
List<Percentile> percentiles = histogram.percentileList(percentileTicksPerHalfDistance);
|
||||
return HdrHistogramModule.toJson(percentiles);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Comparator<double[]> getComparator()
|
||||
{
|
||||
throw new IAE("Comparing arrays of quantiles is not supported");
|
||||
}
|
||||
|
||||
@Override
|
||||
public Set<String> getDependentFields()
|
||||
{
|
||||
return Sets.newHashSet(fieldName);
|
||||
}
|
||||
|
||||
@Override
|
||||
public PostAggregator decorate(Map<String, AggregatorFactory> aggregators) {
|
||||
return this;
|
||||
}
|
||||
|
||||
@Override
|
||||
public byte[] getCacheKey() {
|
||||
CacheKeyBuilder builder = new CacheKeyBuilder(HdrHistogramModule.CACHE_TYPE_ID_OFFSET).appendByte(HdrHistogramModule.QUANTILES_HDRHISTOGRAM_TO_PERCENTILES_CACHE_TYPE_ID)
|
||||
.appendString(fieldName);
|
||||
builder.appendInt(percentileTicksPerHalfDistance);
|
||||
return builder.build();
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean equals(Object o) {
|
||||
if (this == o) {
|
||||
return true;
|
||||
}
|
||||
if (o == null || getClass() != o.getClass()) {
|
||||
return false;
|
||||
}
|
||||
HdrHistogramToPercentilesPostAggregator that = (HdrHistogramToPercentilesPostAggregator) o;
|
||||
|
||||
return percentileTicksPerHalfDistance == that.percentileTicksPerHalfDistance &&
|
||||
name.equals(that.name) &&
|
||||
fieldName.equals(that.fieldName);
|
||||
}
|
||||
|
||||
@Override
|
||||
public int hashCode() {
|
||||
return Objects.hash(name, fieldName, percentileTicksPerHalfDistance);
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return "HdrHistogramToPercentilesPostAggregator{" +
|
||||
"name='" + name + '\'' +
|
||||
", fieldName='" + fieldName + '\'' +
|
||||
", probabilitys=" + percentileTicksPerHalfDistance +
|
||||
'}';
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
package org.apache.druid.query.aggregation.sketch.HdrHistogram;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonCreator;
|
||||
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||
import com.google.common.collect.Sets;
|
||||
import org.HdrHistogram.HistogramSketch;
|
||||
import org.HdrHistogram.Percentile;
|
||||
import org.apache.druid.java.util.common.IAE;
|
||||
import org.apache.druid.query.aggregation.AggregatorFactory;
|
||||
import org.apache.druid.query.aggregation.PostAggregator;
|
||||
import org.apache.druid.query.cache.CacheKeyBuilder;
|
||||
import org.apache.druid.segment.ColumnInspector;
|
||||
import org.apache.druid.segment.column.ColumnType;
|
||||
|
||||
import javax.annotation.Nullable;
|
||||
import java.util.*;
|
||||
|
||||
public class HdrHistogramToPercentilesPostAggregator implements PostAggregator {
|
||||
private final String name;
|
||||
private final String fieldName;
|
||||
private final int percentileTicksPerHalfDistance;
|
||||
|
||||
@JsonCreator
|
||||
public HdrHistogramToPercentilesPostAggregator(
|
||||
@JsonProperty("name") String name,
|
||||
@JsonProperty("fieldName") String fieldName,
|
||||
@JsonProperty("percentileTicksPerHalfDistance") int percentileTicksPerHalfDistance
|
||||
){
|
||||
this.name = name;
|
||||
this.fieldName = fieldName;
|
||||
this.percentileTicksPerHalfDistance = percentileTicksPerHalfDistance;
|
||||
}
|
||||
|
||||
@Override
|
||||
public ColumnType getType(ColumnInspector signature){
|
||||
return ColumnType.STRING;
|
||||
}
|
||||
|
||||
@Override
|
||||
@JsonProperty
|
||||
public String getName() {
|
||||
return name;
|
||||
}
|
||||
|
||||
@JsonProperty
|
||||
public String getFieldName() {
|
||||
return fieldName;
|
||||
}
|
||||
|
||||
@JsonProperty
|
||||
public int getPercentileTicksPerHalfDistance() {
|
||||
return percentileTicksPerHalfDistance;
|
||||
}
|
||||
|
||||
@Nullable
|
||||
@Override
|
||||
public Object compute(Map<String, Object> values) {
|
||||
HistogramSketch histogram = (HistogramSketch) values.get(fieldName);
|
||||
if(histogram == null){
|
||||
return "[]"; //"[]"
|
||||
}
|
||||
List<Percentile> percentiles = histogram.percentileList(percentileTicksPerHalfDistance);
|
||||
return HdrHistogramModule.toJson(percentiles);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Comparator<double[]> getComparator()
|
||||
{
|
||||
throw new IAE("Comparing arrays of quantiles is not supported");
|
||||
}
|
||||
|
||||
@Override
|
||||
public Set<String> getDependentFields()
|
||||
{
|
||||
return Sets.newHashSet(fieldName);
|
||||
}
|
||||
|
||||
@Override
|
||||
public PostAggregator decorate(Map<String, AggregatorFactory> aggregators) {
|
||||
return this;
|
||||
}
|
||||
|
||||
@Override
|
||||
public byte[] getCacheKey() {
|
||||
CacheKeyBuilder builder = new CacheKeyBuilder(HdrHistogramModule.CACHE_TYPE_ID_OFFSET).appendByte(HdrHistogramModule.QUANTILES_HDRHISTOGRAM_TO_PERCENTILES_CACHE_TYPE_ID)
|
||||
.appendString(fieldName);
|
||||
builder.appendInt(percentileTicksPerHalfDistance);
|
||||
return builder.build();
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean equals(Object o) {
|
||||
if (this == o) {
|
||||
return true;
|
||||
}
|
||||
if (o == null || getClass() != o.getClass()) {
|
||||
return false;
|
||||
}
|
||||
HdrHistogramToPercentilesPostAggregator that = (HdrHistogramToPercentilesPostAggregator) o;
|
||||
|
||||
return percentileTicksPerHalfDistance == that.percentileTicksPerHalfDistance &&
|
||||
name.equals(that.name) &&
|
||||
fieldName.equals(that.fieldName);
|
||||
}
|
||||
|
||||
@Override
|
||||
public int hashCode() {
|
||||
return Objects.hash(name, fieldName, percentileTicksPerHalfDistance);
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return "HdrHistogramToPercentilesPostAggregator{" +
|
||||
"name='" + name + '\'' +
|
||||
", fieldName='" + fieldName + '\'' +
|
||||
", probabilitys=" + percentileTicksPerHalfDistance +
|
||||
'}';
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
@@ -1,118 +1,128 @@
|
||||
package org.apache.druid.query.aggregation.sketch.HdrHistogram;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonCreator;
|
||||
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||
import com.google.common.collect.Sets;
|
||||
import org.HdrHistogram.Histogram;
|
||||
import org.HdrHistogram.HistogramSketch;
|
||||
import org.apache.druid.java.util.common.IAE;
|
||||
import org.apache.druid.query.aggregation.AggregatorFactory;
|
||||
import org.apache.druid.query.aggregation.PostAggregator;
|
||||
import org.apache.druid.query.cache.CacheKeyBuilder;
|
||||
|
||||
import javax.annotation.Nullable;
|
||||
import java.util.Comparator;
|
||||
import java.util.Map;
|
||||
import java.util.Objects;
|
||||
import java.util.Set;
|
||||
|
||||
public class HdrHistogramToQuantilePostAggregator implements PostAggregator {
|
||||
private final String name;
|
||||
private final String fieldName;
|
||||
private final float probability;
|
||||
|
||||
@JsonCreator
|
||||
public HdrHistogramToQuantilePostAggregator(
|
||||
@JsonProperty("name") String name,
|
||||
@JsonProperty("fieldName") String fieldName,
|
||||
@JsonProperty("probability") float probability
|
||||
){
|
||||
this.name = name;
|
||||
this.fieldName = fieldName;
|
||||
this.probability = probability;
|
||||
|
||||
if (probability < 0 || probability > 1) {
|
||||
throw new IAE("Illegal probability[%s], must be strictly between 0 and 1", probability);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public Set<String> getDependentFields() {
|
||||
return Sets.newHashSet(fieldName);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Comparator getComparator() {
|
||||
return new Comparator<Long>(){
|
||||
@Override
|
||||
public int compare(final Long a, final Long b){
|
||||
return Long.compare(a, b);
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
@Nullable
|
||||
@Override
|
||||
public Object compute(Map<String, Object> values) {
|
||||
HistogramSketch histogram = (HistogramSketch) values.get(fieldName);
|
||||
return histogram.getValueAtPercentile(probability * 100);
|
||||
}
|
||||
|
||||
@Override
|
||||
@JsonProperty
|
||||
public String getName() {
|
||||
return name;
|
||||
}
|
||||
|
||||
@JsonProperty
|
||||
public String getFieldName() {
|
||||
return fieldName;
|
||||
}
|
||||
|
||||
@JsonProperty
|
||||
public double getProbability() {
|
||||
return probability;
|
||||
}
|
||||
|
||||
@Override
|
||||
public PostAggregator decorate(Map<String, AggregatorFactory> aggregators) {
|
||||
return this;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean equals(Object o) {
|
||||
if (this == o) {
|
||||
return true;
|
||||
}
|
||||
if (o == null || getClass() != o.getClass()) {
|
||||
return false;
|
||||
}
|
||||
HdrHistogramToQuantilePostAggregator that = (HdrHistogramToQuantilePostAggregator) o;
|
||||
|
||||
return Float.compare(that.probability, probability) == 0 &&
|
||||
name.equals(that.name) &&
|
||||
fieldName.equals(that.fieldName);
|
||||
}
|
||||
|
||||
@Override
|
||||
public int hashCode() {
|
||||
return Objects.hash(name, fieldName, probability);
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return "HdrHistogramToQuantilePostAggregator{" +
|
||||
"name='" + name + '\'' +
|
||||
", fieldName='" + fieldName + '\'' +
|
||||
", probability=" + probability +
|
||||
'}';
|
||||
}
|
||||
|
||||
@Override
|
||||
public byte[] getCacheKey() {
|
||||
return new CacheKeyBuilder(HdrHistogramModule.CACHE_TYPE_ID_OFFSET).appendByte(HdrHistogramModule.QUANTILES_HDRHISTOGRAM_TO_QUANTILE_CACHE_TYPE_ID)
|
||||
.appendString(fieldName)
|
||||
.appendFloat(probability)
|
||||
.build();
|
||||
}
|
||||
}
|
||||
package org.apache.druid.query.aggregation.sketch.HdrHistogram;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonCreator;
|
||||
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||
import com.google.common.collect.Sets;
|
||||
import org.HdrHistogram.Histogram;
|
||||
import org.HdrHistogram.HistogramSketch;
|
||||
import org.apache.druid.java.util.common.IAE;
|
||||
import org.apache.druid.query.aggregation.AggregatorFactory;
|
||||
import org.apache.druid.query.aggregation.PostAggregator;
|
||||
import org.apache.druid.query.cache.CacheKeyBuilder;
|
||||
import org.apache.druid.segment.ColumnInspector;
|
||||
import org.apache.druid.segment.column.ColumnType;
|
||||
|
||||
import javax.annotation.Nullable;
|
||||
import java.util.Comparator;
|
||||
import java.util.Map;
|
||||
import java.util.Objects;
|
||||
import java.util.Set;
|
||||
|
||||
public class HdrHistogramToQuantilePostAggregator implements PostAggregator {
|
||||
private final String name;
|
||||
private final String fieldName;
|
||||
private final float probability;
|
||||
|
||||
@JsonCreator
|
||||
public HdrHistogramToQuantilePostAggregator(
|
||||
@JsonProperty("name") String name,
|
||||
@JsonProperty("fieldName") String fieldName,
|
||||
@JsonProperty("probability") float probability
|
||||
){
|
||||
this.name = name;
|
||||
this.fieldName = fieldName;
|
||||
this.probability = probability;
|
||||
|
||||
if (probability < 0 || probability > 1) {
|
||||
throw new IAE("Illegal probability[%s], must be strictly between 0 and 1", probability);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public ColumnType getType(ColumnInspector signature){
|
||||
return ColumnType.LONG;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Set<String> getDependentFields() {
|
||||
return Sets.newHashSet(fieldName);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Comparator getComparator() {
|
||||
return new Comparator<Long>(){
|
||||
@Override
|
||||
public int compare(final Long a, final Long b){
|
||||
return Long.compare(a, b);
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
@Nullable
|
||||
@Override
|
||||
public Object compute(Map<String, Object> values) {
|
||||
HistogramSketch histogram = (HistogramSketch) values.get(fieldName);
|
||||
if(histogram == null){
|
||||
return null;
|
||||
}
|
||||
return histogram.getValueAtPercentile(probability * 100);
|
||||
}
|
||||
|
||||
@Override
|
||||
@JsonProperty
|
||||
public String getName() {
|
||||
return name;
|
||||
}
|
||||
|
||||
@JsonProperty
|
||||
public String getFieldName() {
|
||||
return fieldName;
|
||||
}
|
||||
|
||||
@JsonProperty
|
||||
public double getProbability() {
|
||||
return probability;
|
||||
}
|
||||
|
||||
@Override
|
||||
public PostAggregator decorate(Map<String, AggregatorFactory> aggregators) {
|
||||
return this;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean equals(Object o) {
|
||||
if (this == o) {
|
||||
return true;
|
||||
}
|
||||
if (o == null || getClass() != o.getClass()) {
|
||||
return false;
|
||||
}
|
||||
HdrHistogramToQuantilePostAggregator that = (HdrHistogramToQuantilePostAggregator) o;
|
||||
|
||||
return Float.compare(that.probability, probability) == 0 &&
|
||||
name.equals(that.name) &&
|
||||
fieldName.equals(that.fieldName);
|
||||
}
|
||||
|
||||
@Override
|
||||
public int hashCode() {
|
||||
return Objects.hash(name, fieldName, probability);
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return "HdrHistogramToQuantilePostAggregator{" +
|
||||
"name='" + name + '\'' +
|
||||
", fieldName='" + fieldName + '\'' +
|
||||
", probability=" + probability +
|
||||
'}';
|
||||
}
|
||||
|
||||
@Override
|
||||
public byte[] getCacheKey() {
|
||||
return new CacheKeyBuilder(HdrHistogramModule.CACHE_TYPE_ID_OFFSET).appendByte(HdrHistogramModule.QUANTILES_HDRHISTOGRAM_TO_QUANTILE_CACHE_TYPE_ID)
|
||||
.appendString(fieldName)
|
||||
.appendFloat(probability)
|
||||
.build();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,114 +1,125 @@
|
||||
package org.apache.druid.query.aggregation.sketch.HdrHistogram;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonCreator;
|
||||
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||
import com.google.common.collect.Sets;
|
||||
import org.HdrHistogram.Histogram;
|
||||
import org.HdrHistogram.HistogramSketch;
|
||||
import org.apache.druid.java.util.common.IAE;
|
||||
import org.apache.druid.query.aggregation.AggregatorFactory;
|
||||
import org.apache.druid.query.aggregation.PostAggregator;
|
||||
import org.apache.druid.query.cache.CacheKeyBuilder;
|
||||
|
||||
import javax.annotation.Nullable;
|
||||
import java.util.*;
|
||||
|
||||
public class HdrHistogramToQuantilesPostAggregator implements PostAggregator {
|
||||
private final String name;
|
||||
private final String fieldName;
|
||||
private final float[] probabilitys;
|
||||
|
||||
@JsonCreator
|
||||
public HdrHistogramToQuantilesPostAggregator(
|
||||
@JsonProperty("name") String name,
|
||||
@JsonProperty("fieldName") String fieldName,
|
||||
@JsonProperty("probabilitys") float[] probabilitys
|
||||
){
|
||||
this.name = name;
|
||||
this.fieldName = fieldName;
|
||||
this.probabilitys = probabilitys;
|
||||
}
|
||||
|
||||
@Override
|
||||
@JsonProperty
|
||||
public String getName() {
|
||||
return name;
|
||||
}
|
||||
|
||||
@JsonProperty
|
||||
public String getFieldName() {
|
||||
return fieldName;
|
||||
}
|
||||
|
||||
@JsonProperty
|
||||
public float[] getProbabilitys() {
|
||||
return probabilitys;
|
||||
}
|
||||
|
||||
@Nullable
|
||||
@Override
|
||||
public Object compute(Map<String, Object> values) {
|
||||
HistogramSketch histogram = (HistogramSketch) values.get(fieldName);
|
||||
final long[] counts = new long[probabilitys.length];
|
||||
for (int i = 0; i < probabilitys.length; i++) {
|
||||
counts[i] = histogram.getValueAtPercentile(probabilitys[i] * 100);
|
||||
}
|
||||
return counts;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Comparator<double[]> getComparator()
|
||||
{
|
||||
throw new IAE("Comparing arrays of quantiles is not supported");
|
||||
}
|
||||
|
||||
@Override
|
||||
public Set<String> getDependentFields()
|
||||
{
|
||||
return Sets.newHashSet(fieldName);
|
||||
}
|
||||
|
||||
@Override
|
||||
public PostAggregator decorate(Map<String, AggregatorFactory> aggregators) {
|
||||
return this;
|
||||
}
|
||||
|
||||
@Override
|
||||
public byte[] getCacheKey() {
|
||||
CacheKeyBuilder builder = new CacheKeyBuilder(HdrHistogramModule.CACHE_TYPE_ID_OFFSET).appendByte(HdrHistogramModule.QUANTILES_HDRHISTOGRAM_TO_QUANTILES_CACHE_TYPE_ID)
|
||||
.appendString(fieldName);
|
||||
for (float probability : probabilitys) {
|
||||
builder.appendFloat(probability);
|
||||
}
|
||||
return builder.build();
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean equals(Object o) {
|
||||
if (this == o) {
|
||||
return true;
|
||||
}
|
||||
if (o == null || getClass() != o.getClass()) {
|
||||
return false;
|
||||
}
|
||||
HdrHistogramToQuantilesPostAggregator that = (HdrHistogramToQuantilesPostAggregator) o;
|
||||
|
||||
return Arrays.equals(probabilitys, that.probabilitys) &&
|
||||
name.equals(that.name) &&
|
||||
fieldName.equals(that.fieldName);
|
||||
}
|
||||
|
||||
@Override
|
||||
public int hashCode() {
|
||||
return Objects.hash(name, fieldName, Arrays.hashCode(probabilitys));
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return "HdrHistogramToQuantilesPostAggregator{" +
|
||||
"name='" + name + '\'' +
|
||||
", fieldName='" + fieldName + '\'' +
|
||||
", probabilitys=" + Arrays.toString(probabilitys) +
|
||||
'}';
|
||||
}
|
||||
}
|
||||
package org.apache.druid.query.aggregation.sketch.HdrHistogram;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonCreator;
|
||||
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||
import com.google.common.collect.Sets;
|
||||
import org.HdrHistogram.Histogram;
|
||||
import org.HdrHistogram.HistogramSketch;
|
||||
import org.apache.druid.java.util.common.IAE;
|
||||
import org.apache.druid.query.aggregation.AggregatorFactory;
|
||||
import org.apache.druid.query.aggregation.PostAggregator;
|
||||
import org.apache.druid.query.cache.CacheKeyBuilder;
|
||||
import org.apache.druid.segment.ColumnInspector;
|
||||
import org.apache.druid.segment.column.ColumnType;
|
||||
|
||||
import javax.annotation.Nullable;
|
||||
import java.util.*;
|
||||
|
||||
public class HdrHistogramToQuantilesPostAggregator implements PostAggregator {
|
||||
private final String name;
|
||||
private final String fieldName;
|
||||
private final float[] probabilitys;
|
||||
|
||||
@JsonCreator
|
||||
public HdrHistogramToQuantilesPostAggregator(
|
||||
@JsonProperty("name") String name,
|
||||
@JsonProperty("fieldName") String fieldName,
|
||||
@JsonProperty("probabilitys") float[] probabilitys
|
||||
){
|
||||
this.name = name;
|
||||
this.fieldName = fieldName;
|
||||
this.probabilitys = probabilitys;
|
||||
}
|
||||
|
||||
@Override
|
||||
public ColumnType getType(ColumnInspector signature){
|
||||
return ColumnType.LONG_ARRAY;
|
||||
}
|
||||
|
||||
@Override
|
||||
@JsonProperty
|
||||
public String getName() {
|
||||
return name;
|
||||
}
|
||||
|
||||
@JsonProperty
|
||||
public String getFieldName() {
|
||||
return fieldName;
|
||||
}
|
||||
|
||||
@JsonProperty
|
||||
public float[] getProbabilitys() {
|
||||
return probabilitys;
|
||||
}
|
||||
|
||||
@Nullable
|
||||
@Override
|
||||
public Object compute(Map<String, Object> values) {
|
||||
HistogramSketch histogram = (HistogramSketch) values.get(fieldName);
|
||||
if(histogram == null){
|
||||
//return null;
|
||||
return new Long[probabilitys.length];
|
||||
}
|
||||
final Long[] counts = new Long[probabilitys.length];
|
||||
for (int i = 0; i < probabilitys.length; i++) {
|
||||
counts[i] = histogram.getValueAtPercentile(probabilitys[i] * 100);
|
||||
}
|
||||
return counts;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Comparator<double[]> getComparator()
|
||||
{
|
||||
throw new IAE("Comparing arrays of quantiles is not supported");
|
||||
}
|
||||
|
||||
@Override
|
||||
public Set<String> getDependentFields()
|
||||
{
|
||||
return Sets.newHashSet(fieldName);
|
||||
}
|
||||
|
||||
@Override
|
||||
public PostAggregator decorate(Map<String, AggregatorFactory> aggregators) {
|
||||
return this;
|
||||
}
|
||||
|
||||
@Override
|
||||
public byte[] getCacheKey() {
|
||||
CacheKeyBuilder builder = new CacheKeyBuilder(HdrHistogramModule.CACHE_TYPE_ID_OFFSET).appendByte(HdrHistogramModule.QUANTILES_HDRHISTOGRAM_TO_QUANTILES_CACHE_TYPE_ID)
|
||||
.appendString(fieldName);
|
||||
for (float probability : probabilitys) {
|
||||
builder.appendFloat(probability);
|
||||
}
|
||||
return builder.build();
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean equals(Object o) {
|
||||
if (this == o) {
|
||||
return true;
|
||||
}
|
||||
if (o == null || getClass() != o.getClass()) {
|
||||
return false;
|
||||
}
|
||||
HdrHistogramToQuantilesPostAggregator that = (HdrHistogramToQuantilesPostAggregator) o;
|
||||
|
||||
return Arrays.equals(probabilitys, that.probabilitys) &&
|
||||
name.equals(that.name) &&
|
||||
fieldName.equals(that.fieldName);
|
||||
}
|
||||
|
||||
@Override
|
||||
public int hashCode() {
|
||||
return Objects.hash(name, fieldName, Arrays.hashCode(probabilitys));
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return "HdrHistogramToQuantilesPostAggregator{" +
|
||||
"name='" + name + '\'' +
|
||||
", fieldName='" + fieldName + '\'' +
|
||||
", probabilitys=" + Arrays.toString(probabilitys) +
|
||||
'}';
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,77 @@
|
||||
package org.apache.druid.query.aggregation.sketch.HdrHistogram.sql;
|
||||
|
||||
import org.apache.calcite.rex.RexCall;
|
||||
import org.apache.calcite.rex.RexNode;
|
||||
import org.apache.calcite.sql.SqlFunction;
|
||||
import org.apache.calcite.sql.SqlOperator;
|
||||
import org.apache.calcite.sql.type.ReturnTypes;
|
||||
import org.apache.calcite.sql.type.SqlTypeFamily;
|
||||
import org.apache.calcite.sql.type.SqlTypeName;
|
||||
import org.apache.druid.java.util.common.StringUtils;
|
||||
import org.apache.druid.query.aggregation.PostAggregator;
|
||||
import org.apache.druid.query.aggregation.post.FieldAccessPostAggregator;
|
||||
import org.apache.druid.query.aggregation.sketch.HdrHistogram.HdrHistogramToDescribePostAggregator;
|
||||
import org.apache.druid.segment.column.RowSignature;
|
||||
import org.apache.druid.sql.calcite.expression.DruidExpression;
|
||||
import org.apache.druid.sql.calcite.expression.OperatorConversions;
|
||||
import org.apache.druid.sql.calcite.expression.PostAggregatorVisitor;
|
||||
import org.apache.druid.sql.calcite.expression.SqlOperatorConversion;
|
||||
import org.apache.druid.sql.calcite.planner.PlannerContext;
|
||||
|
||||
import javax.annotation.Nullable;
|
||||
import java.util.List;
|
||||
|
||||
public class HdrHistogramDescribeOperatorConversion implements SqlOperatorConversion {
|
||||
private static final String FUNCTION_NAME = "HDR_DESCRIBE";
|
||||
private static final SqlFunction SQL_FUNCTION = OperatorConversions
|
||||
.operatorBuilder(StringUtils.toUpperCase(FUNCTION_NAME))
|
||||
.operandTypes(SqlTypeFamily.ANY)
|
||||
.requiredOperands(1)
|
||||
.returnTypeInference(ReturnTypes.explicit(SqlTypeName.VARCHAR))
|
||||
.build();
|
||||
|
||||
@Override
|
||||
public SqlOperator calciteOperator()
|
||||
{
|
||||
return SQL_FUNCTION;
|
||||
}
|
||||
|
||||
@Override
|
||||
public DruidExpression toDruidExpression(
|
||||
PlannerContext plannerContext,
|
||||
RowSignature rowSignature,
|
||||
RexNode rexNode
|
||||
)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
@Nullable
|
||||
@Override
|
||||
public PostAggregator toPostAggregator(
|
||||
PlannerContext plannerContext,
|
||||
RowSignature rowSignature,
|
||||
RexNode rexNode,
|
||||
PostAggregatorVisitor postAggregatorVisitor
|
||||
)
|
||||
{
|
||||
final List<RexNode> operands = ((RexCall) rexNode).getOperands();
|
||||
final PostAggregator postAgg = OperatorConversions.toPostAggregator(
|
||||
plannerContext,
|
||||
rowSignature,
|
||||
operands.get(0),
|
||||
postAggregatorVisitor,
|
||||
true
|
||||
);
|
||||
|
||||
if (postAgg == null) {
|
||||
return null;
|
||||
}
|
||||
|
||||
|
||||
return new HdrHistogramToDescribePostAggregator(
|
||||
postAggregatorVisitor.getOutputNamePrefix() + postAggregatorVisitor.getAndIncrementCounter(),
|
||||
((FieldAccessPostAggregator)postAgg).getFieldName()
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -18,6 +18,7 @@ import org.apache.druid.query.aggregation.AggregatorFactory;
|
||||
import org.apache.druid.query.aggregation.sketch.HdrHistogram.HdrHistogramAggregatorFactory;
|
||||
import org.apache.druid.query.aggregation.sketch.HdrHistogram.HdrHistogramMergeAggregatorFactory;
|
||||
import org.apache.druid.segment.VirtualColumn;
|
||||
import org.apache.druid.segment.column.ColumnType;
|
||||
import org.apache.druid.segment.column.RowSignature;
|
||||
import org.apache.druid.segment.column.ValueType;
|
||||
import org.apache.druid.sql.calcite.aggregation.Aggregation;
|
||||
@@ -118,11 +119,11 @@ public class HdrHistogramObjectSqlAggregator implements SqlAggregator {
|
||||
}
|
||||
|
||||
// No existing match found. Create a new one.
|
||||
final List<VirtualColumn> virtualColumns = new ArrayList<>();
|
||||
// 新版本删除了final List<VirtualColumn> virtualColumns = new ArrayList<>();
|
||||
|
||||
if (input.isDirectColumnAccess()) {
|
||||
// 参数是Histogram对象
|
||||
if (rowSignature.getColumnType(input.getDirectColumn()).orElse(null) == ValueType.COMPLEX) {
|
||||
if (rowSignature.getColumnType(input.getDirectColumn()).map(type -> type.is(ValueType.COMPLEX)).orElse(false)) {
|
||||
aggregatorFactory = new HdrHistogramMergeAggregatorFactory(
|
||||
histogramName,
|
||||
input.getDirectColumn(),
|
||||
@@ -142,12 +143,11 @@ public class HdrHistogramObjectSqlAggregator implements SqlAggregator {
|
||||
);
|
||||
}
|
||||
} else {
|
||||
final VirtualColumn virtualColumn =
|
||||
virtualColumnRegistry.getOrCreateVirtualColumnForExpression(plannerContext, input, SqlTypeName.BIGINT);
|
||||
virtualColumns.add(virtualColumn);
|
||||
final String virtualColumnName =
|
||||
virtualColumnRegistry.getOrCreateVirtualColumnForExpression(input, ColumnType.LONG);
|
||||
aggregatorFactory = new HdrHistogramAggregatorFactory(
|
||||
histogramName,
|
||||
virtualColumn.getOutputName(),
|
||||
virtualColumnName,
|
||||
lowestDiscernibleValue,
|
||||
highestTrackableValue,
|
||||
numberOfSignificantValueDigits,
|
||||
@@ -156,7 +156,6 @@ public class HdrHistogramObjectSqlAggregator implements SqlAggregator {
|
||||
}
|
||||
|
||||
return Aggregation.create(
|
||||
virtualColumns,
|
||||
ImmutableList.of(aggregatorFactory),
|
||||
null
|
||||
);
|
||||
|
||||
@@ -0,0 +1,88 @@
|
||||
package org.apache.druid.query.aggregation.sketch.HdrHistogram.sql;
|
||||
|
||||
import org.apache.calcite.rex.RexCall;
|
||||
import org.apache.calcite.rex.RexLiteral;
|
||||
import org.apache.calcite.rex.RexNode;
|
||||
import org.apache.calcite.sql.SqlFunction;
|
||||
import org.apache.calcite.sql.SqlKind;
|
||||
import org.apache.calcite.sql.SqlOperator;
|
||||
import org.apache.calcite.sql.type.ReturnTypes;
|
||||
import org.apache.calcite.sql.type.SqlTypeFamily;
|
||||
import org.apache.calcite.sql.type.SqlTypeName;
|
||||
import org.apache.druid.java.util.common.StringUtils;
|
||||
import org.apache.druid.query.aggregation.PostAggregator;
|
||||
import org.apache.druid.query.aggregation.post.FieldAccessPostAggregator;
|
||||
import org.apache.druid.query.aggregation.sketch.HdrHistogram.HdrHistogramToPercentilesDescribePostAggregator;
|
||||
import org.apache.druid.segment.column.RowSignature;
|
||||
import org.apache.druid.sql.calcite.expression.DruidExpression;
|
||||
import org.apache.druid.sql.calcite.expression.OperatorConversions;
|
||||
import org.apache.druid.sql.calcite.expression.PostAggregatorVisitor;
|
||||
import org.apache.druid.sql.calcite.expression.SqlOperatorConversion;
|
||||
import org.apache.druid.sql.calcite.planner.PlannerContext;
|
||||
|
||||
import javax.annotation.Nullable;
|
||||
import java.util.List;
|
||||
|
||||
public class HdrHistogramPercentilesDescribeOperatorConversion implements SqlOperatorConversion {
|
||||
private static final String FUNCTION_NAME = "HDR_GET_PERCENTILES_DESCRIPTION";
|
||||
private static final SqlFunction SQL_FUNCTION = OperatorConversions
|
||||
.operatorBuilder(StringUtils.toUpperCase(FUNCTION_NAME))
|
||||
.operandTypes(SqlTypeFamily.ANY, SqlTypeFamily.NUMERIC)
|
||||
.requiredOperands(1)
|
||||
.returnTypeInference(ReturnTypes.explicit(SqlTypeName.VARCHAR))
|
||||
.build();
|
||||
|
||||
@Override
|
||||
public SqlOperator calciteOperator()
|
||||
{
|
||||
return SQL_FUNCTION;
|
||||
}
|
||||
|
||||
@Override
|
||||
public DruidExpression toDruidExpression(
|
||||
PlannerContext plannerContext,
|
||||
RowSignature rowSignature,
|
||||
RexNode rexNode
|
||||
)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
@Nullable
|
||||
@Override
|
||||
public PostAggregator toPostAggregator(
|
||||
PlannerContext plannerContext,
|
||||
RowSignature rowSignature,
|
||||
RexNode rexNode,
|
||||
PostAggregatorVisitor postAggregatorVisitor
|
||||
)
|
||||
{
|
||||
final List<RexNode> operands = ((RexCall) rexNode).getOperands();
|
||||
final PostAggregator postAgg = OperatorConversions.toPostAggregator(
|
||||
plannerContext,
|
||||
rowSignature,
|
||||
operands.get(0),
|
||||
postAggregatorVisitor,
|
||||
true
|
||||
);
|
||||
|
||||
if (postAgg == null) {
|
||||
return null;
|
||||
}
|
||||
|
||||
int percentileTicksPerHalfDistance = 5;
|
||||
if (operands.size() == 2) {
|
||||
if (!operands.get(1).isA(SqlKind.LITERAL)) {
|
||||
return null;
|
||||
}
|
||||
|
||||
percentileTicksPerHalfDistance = RexLiteral.intValue(operands.get(1));
|
||||
}
|
||||
|
||||
return new HdrHistogramToPercentilesDescribePostAggregator(
|
||||
postAggregatorVisitor.getOutputNamePrefix() + postAggregatorVisitor.getAndIncrementCounter(),
|
||||
((FieldAccessPostAggregator)postAgg).getFieldName(),
|
||||
percentileTicksPerHalfDistance
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -14,16 +14,16 @@ import org.apache.druid.query.aggregation.sketch.HdrHistogram.HdrHistogramToPerc
|
||||
import org.apache.druid.query.aggregation.PostAggregator;
|
||||
import org.apache.druid.query.aggregation.post.FieldAccessPostAggregator;
|
||||
import org.apache.druid.segment.column.RowSignature;
|
||||
import org.apache.druid.sql.calcite.expression.DirectOperatorConversion;
|
||||
import org.apache.druid.sql.calcite.expression.DruidExpression;
|
||||
import org.apache.druid.sql.calcite.expression.OperatorConversions;
|
||||
import org.apache.druid.sql.calcite.expression.PostAggregatorVisitor;
|
||||
import org.apache.druid.sql.calcite.expression.SqlOperatorConversion;
|
||||
import org.apache.druid.sql.calcite.planner.PlannerContext;
|
||||
|
||||
import javax.annotation.Nullable;
|
||||
import java.util.List;
|
||||
|
||||
public class HdrHistogramPercentilesOperatorConversion extends DirectOperatorConversion {
|
||||
public class HdrHistogramPercentilesOperatorConversion implements SqlOperatorConversion {
|
||||
private static final String FUNCTION_NAME = "HDR_GET_PERCENTILES";
|
||||
private static final SqlFunction SQL_FUNCTION = OperatorConversions
|
||||
.operatorBuilder(StringUtils.toUpperCase(FUNCTION_NAME))
|
||||
@@ -32,10 +32,6 @@ public class HdrHistogramPercentilesOperatorConversion extends DirectOperatorCon
|
||||
.returnTypeInference(ReturnTypes.explicit(SqlTypeName.VARCHAR))
|
||||
.build();
|
||||
|
||||
public HdrHistogramPercentilesOperatorConversion() {
|
||||
super(SQL_FUNCTION, FUNCTION_NAME);
|
||||
}
|
||||
|
||||
@Override
|
||||
public SqlOperator calciteOperator()
|
||||
{
|
||||
@@ -66,7 +62,8 @@ public class HdrHistogramPercentilesOperatorConversion extends DirectOperatorCon
|
||||
plannerContext,
|
||||
rowSignature,
|
||||
operands.get(0),
|
||||
postAggregatorVisitor
|
||||
postAggregatorVisitor,
|
||||
true
|
||||
);
|
||||
|
||||
if (postAgg == null) {
|
||||
|
||||
@@ -16,6 +16,7 @@ import org.apache.druid.query.aggregation.sketch.HdrHistogram.HdrHistogramAggreg
|
||||
import org.apache.druid.query.aggregation.sketch.HdrHistogram.HdrHistogramMergeAggregatorFactory;
|
||||
import org.apache.druid.query.aggregation.sketch.HdrHistogram.HdrHistogramToQuantilePostAggregator;
|
||||
import org.apache.druid.segment.VirtualColumn;
|
||||
import org.apache.druid.segment.column.ColumnType;
|
||||
import org.apache.druid.segment.column.RowSignature;
|
||||
import org.apache.druid.segment.column.ValueType;
|
||||
import org.apache.druid.segment.virtual.ExpressionVirtualColumn;
|
||||
@@ -141,22 +142,16 @@ public class HdrHistogramQuantileSqlAggregator implements SqlAggregator {
|
||||
|
||||
// Check input for equivalence.
|
||||
final boolean inputMatches;
|
||||
final VirtualColumn virtualInput = existing.getVirtualColumns()
|
||||
.stream()
|
||||
.filter(
|
||||
virtualColumn ->
|
||||
virtualColumn.getOutputName()
|
||||
.equals(theFactory.getFieldName())
|
||||
)
|
||||
.findFirst()
|
||||
.orElse(null);
|
||||
final DruidExpression virtualInput =
|
||||
virtualColumnRegistry.findVirtualColumnExpressions(theFactory.requiredFields())
|
||||
.stream()
|
||||
.findFirst()
|
||||
.orElse(null);
|
||||
|
||||
if (virtualInput == null) {
|
||||
inputMatches = input.isDirectColumnAccess()
|
||||
&& input.getDirectColumn().equals(theFactory.getFieldName());
|
||||
inputMatches = input.isDirectColumnAccess() && input.getDirectColumn().equals(theFactory.getFieldName());
|
||||
} else {
|
||||
inputMatches = ((ExpressionVirtualColumn) virtualInput).getExpression()
|
||||
.equals(input.getExpression());
|
||||
inputMatches = virtualInput.equals(input);
|
||||
}
|
||||
|
||||
final boolean matches = inputMatches
|
||||
@@ -177,11 +172,11 @@ public class HdrHistogramQuantileSqlAggregator implements SqlAggregator {
|
||||
}
|
||||
|
||||
// No existing match found. Create a new one.
|
||||
final List<VirtualColumn> virtualColumns = new ArrayList<>();
|
||||
//final List<VirtualColumn> virtualColumns = new ArrayList<>();
|
||||
|
||||
if (input.isDirectColumnAccess()) {
|
||||
// 参数是Histogram对象
|
||||
if (rowSignature.getColumnType(input.getDirectColumn()).orElse(null) == ValueType.COMPLEX) {
|
||||
if (rowSignature.getColumnType(input.getDirectColumn()).map(type -> type.is(ValueType.COMPLEX)).orElse(false)) {
|
||||
aggregatorFactory = new HdrHistogramMergeAggregatorFactory(
|
||||
histogramName,
|
||||
input.getDirectColumn(),
|
||||
@@ -201,12 +196,11 @@ public class HdrHistogramQuantileSqlAggregator implements SqlAggregator {
|
||||
);
|
||||
}
|
||||
} else {
|
||||
final VirtualColumn virtualColumn =
|
||||
virtualColumnRegistry.getOrCreateVirtualColumnForExpression(plannerContext, input, SqlTypeName.BIGINT);
|
||||
virtualColumns.add(virtualColumn);
|
||||
final String virtualColumnName =
|
||||
virtualColumnRegistry.getOrCreateVirtualColumnForExpression(input, ColumnType.LONG);
|
||||
aggregatorFactory = new HdrHistogramAggregatorFactory(
|
||||
histogramName,
|
||||
virtualColumn.getOutputName(),
|
||||
virtualColumnName,
|
||||
lowestDiscernibleValue,
|
||||
highestTrackableValue,
|
||||
numberOfSignificantValueDigits,
|
||||
@@ -234,7 +228,6 @@ public class HdrHistogramQuantileSqlAggregator implements SqlAggregator {
|
||||
}
|
||||
|
||||
return Aggregation.create(
|
||||
virtualColumns,
|
||||
ImmutableList.of(aggregatorFactory),
|
||||
new HdrHistogramToQuantilePostAggregator(name, histogramName, probability)
|
||||
);
|
||||
|
||||
@@ -62,50 +62,30 @@ public class HdrHistogramQuantilesOperatorConversion implements SqlOperatorConve
|
||||
{
|
||||
final List<RexNode> operands = ((RexCall) rexNode).getOperands();
|
||||
final float[] args = new float[operands.size() - 1];
|
||||
PostAggregator postAgg = null;
|
||||
|
||||
int operandCounter = 0;
|
||||
for (RexNode operand : operands) {
|
||||
final PostAggregator convertedPostAgg = OperatorConversions.toPostAggregator(
|
||||
plannerContext,
|
||||
rowSignature,
|
||||
operand,
|
||||
postAggregatorVisitor
|
||||
);
|
||||
if (convertedPostAgg == null) {
|
||||
if (operandCounter > 0) {
|
||||
try {
|
||||
if (!operand.isA(SqlKind.LITERAL)) {
|
||||
return null;
|
||||
}
|
||||
float arg = ((Number) RexLiteral.value(operand)).floatValue();
|
||||
args[operandCounter - 1] = arg;
|
||||
}
|
||||
catch (ClassCastException cce) {
|
||||
return null;
|
||||
}
|
||||
} else {
|
||||
return null;
|
||||
}
|
||||
} else {
|
||||
if (operandCounter == 0) {
|
||||
postAgg = convertedPostAgg;
|
||||
} else {
|
||||
if (!operand.isA(SqlKind.LITERAL)) {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
}
|
||||
operandCounter++;
|
||||
// 新版本直接就从第一个参数取
|
||||
final PostAggregator inputSketchPostAgg = OperatorConversions.toPostAggregator(
|
||||
plannerContext,
|
||||
rowSignature,
|
||||
operands.get(0),
|
||||
postAggregatorVisitor,
|
||||
true
|
||||
);
|
||||
|
||||
if (inputSketchPostAgg == null) {
|
||||
return null;
|
||||
}
|
||||
|
||||
if (postAgg == null) {
|
||||
return null;
|
||||
// 直接解析
|
||||
for (int i = 1; i < operands.size(); i++) {
|
||||
RexNode operand = operands.get(i);
|
||||
float arg = ((Number) RexLiteral.value(operand)).floatValue();
|
||||
args[i - 1] = arg;
|
||||
}
|
||||
|
||||
return new HdrHistogramToQuantilesPostAggregator(
|
||||
postAggregatorVisitor.getOutputNamePrefix() + postAggregatorVisitor.getAndIncrementCounter(),
|
||||
((FieldAccessPostAggregator)postAgg).getFieldName(),
|
||||
((FieldAccessPostAggregator)inputSketchPostAgg).getFieldName(),
|
||||
args
|
||||
);
|
||||
}
|
||||
|
||||
@@ -2,17 +2,13 @@ package org.apache.druid.query.aggregation.sketch.HdrHistogram;
|
||||
|
||||
import com.google.common.collect.ImmutableMap;
|
||||
import org.HdrHistogram.*;
|
||||
import org.apache.datasketches.theta.Sketches;
|
||||
import org.apache.datasketches.theta.UpdateSketch;
|
||||
import org.apache.druid.data.input.MapBasedRow;
|
||||
import org.apache.druid.query.aggregation.AggregatorFactory;
|
||||
import org.apache.druid.query.aggregation.BufferAggregator;
|
||||
import org.apache.druid.query.aggregation.TestLongColumnSelector;
|
||||
import org.apache.druid.query.aggregation.TestObjectColumnSelector;
|
||||
import org.apache.druid.query.aggregation.datasketches.theta.SketchHolder;
|
||||
import org.apache.druid.query.aggregation.datasketches.theta.SketchMergeAggregatorFactory;
|
||||
import org.apache.druid.query.groupby.epinephelinae.GrouperTestUtil;
|
||||
import org.apache.druid.query.groupby.epinephelinae.TestColumnSelectorFactory;
|
||||
import org.apache.druid.query.groupby.epinephelinae.GroupByTestColumnSelectorFactory;
|
||||
import org.apache.druid.segment.ColumnSelectorFactory;
|
||||
import org.junit.Assert;
|
||||
import org.junit.Test;
|
||||
@@ -230,7 +226,7 @@ public class HdrHistogramBufferAggregatorTest {
|
||||
|
||||
@Test
|
||||
public void testMergeAggregatorRelocate() {
|
||||
final TestColumnSelectorFactory columnSelectorFactory = GrouperTestUtil.newColumnSelectorFactory();
|
||||
final GroupByTestColumnSelectorFactory columnSelectorFactory = GrouperTestUtil.newColumnSelectorFactory();
|
||||
HistogramSketch histogram = new HistogramSketch(3);
|
||||
for (int i = 0; i < 100000; i++) {
|
||||
histogram.recordValue(i);
|
||||
@@ -252,7 +248,7 @@ public class HdrHistogramBufferAggregatorTest {
|
||||
|
||||
@Test
|
||||
public void testAggregatorRelocate() {
|
||||
final TestColumnSelectorFactory columnSelectorFactory = GrouperTestUtil.newColumnSelectorFactory();
|
||||
final GroupByTestColumnSelectorFactory columnSelectorFactory = GrouperTestUtil.newColumnSelectorFactory();
|
||||
HistogramSketch histogram = new HistogramSketch(3);
|
||||
for (int i = 0; i < 100000; i++) {
|
||||
histogram.recordValue(i);
|
||||
|
||||
@@ -0,0 +1,79 @@
|
||||
package org.apache.druid.query.aggregation.sketch.HdrHistogram;
|
||||
|
||||
import org.HdrHistogram.DirectArrayHistogram;
|
||||
import org.HdrHistogram.HistogramSketch;
|
||||
import org.HdrHistogram.Histogramer;
|
||||
import org.HdrHistogram.Percentile;
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import org.junit.Test;
|
||||
|
||||
import java.io.BufferedWriter;
|
||||
import java.nio.ByteBuffer;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.nio.file.FileSystems;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.util.Random;
|
||||
import java.util.concurrent.ThreadLocalRandom;
|
||||
|
||||
public class HistogramSketchTest {
|
||||
|
||||
@Test
|
||||
public void describeTest() throws Exception{
|
||||
DirectArrayHistogram histogram = new DirectArrayHistogram(1, 1000000, 3,
|
||||
ByteBuffer.allocate(HistogramSketch.getUpdatableSerializationBytes(1, 1000000, 3)));
|
||||
System.out.println(histogram.describe());
|
||||
for (int i = 0; i < 10000; i++) {
|
||||
histogram.recordValue(i);
|
||||
}
|
||||
System.out.println(histogram.describe());
|
||||
for (Percentile percentile : histogram.percentileList(100)) {
|
||||
System.out.println(percentile);
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void describeTest1() throws Exception{
|
||||
HistogramSketch histogram = new HistogramSketch(1);
|
||||
System.out.println(histogram.describe());
|
||||
for (int i = 0; i < 10000; i++) {
|
||||
histogram.recordValue(i);
|
||||
}
|
||||
System.out.println(histogram.describe());
|
||||
for (Percentile percentile : histogram.percentileList(100)) {
|
||||
System.out.println(percentile);
|
||||
}
|
||||
System.out.println(StringUtils.repeat('#', 100));
|
||||
histogram = new HistogramSketch(1);
|
||||
for (int i = 0; i < 10000; i++) {
|
||||
histogram.recordValue(ThreadLocalRandom.current().nextLong(100000));
|
||||
}
|
||||
System.out.println(histogram.describe());
|
||||
for (Percentile percentile : histogram.percentileList(100)) {
|
||||
System.out.println(percentile);
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void describeTest3() throws Exception{
|
||||
HistogramSketch histogram = new HistogramSketch(3);
|
||||
System.out.println(histogram.describe());
|
||||
for (int i = 0; i < 10000; i++) {
|
||||
histogram.recordValue(i);
|
||||
}
|
||||
System.out.println(histogram.describe());
|
||||
for (Percentile percentile : histogram.percentileList(100)) {
|
||||
System.out.println(percentile);
|
||||
}
|
||||
System.out.println(StringUtils.repeat('#', 100));
|
||||
histogram = new HistogramSketch(3);
|
||||
for (int i = 0; i < 10000; i++) {
|
||||
histogram.recordValue(ThreadLocalRandom.current().nextLong(100000));
|
||||
}
|
||||
System.out.println(histogram.describe());
|
||||
for (Percentile percentile : histogram.percentileList(100)) {
|
||||
System.out.println(percentile);
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
@@ -1,12 +1,15 @@
|
||||
package org.apache.druid.query.aggregation.sketch.HdrHistogram.sql;
|
||||
|
||||
import com.alibaba.fastjson2.JSON;
|
||||
import com.fasterxml.jackson.databind.Module;
|
||||
import com.google.common.collect.ImmutableList;
|
||||
import com.google.common.collect.ImmutableMap;
|
||||
import com.google.common.collect.ImmutableSet;
|
||||
import com.google.common.collect.Iterables;
|
||||
import com.google.inject.Injector;
|
||||
import org.apache.calcite.schema.SchemaPlus;
|
||||
import org.apache.druid.data.input.InputRow;
|
||||
import org.apache.druid.guice.DruidInjectorBuilder;
|
||||
import org.apache.druid.java.util.common.granularity.Granularities;
|
||||
import org.apache.druid.java.util.common.io.Closer;
|
||||
import org.apache.druid.query.Druids;
|
||||
@@ -27,66 +30,49 @@ import org.apache.druid.query.spec.MultipleIntervalSegmentSpec;
|
||||
import org.apache.druid.segment.IndexBuilder;
|
||||
import org.apache.druid.segment.QueryableIndex;
|
||||
import org.apache.druid.segment.TestHelper;
|
||||
import org.apache.druid.segment.column.ColumnType;
|
||||
import org.apache.druid.segment.column.ValueType;
|
||||
import org.apache.druid.segment.incremental.IncrementalIndexSchema;
|
||||
import org.apache.druid.segment.join.JoinableFactoryWrapper;
|
||||
import org.apache.druid.segment.virtual.ExpressionVirtualColumn;
|
||||
import org.apache.druid.segment.writeout.OffHeapMemorySegmentWriteOutMediumFactory;
|
||||
import org.apache.druid.server.QueryStackTests;
|
||||
import org.apache.druid.server.security.AuthTestUtils;
|
||||
import org.apache.druid.server.security.AuthenticationResult;
|
||||
import org.apache.druid.sql.SqlLifecycle;
|
||||
import org.apache.druid.sql.SqlLifecycleFactory;
|
||||
import org.apache.druid.sql.calcite.BaseCalciteQueryTest;
|
||||
import org.apache.druid.sql.calcite.QueryTestBuilder;
|
||||
import org.apache.druid.sql.calcite.QueryTestRunner;
|
||||
import org.apache.druid.sql.calcite.filtration.Filtration;
|
||||
import org.apache.druid.sql.calcite.planner.DruidOperatorTable;
|
||||
import org.apache.druid.sql.calcite.planner.PlannerConfig;
|
||||
import org.apache.druid.sql.calcite.planner.PlannerContext;
|
||||
import org.apache.druid.sql.calcite.planner.PlannerFactory;
|
||||
import org.apache.druid.sql.calcite.util.CalciteTestBase;
|
||||
import org.apache.druid.sql.calcite.util.CalciteTests;
|
||||
import org.apache.druid.sql.calcite.util.QueryLogHook;
|
||||
import org.apache.druid.sql.calcite.util.SpecificSegmentsQuerySegmentWalker;
|
||||
import org.apache.druid.sql.calcite.util.*;
|
||||
import org.apache.druid.timeline.DataSegment;
|
||||
import org.apache.druid.timeline.partition.LinearShardSpec;
|
||||
import org.junit.*;
|
||||
import org.junit.rules.TemporaryFolder;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.util.Arrays;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.*;
|
||||
|
||||
public class HdrHistogramQuantileSqlAggregatorTest extends CalciteTestBase {
|
||||
private static final String DATA_SOURCE = "foo";
|
||||
|
||||
private static QueryRunnerFactoryConglomerate conglomerate;
|
||||
private static Closer resourceCloser;
|
||||
private static AuthenticationResult authenticationResult = CalciteTests.REGULAR_USER_AUTH_RESULT;
|
||||
private static final Map<String, Object> QUERY_CONTEXT_DEFAULT = ImmutableMap.of(
|
||||
PlannerContext.CTX_SQL_QUERY_ID, "dummy"
|
||||
);
|
||||
|
||||
@Rule
|
||||
public TemporaryFolder temporaryFolder = new TemporaryFolder();
|
||||
|
||||
@Rule
|
||||
public QueryLogHook queryLogHook = QueryLogHook.create();
|
||||
|
||||
private SpecificSegmentsQuerySegmentWalker walker;
|
||||
private SqlLifecycleFactory sqlLifecycleFactory;
|
||||
|
||||
@BeforeClass
|
||||
public static void setUpClass() {
|
||||
resourceCloser = Closer.create();
|
||||
conglomerate = QueryStackTests.createQueryRunnerFactoryConglomerate(resourceCloser);
|
||||
public class HdrHistogramQuantileSqlAggregatorTest extends BaseCalciteQueryTest {
|
||||
@Override
|
||||
public void gatherProperties(Properties properties)
|
||||
{
|
||||
super.gatherProperties(properties);
|
||||
}
|
||||
|
||||
@AfterClass
|
||||
public static void tearDownClass() throws IOException {
|
||||
resourceCloser.close();
|
||||
@Override
|
||||
public void configureGuice(DruidInjectorBuilder builder)
|
||||
{
|
||||
super.configureGuice(builder);
|
||||
builder.addModule(new HdrHistogramModule());
|
||||
}
|
||||
|
||||
public static final List<InputRow> ROWS1 = ImmutableList.of(
|
||||
CalciteTests.createRow(
|
||||
TestDataBuilder.createRow(
|
||||
ImmutableMap.<String, Object>builder()
|
||||
.put("t", "2000-01-01")
|
||||
.put("m1", "1")
|
||||
@@ -96,7 +82,7 @@ public class HdrHistogramQuantileSqlAggregatorTest extends CalciteTestBase {
|
||||
.put("dim3", ImmutableList.of("a", "b"))
|
||||
.build()
|
||||
),
|
||||
CalciteTests.createRow(
|
||||
TestDataBuilder.createRow(
|
||||
ImmutableMap.<String, Object>builder()
|
||||
.put("t", "2000-01-02")
|
||||
.put("m1", "2.0")
|
||||
@@ -106,7 +92,7 @@ public class HdrHistogramQuantileSqlAggregatorTest extends CalciteTestBase {
|
||||
.put("dim3", ImmutableList.of("b", "c"))
|
||||
.build()
|
||||
),
|
||||
CalciteTests.createRow(
|
||||
TestDataBuilder.createRow(
|
||||
ImmutableMap.<String, Object>builder()
|
||||
.put("t", "2000-01-03")
|
||||
.put("m1", "3.0")
|
||||
@@ -116,7 +102,7 @@ public class HdrHistogramQuantileSqlAggregatorTest extends CalciteTestBase {
|
||||
.put("dim3", ImmutableList.of("d"))
|
||||
.build()
|
||||
),
|
||||
CalciteTests.createRow(
|
||||
TestDataBuilder.createRow(
|
||||
ImmutableMap.<String, Object>builder()
|
||||
.put("t", "2001-01-01")
|
||||
.put("m1", "4.0")
|
||||
@@ -126,7 +112,7 @@ public class HdrHistogramQuantileSqlAggregatorTest extends CalciteTestBase {
|
||||
.put("dim3", ImmutableList.of(""))
|
||||
.build()
|
||||
),
|
||||
CalciteTests.createRow(
|
||||
TestDataBuilder.createRow(
|
||||
ImmutableMap.<String, Object>builder()
|
||||
.put("t", "2001-01-02")
|
||||
.put("m1", "5.0")
|
||||
@@ -136,7 +122,7 @@ public class HdrHistogramQuantileSqlAggregatorTest extends CalciteTestBase {
|
||||
.put("dim3", ImmutableList.of())
|
||||
.build()
|
||||
),
|
||||
CalciteTests.createRow(
|
||||
TestDataBuilder.createRow(
|
||||
ImmutableMap.<String, Object>builder()
|
||||
.put("t", "2001-01-03")
|
||||
.put("m1", "6.0")
|
||||
@@ -146,15 +132,20 @@ public class HdrHistogramQuantileSqlAggregatorTest extends CalciteTestBase {
|
||||
)
|
||||
);
|
||||
|
||||
@Before
|
||||
public void setUp() throws Exception {
|
||||
@SuppressWarnings("resource")
|
||||
@Override
|
||||
public SpecificSegmentsQuerySegmentWalker createQuerySegmentWalker(
|
||||
final QueryRunnerFactoryConglomerate conglomerate,
|
||||
final JoinableFactoryWrapper joinableFactory,
|
||||
final Injector injector
|
||||
) throws IOException{
|
||||
HdrHistogramModule.registerSerde();
|
||||
for (Module mod : new HdrHistogramModule().getJacksonModules()) {
|
||||
CalciteTests.getJsonMapper().registerModule(mod);
|
||||
TestHelper.JSON_MAPPER.registerModule(mod);
|
||||
}
|
||||
|
||||
final QueryableIndex index = IndexBuilder.create()
|
||||
//final QueryableIndex index = TestHelper.getTestIndexIO().loadIndex(new File("D:/doc/datas/testIndex-6201298"));
|
||||
/*final QueryableIndex index = IndexBuilder.create()
|
||||
.tmpDir(temporaryFolder.newFolder())
|
||||
.segmentWriteOutMediumFactory(OffHeapMemorySegmentWriteOutMediumFactory.instance())
|
||||
.schema(
|
||||
@@ -176,81 +167,207 @@ public class HdrHistogramQuantileSqlAggregatorTest extends CalciteTestBase {
|
||||
)
|
||||
//.rows(CalciteTests.ROWS1)
|
||||
.rows(ROWS1)
|
||||
.buildMMappedIndex();
|
||||
.buildMMappedIndex();*/
|
||||
|
||||
walker = new SpecificSegmentsQuerySegmentWalker(conglomerate).add(
|
||||
DataSegment.builder()
|
||||
.dataSource(DATA_SOURCE)
|
||||
.interval(index.getDataInterval())
|
||||
.version("1")
|
||||
.shardSpec(new LinearShardSpec(0))
|
||||
.size(0)
|
||||
.build(),
|
||||
index
|
||||
);
|
||||
String[] files = new String[]{
|
||||
"D:\\doc\\datas\\statistics_rule_segments\\2023-10-16T00_00_00.000Z_2023-10-17T00_00_00.000Z\\2023-10-16T07_51_47.981Z\\0\\17a457e4-599d-49c2-86e7-6655851bb99a\\index",
|
||||
"D:\\doc\\datas\\statistics_rule_segments\\2023-10-15T00_00_00.000Z_2023-10-16T00_00_00.000Z\\2023-10-15T00_00_04.240Z\\15\\9a766f6c-779d-4f9f-9ff5-6a12c19b8c6c\\index"
|
||||
};
|
||||
files = new String[]{
|
||||
"D:/doc/datas/testIndex-6201298"
|
||||
};
|
||||
SpecificSegmentsQuerySegmentWalker walker = new SpecificSegmentsQuerySegmentWalker(conglomerate);
|
||||
|
||||
final PlannerConfig plannerConfig = new PlannerConfig();
|
||||
final DruidOperatorTable operatorTable = new DruidOperatorTable(
|
||||
ImmutableSet.of(
|
||||
new HdrHistogramQuantileSqlAggregator(),
|
||||
new HdrHistogramObjectSqlAggregator()
|
||||
),
|
||||
ImmutableSet.of(
|
||||
new HdrHistogramQuantilesOperatorConversion(),
|
||||
new HdrHistogramPercentilesOperatorConversion()
|
||||
)
|
||||
);
|
||||
SchemaPlus rootSchema =
|
||||
CalciteTests.createMockRootSchema(conglomerate, walker, plannerConfig, AuthTestUtils.TEST_AUTHORIZER_MAPPER);
|
||||
for (int i = 0; i < files.length; i++) {
|
||||
QueryableIndex index = TestHelper.getTestIndexIO().loadIndex(new File(files[i]));
|
||||
return walker.add(
|
||||
DataSegment.builder()
|
||||
.dataSource(CalciteTests.DATASOURCE1)
|
||||
.interval(index.getDataInterval())
|
||||
.version("1")
|
||||
.shardSpec(new LinearShardSpec(i))
|
||||
.size(0)
|
||||
.build(),
|
||||
index
|
||||
);
|
||||
}
|
||||
|
||||
sqlLifecycleFactory = CalciteTests.createSqlLifecycleFactory(
|
||||
new PlannerFactory(
|
||||
rootSchema,
|
||||
CalciteTests.createMockQueryLifecycleFactory(walker, conglomerate),
|
||||
operatorTable,
|
||||
CalciteTests.createExprMacroTable(),
|
||||
plannerConfig,
|
||||
AuthTestUtils.TEST_AUTHORIZER_MAPPER,
|
||||
CalciteTests.getJsonMapper(),
|
||||
CalciteTests.DRUID_SCHEMA_NAME
|
||||
)
|
||||
);
|
||||
return walker;
|
||||
}
|
||||
|
||||
@After
|
||||
public void tearDown() throws Exception {
|
||||
walker.close();
|
||||
walker = null;
|
||||
@Test
|
||||
public void testCount0() throws Exception {
|
||||
String sql = "select count(1) cnt, APPROX_QUANTILE_HDR(hist_m1, 0.5, 1, 100, 2) from druid.foo where dim1 = 'aaa'";
|
||||
QueryTestBuilder builder = testBuilder().sql(sql).skipVectorize();
|
||||
builder.run();
|
||||
QueryTestRunner.QueryResults queryResults = builder.results();
|
||||
List<Object[]> results = queryResults.results;
|
||||
for (Object[] result : results) {
|
||||
System.out.println(Arrays.toString(result));
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testSqlQueryError() throws Exception {
|
||||
String sql = "select min(__time) min_time,max(__time) max_time, HDR_HISTOGRAM(latency_ms_sketch) hdr from druid.foo";
|
||||
QueryTestBuilder builder = testBuilder().sql(sql).skipVectorize();
|
||||
builder.run();
|
||||
QueryTestRunner.QueryResults queryResults = builder.results();
|
||||
List<Object[]> results = queryResults.results;
|
||||
for (Object[] result : results) {
|
||||
System.out.println(Arrays.toString(result));
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testSqlDESCRIBE() throws Exception {
|
||||
String sql = "select HDR_GET_QUANTILES(HDR_HISTOGRAM(m1, 1, 100, 2), 0, 0.25, 0.5, 0.75, 1) a, HDR_DESCRIBE(HDR_HISTOGRAM(m1, 1, 100, 2)) b, HDR_DESCRIBE(HDR_HISTOGRAM(hist_m1, 1, 100, 2)) c from druid.foo";
|
||||
QueryTestBuilder builder = testBuilder().sql(sql).skipVectorize();
|
||||
builder.run();
|
||||
QueryTestRunner.QueryResults queryResults = builder.results();
|
||||
List<Object[]> results = queryResults.results;
|
||||
for (Object[] result : results) {
|
||||
System.out.println(Arrays.toString(result));
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testSqlDESCRIBE2() throws Exception {
|
||||
String sql = "select HDR_GET_QUANTILES(HDR_HISTOGRAM(m1, 1, 100, 2), 0, 0.25, 0.5, 0.75, 1) a, HDR_GET_PERCENTILES_DESCRIPTION(HDR_HISTOGRAM(m1, 1, 100, 2)) b, HDR_GET_PERCENTILES_DESCRIPTION(HDR_HISTOGRAM(hist_m1, 1, 100, 2)) c from druid.foo";
|
||||
QueryTestBuilder builder = testBuilder().sql(sql).skipVectorize();
|
||||
builder.run();
|
||||
QueryTestRunner.QueryResults queryResults = builder.results();
|
||||
List<Object[]> results = queryResults.results;
|
||||
for (Object[] result : results) {
|
||||
System.out.println(Arrays.toString(result));
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testSqlQuery() throws Exception {
|
||||
SqlLifecycle sqlLifecycle = sqlLifecycleFactory.factorize();
|
||||
String sql = "select * from druid.foo";
|
||||
final List<Object[]> results =
|
||||
sqlLifecycle.runSimple(sql, QUERY_CONTEXT_DEFAULT, DEFAULT_PARAMETERS, authenticationResult).toList();
|
||||
String[] columns = new String[]{"__time", "dim1", "dim2", "dim3", "cnt", "hist_m1", "m1"};
|
||||
String sql = "select " + String.join(",", columns) + " from druid.foo";
|
||||
QueryTestBuilder builder = testBuilder().sql(sql);
|
||||
builder.run();
|
||||
QueryTestRunner.QueryResults queryResults = builder.results();
|
||||
List<Object[]> results = queryResults.results;
|
||||
for (Object[] result : results) {
|
||||
Map row = new LinkedHashMap();
|
||||
for (int i = 0; i < result.length; i++) {
|
||||
row.put(columns[i], result[i]);
|
||||
}
|
||||
System.out.println(JSON.toJSONString(row));
|
||||
// System.out.println(Arrays.toString(result));
|
||||
}
|
||||
|
||||
for (int i = 0; i < columns.length; i++) {
|
||||
Object[] values = new Object[results.size()];
|
||||
for (int j = 0; j < results.size(); j++) {
|
||||
values[j] = results.get(j)[i];
|
||||
}
|
||||
System.out.println(columns[i] + ":" + Arrays.toString(values));
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testSqlQuery3() throws Exception {
|
||||
//cannotVectorize();
|
||||
//String sql = "select HLLD_ESTIMATE(HLLD(hll_dim1)) from druid.foo where dim1 = ''";
|
||||
String sql = "select HDR_HISTOGRAM(hist_m1) hdr from druid.foo ";
|
||||
QueryTestBuilder builder = testBuilder().sql(sql).skipVectorize();
|
||||
builder.run();
|
||||
QueryTestRunner.QueryResults queryResults = builder.results();
|
||||
List<Object[]> results = queryResults.results;
|
||||
for (Object[] result : results) {
|
||||
System.out.println(Arrays.toString(result));
|
||||
}
|
||||
}
|
||||
@Test
|
||||
public void testSqlQuery4() throws Exception {
|
||||
//cannotVectorize();
|
||||
//String sql = "select HLLD_ESTIMATE(HLLD(hll_dim1)) from druid.foo where dim1 = ''";
|
||||
String sql = "select APPROX_QUANTILE_HDR (hdr, 0.95) as p95th_tcp_latency_ms from (select HDR_HISTOGRAM(hist_m1) hdr from druid.foo) t ";
|
||||
QueryTestBuilder builder = testBuilder().sql(sql).skipVectorize();
|
||||
builder.run();
|
||||
QueryTestRunner.QueryResults queryResults = builder.results();
|
||||
List<Object[]> results = queryResults.results;
|
||||
for (Object[] result : results) {
|
||||
System.out.println(Arrays.toString(result));
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGroup() throws Exception {
|
||||
SqlLifecycle sqlLifecycle = sqlLifecycleFactory.factorize();
|
||||
String sql = "select cnt, APPROX_QUANTILE_HDR(hist_m1, 0.5, 1, 100, 2) from druid.foo group by cnt";
|
||||
final List<Object[]> results =
|
||||
sqlLifecycle.runSimple(sql, QUERY_CONTEXT_DEFAULT, DEFAULT_PARAMETERS, authenticationResult).toList();
|
||||
public void testSqlQuery5() throws Exception {
|
||||
//cannotVectorize();
|
||||
//String sql = "select HLLD_ESTIMATE(HLLD(hll_dim1)) from druid.foo where dim1 = ''";
|
||||
String sql = "select dim1, APPROX_QUANTILE_HDR (hdr, 0.95) as p95th_tcp_latency_ms from (select dim1, HDR_HISTOGRAM(hist_m1) hdr from druid.foo group by dim1) t group by dim1";
|
||||
QueryTestBuilder builder = testBuilder().sql(sql).skipVectorize();
|
||||
builder.run();
|
||||
QueryTestRunner.QueryResults queryResults = builder.results();
|
||||
List<Object[]> results = queryResults.results;
|
||||
for (Object[] result : results) {
|
||||
System.out.println(Arrays.toString(result));
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testSqlQuery6() throws Exception {
|
||||
//cannotVectorize();
|
||||
//String sql = "select HLLD_ESTIMATE(HLLD(hll_dim1)) from druid.foo where dim1 = ''";
|
||||
//String sql = "select dim1, APPROX_QUANTILE_HDR (hdr, 0.95) as p95th_tcp_latency_ms from (select dim1, HDR_HISTOGRAM(hist_m1) hdr from druid.foo group by dim1 limit 10) t group by dim1";
|
||||
String sql = "select dim1, HDR_GET_QUANTILES(HDR_HISTOGRAM(hdr), 0.95) as p95th_tcp_latency_ms from (select dim1, HDR_HISTOGRAM(hist_m1) hdr from druid.foo group by dim1 limit 10) t group by dim1";
|
||||
QueryTestBuilder builder = testBuilder().sql(sql).skipVectorize();
|
||||
builder.run();
|
||||
QueryTestRunner.QueryResults queryResults = builder.results();
|
||||
List<Object[]> results = queryResults.results;
|
||||
for (Object[] result : results) {
|
||||
System.out.println(Arrays.toString(result));
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGroup() throws Exception {
|
||||
String sql = "select cnt, APPROX_QUANTILE_HDR(hist_m1, 0.5, 1, 100, 2) from druid.foo group by cnt";
|
||||
QueryTestBuilder builder = testBuilder().sql(sql).skipVectorize();
|
||||
builder.run();
|
||||
QueryTestRunner.QueryResults queryResults = builder.results();
|
||||
List<Object[]> results = queryResults.results;
|
||||
for (Object[] result : results) {
|
||||
System.out.println(Arrays.toString(result));
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGroup2() throws Exception {
|
||||
SqlLifecycle sqlLifecycle = sqlLifecycleFactory.factorize();
|
||||
String sql = "select HDR_HISTOGRAM(hist_m1) from druid.foo";
|
||||
final List<Object[]> results =
|
||||
sqlLifecycle.runSimple(sql, QUERY_CONTEXT_DEFAULT, DEFAULT_PARAMETERS, authenticationResult).toList();
|
||||
QueryTestBuilder builder = testBuilder().sql(sql).skipVectorize();
|
||||
builder.run();
|
||||
QueryTestRunner.QueryResults queryResults = builder.results();
|
||||
List<Object[]> results = queryResults.results;
|
||||
for (Object[] result : results) {
|
||||
System.out.println(Arrays.toString(result));
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGroup3() throws Exception {
|
||||
String sql = "select APPROX_QUANTILE_HDR(h, 0.5) from(select HDR_HISTOGRAM(hist_m1) h from druid.foo) t";
|
||||
QueryTestBuilder builder = testBuilder().sql(sql).skipVectorize();
|
||||
builder.run();
|
||||
QueryTestRunner.QueryResults queryResults = builder.results();
|
||||
List<Object[]> results = queryResults.results;
|
||||
for (Object[] result : results) {
|
||||
System.out.println(Arrays.toString(result));
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGroup4() throws Exception {
|
||||
String sql = "select hdr_get_quantiles(h, 0.1, 0.2, 0.3, 0.5, 0.9, 0.99, 1) from(select HDR_HISTOGRAM(hist_m1) h from druid.foo) t";
|
||||
QueryTestBuilder builder = testBuilder().sql(sql).skipVectorize();
|
||||
builder.run();
|
||||
QueryTestRunner.QueryResults queryResults = builder.results();
|
||||
List<Object[]> results = queryResults.results;
|
||||
for (Object[] result : results) {
|
||||
System.out.println(Arrays.toString(result));
|
||||
}
|
||||
@@ -258,10 +375,11 @@ public class HdrHistogramQuantileSqlAggregatorTest extends CalciteTestBase {
|
||||
|
||||
@Test
|
||||
public void testSqlQueryGeneHdr() throws Exception {
|
||||
SqlLifecycle sqlLifecycle = sqlLifecycleFactory.factorize();
|
||||
String sql = "select HDR_HISTOGRAM(hist_m1, 1, 100, 2), HDR_HISTOGRAM(cnt, 1, 100, 2) from druid.foo";
|
||||
final List<Object[]> results =
|
||||
sqlLifecycle.runSimple(sql, QUERY_CONTEXT_DEFAULT, DEFAULT_PARAMETERS, authenticationResult).toList();
|
||||
QueryTestBuilder builder = testBuilder().sql(sql).skipVectorize();
|
||||
builder.run();
|
||||
QueryTestRunner.QueryResults queryResults = builder.results();
|
||||
List<Object[]> results = queryResults.results;
|
||||
for (Object[] result : results) {
|
||||
System.out.println(Arrays.toString(result));
|
||||
}
|
||||
@@ -269,11 +387,12 @@ public class HdrHistogramQuantileSqlAggregatorTest extends CalciteTestBase {
|
||||
|
||||
@Test
|
||||
public void testSqlQueryGeneHdr2() throws Exception {
|
||||
SqlLifecycle sqlLifecycle = sqlLifecycleFactory.factorize();
|
||||
// HDR_HISTOGRAM(hist_m1, 1, 100, 2),
|
||||
String sql = "select HDR_GET_QUANTILES(HDR_HISTOGRAM(m1, 1, 100, 2), 0.1, 0.2, 0.3, 0.5, 0.9, 1) from druid.foo";
|
||||
final List<Object[]> results =
|
||||
sqlLifecycle.runSimple(sql, QUERY_CONTEXT_DEFAULT, DEFAULT_PARAMETERS, authenticationResult).toList();
|
||||
QueryTestBuilder builder = testBuilder().sql(sql).skipVectorize();
|
||||
builder.run();
|
||||
QueryTestRunner.QueryResults queryResults = builder.results();
|
||||
List<Object[]> results = queryResults.results;
|
||||
for (Object[] result : results) {
|
||||
System.out.println(Arrays.toString(result));
|
||||
}
|
||||
@@ -281,44 +400,47 @@ public class HdrHistogramQuantileSqlAggregatorTest extends CalciteTestBase {
|
||||
|
||||
@Test
|
||||
public void testSqlQueryGeneHdrArgs() throws Exception {
|
||||
SqlLifecycle sqlLifecycle = sqlLifecycleFactory.factorize();
|
||||
String sql = "select HDR_GET_QUANTILEs(HDR_HISTOGRAM(m1), 0.1, 0.2, 0.3, 0.5, 0.9, 1), "
|
||||
+ "HDR_GET_QUANTILEs(HDR_HISTOGRAM(m1, 2), 0.1, 0.2, 0.3, 0.5, 0.9, 1) ,\n"
|
||||
+ "HDR_GET_QUANTILEs(HDR_HISTOGRAM(m1, 1, 110, 2), 0.1, 0.2, 0.3, 0.5, 0.9, 1) ,\n"
|
||||
+ "HDR_GET_QUANTILEs(HDR_HISTOGRAM(m1, 1, 110, 2, false), 0.1, 0.2, 0.3, 0.5, 0.9, 1) \n"
|
||||
+ "from druid.foo";
|
||||
final List<Object[]> results =
|
||||
sqlLifecycle.runSimple(sql, QUERY_CONTEXT_DEFAULT, DEFAULT_PARAMETERS, authenticationResult).toList();
|
||||
QueryTestBuilder builder = testBuilder().sql(sql).skipVectorize();
|
||||
builder.run();
|
||||
QueryTestRunner.QueryResults queryResults = builder.results();
|
||||
List<Object[]> results = queryResults.results;
|
||||
for (Object[] result : results) {
|
||||
System.out.println(Arrays.toString(result));
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
@Test
|
||||
public void testSqlQueryGeneHdrArgs2() throws Exception {
|
||||
SqlLifecycle sqlLifecycle = sqlLifecycleFactory.factorize();
|
||||
String sql = "select APPROX_QUANTILE_HDR(m1, 0.1), "
|
||||
+ "APPROX_QUANTILE_HDR(m1, 0.1, 2) ,\n"
|
||||
+ "APPROX_QUANTILE_HDR(m1, 0.1, 1, 110, 2) ,\n"
|
||||
+ "APPROX_QUANTILE_HDR(m1, 0.1, 1, 110, 2, false)\n"
|
||||
+ "from druid.foo";
|
||||
final List<Object[]> results =
|
||||
sqlLifecycle.runSimple(sql, QUERY_CONTEXT_DEFAULT, DEFAULT_PARAMETERS, authenticationResult).toList();
|
||||
for (Object[] result : results) {
|
||||
System.out.println(Arrays.toString(result));
|
||||
}
|
||||
QueryTestBuilder builder = testBuilder().sql(sql).skipVectorize();
|
||||
builder.run();
|
||||
QueryTestRunner.QueryResults queryResults = builder.results();
|
||||
List<Object[]> results = queryResults.results;
|
||||
for (Object[] result : results) {
|
||||
System.out.println(Arrays.toString(result));
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testSqlQueryGeneHdr3() throws Exception {
|
||||
SqlLifecycle sqlLifecycle = sqlLifecycleFactory.factorize();
|
||||
// 函数不区分大小写
|
||||
// HDR_HISTOGRAM(hist_m1, 1, 100, 2),
|
||||
//String sql = "select HDR_GET_PERCENTILES(HDR_HISTOGRAM(m1, 1, 100, 2)) from druid.foo";
|
||||
//String sql = "select hdr_get_percentiles(hdr_histogram(m1, 1, 100, 2)) from druid.foo";
|
||||
String sql = "select hdr_get_percentiles(hdr_histogram(hist_m1, 1, 100, 2)) from druid.foo";
|
||||
final List<Object[]> results =
|
||||
sqlLifecycle.runSimple(sql, QUERY_CONTEXT_DEFAULT, DEFAULT_PARAMETERS, authenticationResult).toList();
|
||||
QueryTestBuilder builder = testBuilder().sql(sql).skipVectorize();
|
||||
builder.run();
|
||||
QueryTestRunner.QueryResults queryResults = builder.results();
|
||||
List<Object[]> results = queryResults.results;
|
||||
for (Object[] result : results) {
|
||||
System.out.println(Arrays.toString(result));
|
||||
}
|
||||
@@ -326,7 +448,6 @@ public class HdrHistogramQuantileSqlAggregatorTest extends CalciteTestBase {
|
||||
|
||||
@Test
|
||||
public void testSqlQueryQuantiles() throws Exception {
|
||||
SqlLifecycle sqlLifecycle = sqlLifecycleFactory.factorize();
|
||||
String sql = "SELECT\n"
|
||||
+ "APPROX_QUANTILE_HDR(m1, 0.01, 1, 100, 2),\n"
|
||||
+ "APPROX_QUANTILE_HDR(m1, 0.5, 1, 100, 2),\n"
|
||||
@@ -338,9 +459,10 @@ public class HdrHistogramQuantileSqlAggregatorTest extends CalciteTestBase {
|
||||
+ "APPROX_QUANTILE_HDR(m1, 0.999, 1, 100, 2) FILTER(WHERE dim1 = 'abc'),\n"
|
||||
+ "APPROX_QUANTILE_HDR(cnt, 0.5, 1, 100, 2)\n"
|
||||
+ "FROM foo";
|
||||
final List<Object[]> results =
|
||||
sqlLifecycle.runSimple(sql, QUERY_CONTEXT_DEFAULT, DEFAULT_PARAMETERS, authenticationResult).toList();
|
||||
System.out.println(sql);
|
||||
QueryTestBuilder builder = testBuilder().sql(sql).skipVectorize();
|
||||
builder.run();
|
||||
QueryTestRunner.QueryResults queryResults = builder.results();
|
||||
List<Object[]> results = queryResults.results;
|
||||
for (Object[] result : results) {
|
||||
System.out.println(Arrays.toString(result));
|
||||
}
|
||||
@@ -348,7 +470,6 @@ public class HdrHistogramQuantileSqlAggregatorTest extends CalciteTestBase {
|
||||
|
||||
@Test
|
||||
public void testSqlQueryQuantilesOnComplexColumn() throws Exception {
|
||||
SqlLifecycle sqlLifecycle = sqlLifecycleFactory.factorize();
|
||||
String sql = "SELECT\n"
|
||||
+ "APPROX_QUANTILE_HDR(hist_m1, 0.01, 1, 100, 2),\n"
|
||||
+ "APPROX_QUANTILE_HDR(hist_m1, 0.5, 1, 100, 2),\n"
|
||||
@@ -358,9 +479,10 @@ public class HdrHistogramQuantileSqlAggregatorTest extends CalciteTestBase {
|
||||
+ "APPROX_QUANTILE_HDR(hist_m1, 0.999, 1, 100, 2) FILTER(WHERE dim1 <> 'abc'),\n"
|
||||
+ "APPROX_QUANTILE_HDR(hist_m1, 0.999, 1, 100, 2) FILTER(WHERE dim1 = 'abc')\n"
|
||||
+ "FROM foo";
|
||||
final List<Object[]> results =
|
||||
sqlLifecycle.runSimple(sql, QUERY_CONTEXT_DEFAULT, DEFAULT_PARAMETERS, authenticationResult).toList();
|
||||
System.out.println(sql);
|
||||
QueryTestBuilder builder = testBuilder().sql(sql).skipVectorize();
|
||||
builder.run();
|
||||
QueryTestRunner.QueryResults queryResults = builder.results();
|
||||
List<Object[]> results = queryResults.results;
|
||||
for (Object[] result : results) {
|
||||
System.out.println(Arrays.toString(result));
|
||||
}
|
||||
@@ -373,7 +495,6 @@ public class HdrHistogramQuantileSqlAggregatorTest extends CalciteTestBase {
|
||||
|
||||
@Test
|
||||
public void testQuantileOnFloatAndLongs() throws Exception {
|
||||
SqlLifecycle sqlLifecycle = sqlLifecycleFactory.factorize();
|
||||
String sql = "SELECT\n"
|
||||
+ "APPROX_QUANTILE_HDR(m1, 0.01, 1, 100, 2),\n"
|
||||
+ "APPROX_QUANTILE_HDR(m1, 0.5, 1, 100, 2),\n"
|
||||
@@ -385,60 +506,55 @@ public class HdrHistogramQuantileSqlAggregatorTest extends CalciteTestBase {
|
||||
+ "APPROX_QUANTILE_HDR(m1, 0.999, 1, 100, 2) FILTER(WHERE dim1 = 'abc'),\n"
|
||||
+ "APPROX_QUANTILE_HDR(cnt, 0.5, 1, 100, 2)\n"
|
||||
+ "FROM foo";
|
||||
final List<Object[]> results =
|
||||
sqlLifecycle.runSimple(sql, QUERY_CONTEXT_DEFAULT, DEFAULT_PARAMETERS, authenticationResult).toList();
|
||||
System.out.println(sql);
|
||||
QueryTestBuilder builder = testBuilder().sql(sql).skipVectorize();
|
||||
builder = builder.expectedQueries(Collections.singletonList(Druids.newTimeseriesQueryBuilder()
|
||||
.dataSource(CalciteTests.DATASOURCE1)
|
||||
.intervals(new MultipleIntervalSegmentSpec(ImmutableList.of(Filtration.eternity())))
|
||||
.granularity(Granularities.ALL)
|
||||
.virtualColumns(
|
||||
new ExpressionVirtualColumn(
|
||||
"v0",
|
||||
"(\"m1\" * 2)",
|
||||
ColumnType.LONG,
|
||||
TestExprMacroTable.INSTANCE
|
||||
)
|
||||
)
|
||||
.aggregators(ImmutableList.of(
|
||||
new HdrHistogramAggregatorFactory("a0:agg", "m1", 1L, 100L, 2, true),
|
||||
new HdrHistogramAggregatorFactory("a4:agg", "v0", 1L, 100L, 2, true),
|
||||
new FilteredAggregatorFactory(
|
||||
new HdrHistogramAggregatorFactory("a5:agg", "m1", 1L, 100L, 2, true),
|
||||
new SelectorDimFilter("dim1", "abc", null)
|
||||
),
|
||||
new FilteredAggregatorFactory(
|
||||
new HdrHistogramAggregatorFactory("a6:agg", "m1", 1L, 100L, 2, true),
|
||||
new NotDimFilter(new SelectorDimFilter("dim1", "abc", null))
|
||||
),
|
||||
new HdrHistogramAggregatorFactory("a8:agg", "cnt", 1L, 100L, 2, true)
|
||||
))
|
||||
.postAggregators(
|
||||
new HdrHistogramToQuantilePostAggregator("a0", "a0:agg", 0.01f),
|
||||
new HdrHistogramToQuantilePostAggregator("a1", "a0:agg", 0.50f),
|
||||
new HdrHistogramToQuantilePostAggregator("a2", "a0:agg", 0.98f),
|
||||
new HdrHistogramToQuantilePostAggregator("a3", "a0:agg", 0.99f),
|
||||
new HdrHistogramToQuantilePostAggregator("a4", "a4:agg", 0.97f),
|
||||
new HdrHistogramToQuantilePostAggregator("a5", "a5:agg", 0.99f),
|
||||
new HdrHistogramToQuantilePostAggregator("a6", "a6:agg", 0.999f),
|
||||
new HdrHistogramToQuantilePostAggregator("a7", "a5:agg", 0.999f),
|
||||
new HdrHistogramToQuantilePostAggregator("a8", "a8:agg", 0.50f)
|
||||
)
|
||||
.context(QUERY_CONTEXT_DEFAULT)
|
||||
.build()));
|
||||
builder.run();
|
||||
QueryTestRunner.QueryResults queryResults = builder.results();
|
||||
List<Object[]> results = queryResults.results;
|
||||
for (Object[] result : results) {
|
||||
System.out.println(Arrays.toString(result));
|
||||
}
|
||||
|
||||
// Verify query
|
||||
Assert.assertEquals(
|
||||
Druids.newTimeseriesQueryBuilder()
|
||||
.dataSource(CalciteTests.DATASOURCE1)
|
||||
.intervals(new MultipleIntervalSegmentSpec(ImmutableList.of(Filtration.eternity())))
|
||||
.granularity(Granularities.ALL)
|
||||
.virtualColumns(
|
||||
new ExpressionVirtualColumn(
|
||||
"v0",
|
||||
"(\"m1\" * 2)",
|
||||
ValueType.LONG,
|
||||
TestExprMacroTable.INSTANCE
|
||||
)
|
||||
)
|
||||
.aggregators(ImmutableList.of(
|
||||
new HdrHistogramAggregatorFactory("a0:agg", "m1", 1L, 100L, 2, true),
|
||||
new HdrHistogramAggregatorFactory("a4:agg", "v0", 1L, 100L, 2, true),
|
||||
new FilteredAggregatorFactory(
|
||||
new HdrHistogramAggregatorFactory("a5:agg", "m1", 1L, 100L, 2, true),
|
||||
new SelectorDimFilter("dim1", "abc", null)
|
||||
),
|
||||
new FilteredAggregatorFactory(
|
||||
new HdrHistogramAggregatorFactory("a6:agg", "m1", 1L, 100L, 2, true),
|
||||
new NotDimFilter(new SelectorDimFilter("dim1", "abc", null))
|
||||
),
|
||||
new HdrHistogramAggregatorFactory("a8:agg", "cnt", 1L, 100L, 2, true)
|
||||
))
|
||||
.postAggregators(
|
||||
new HdrHistogramToQuantilePostAggregator("a0", "a0:agg", 0.01f),
|
||||
new HdrHistogramToQuantilePostAggregator("a1", "a0:agg", 0.50f),
|
||||
new HdrHistogramToQuantilePostAggregator("a2", "a0:agg", 0.98f),
|
||||
new HdrHistogramToQuantilePostAggregator("a3", "a0:agg", 0.99f),
|
||||
new HdrHistogramToQuantilePostAggregator("a4", "a4:agg", 0.97f),
|
||||
new HdrHistogramToQuantilePostAggregator("a5", "a5:agg", 0.99f),
|
||||
new HdrHistogramToQuantilePostAggregator("a6", "a6:agg", 0.999f),
|
||||
new HdrHistogramToQuantilePostAggregator("a7", "a5:agg", 0.999f),
|
||||
new HdrHistogramToQuantilePostAggregator("a8", "a8:agg", 0.50f)
|
||||
)
|
||||
.context(ImmutableMap.of("skipEmptyBuckets", true, PlannerContext.CTX_SQL_QUERY_ID, "dummy"))
|
||||
.build(),
|
||||
Iterables.getOnlyElement(queryLogHook.getRecordedQueries())
|
||||
);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testQuantileOnComplexColumn() throws Exception{
|
||||
SqlLifecycle sqlLifecycle = sqlLifecycleFactory.factorize();
|
||||
String sql = "SELECT\n"
|
||||
+ "APPROX_QUANTILE_HDR(hist_m1, 0.01, 1, 100, 2),\n"
|
||||
+ "APPROX_QUANTILE_HDR(hist_m1, 0.5, 1, 100, 2),\n"
|
||||
@@ -448,43 +564,42 @@ public class HdrHistogramQuantileSqlAggregatorTest extends CalciteTestBase {
|
||||
+ "APPROX_QUANTILE_HDR(hist_m1, 0.999, 1, 100, 2) FILTER(WHERE dim1 <> 'abc'),\n"
|
||||
+ "APPROX_QUANTILE_HDR(hist_m1, 0.999, 1, 100, 2) FILTER(WHERE dim1 = 'abc')\n"
|
||||
+ "FROM foo";
|
||||
final List<Object[]> results =
|
||||
sqlLifecycle.runSimple(sql, QUERY_CONTEXT_DEFAULT, DEFAULT_PARAMETERS, authenticationResult).toList();
|
||||
System.out.println(sql);
|
||||
QueryTestBuilder builder = testBuilder().sql(sql).skipVectorize();
|
||||
builder = builder.expectedQueries(Collections.singletonList(Druids.newTimeseriesQueryBuilder()
|
||||
.dataSource(CalciteTests.DATASOURCE1)
|
||||
.intervals(new MultipleIntervalSegmentSpec(ImmutableList.of(Filtration.eternity())))
|
||||
.granularity(Granularities.ALL)
|
||||
.aggregators(ImmutableList.of(
|
||||
new HdrHistogramMergeAggregatorFactory("a0:agg", "hist_m1", 1L, 100L, 2, true),
|
||||
new FilteredAggregatorFactory(
|
||||
new HdrHistogramMergeAggregatorFactory("a4:agg", "hist_m1", 1L, 100L, 2, true),
|
||||
new SelectorDimFilter("dim1", "abc", null)
|
||||
),
|
||||
new FilteredAggregatorFactory(
|
||||
new HdrHistogramMergeAggregatorFactory("a5:agg", "hist_m1", 1L, 100L, 2, true),
|
||||
new NotDimFilter(new SelectorDimFilter("dim1", "abc", null))
|
||||
)
|
||||
))
|
||||
.postAggregators(
|
||||
new HdrHistogramToQuantilePostAggregator("a0", "a0:agg", 0.01f),
|
||||
new HdrHistogramToQuantilePostAggregator("a1", "a0:agg", 0.50f),
|
||||
new HdrHistogramToQuantilePostAggregator("a2", "a0:agg", 0.98f),
|
||||
new HdrHistogramToQuantilePostAggregator("a3", "a0:agg", 0.99f),
|
||||
new HdrHistogramToQuantilePostAggregator("a4", "a4:agg", 0.99f),
|
||||
new HdrHistogramToQuantilePostAggregator("a5", "a5:agg", 0.999f),
|
||||
new HdrHistogramToQuantilePostAggregator("a6", "a4:agg", 0.999f)
|
||||
)
|
||||
.context(QUERY_CONTEXT_DEFAULT)
|
||||
.build()));
|
||||
builder.run();
|
||||
QueryTestRunner.QueryResults queryResults = builder.results();
|
||||
List<Object[]> results = queryResults.results;
|
||||
for (Object[] result : results) {
|
||||
System.out.println(Arrays.toString(result));
|
||||
}
|
||||
|
||||
// Verify query
|
||||
Assert.assertEquals(
|
||||
Druids.newTimeseriesQueryBuilder()
|
||||
.dataSource(CalciteTests.DATASOURCE1)
|
||||
.intervals(new MultipleIntervalSegmentSpec(ImmutableList.of(Filtration.eternity())))
|
||||
.granularity(Granularities.ALL)
|
||||
.aggregators(ImmutableList.of(
|
||||
new HdrHistogramMergeAggregatorFactory("a0:agg", "hist_m1", 1L, 100L, 2, true),
|
||||
new FilteredAggregatorFactory(
|
||||
new HdrHistogramMergeAggregatorFactory("a4:agg", "hist_m1", 1L, 100L, 2, true),
|
||||
new SelectorDimFilter("dim1", "abc", null)
|
||||
),
|
||||
new FilteredAggregatorFactory(
|
||||
new HdrHistogramMergeAggregatorFactory("a5:agg", "hist_m1", 1L, 100L, 2, true),
|
||||
new NotDimFilter(new SelectorDimFilter("dim1", "abc", null))
|
||||
)
|
||||
))
|
||||
.postAggregators(
|
||||
new HdrHistogramToQuantilePostAggregator("a0", "a0:agg", 0.01f),
|
||||
new HdrHistogramToQuantilePostAggregator("a1", "a0:agg", 0.50f),
|
||||
new HdrHistogramToQuantilePostAggregator("a2", "a0:agg", 0.98f),
|
||||
new HdrHistogramToQuantilePostAggregator("a3", "a0:agg", 0.99f),
|
||||
new HdrHistogramToQuantilePostAggregator("a4", "a4:agg", 0.99f),
|
||||
new HdrHistogramToQuantilePostAggregator("a5", "a5:agg", 0.999f),
|
||||
new HdrHistogramToQuantilePostAggregator("a6", "a4:agg", 0.999f)
|
||||
)
|
||||
.context(ImmutableMap.of("skipEmptyBuckets", true, PlannerContext.CTX_SQL_QUERY_ID, "dummy"))
|
||||
.build(),
|
||||
Iterables.getOnlyElement(queryLogHook.getRecordedQueries())
|
||||
);
|
||||
|
||||
|
||||
}
|
||||
|
||||
private static PostAggregator makeFieldAccessPostAgg(String name) {
|
||||
|
||||
@@ -5,7 +5,7 @@
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
|
||||
<groupId>org.apache.druid.extensions</groupId>
|
||||
<artifactId>druid-hlld_0.18.1</artifactId>
|
||||
<artifactId>druid-hlld_26.0.0</artifactId>
|
||||
<name>druid-hlld</name>
|
||||
<version>1.0-SNAPSHOT</version>
|
||||
|
||||
@@ -14,7 +14,7 @@
|
||||
<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
|
||||
<maven.compiler.source>1.8</maven.compiler.source>
|
||||
<maven.compiler.target>1.8</maven.compiler.target>
|
||||
<druid.version>0.18.1</druid.version>
|
||||
<druid.version>26.0.0</druid.version>
|
||||
</properties>
|
||||
|
||||
<dependencies>
|
||||
@@ -33,6 +33,14 @@
|
||||
</dependency>
|
||||
|
||||
<!-- Tests -->
|
||||
|
||||
<dependency>
|
||||
<groupId>org.easymock</groupId>
|
||||
<artifactId>easymock</artifactId>
|
||||
<version>4.3</version>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.apache.druid</groupId>
|
||||
<artifactId>druid-processing</artifactId>
|
||||
@@ -42,9 +50,17 @@
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.druid</groupId>
|
||||
<artifactId>druid-benchmarks</artifactId>
|
||||
<artifactId>druid-server</artifactId>
|
||||
<version>${druid.version}</version>
|
||||
<scope>test</scope>
|
||||
<type>test-jar</type>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.druid</groupId>
|
||||
<artifactId>druid-sql</artifactId>
|
||||
<version>${druid.version}</version>
|
||||
<type>test-jar</type>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>junit</groupId>
|
||||
|
||||
@@ -1,256 +1,287 @@
|
||||
package org.apache.druid.query.aggregation.sketch.hlld;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||
import com.zdjz.galaxy.sketch.hlld.Hll;
|
||||
import com.zdjz.galaxy.sketch.hlld.HllUnion;
|
||||
import org.apache.druid.java.util.common.IAE;
|
||||
import org.apache.druid.java.util.common.logger.Logger;
|
||||
import org.apache.druid.query.aggregation.*;
|
||||
import org.apache.druid.query.cache.CacheKeyBuilder;
|
||||
import org.apache.druid.segment.ColumnSelectorFactory;
|
||||
import org.apache.druid.segment.ColumnValueSelector;
|
||||
|
||||
import javax.annotation.Nullable;
|
||||
import java.util.Collections;
|
||||
import java.util.Comparator;
|
||||
import java.util.List;
|
||||
import java.util.Objects;
|
||||
|
||||
public class HllAggregatorFactory extends AggregatorFactory {
|
||||
private static final Logger LOG = new Logger(HllAggregatorFactory.class);
|
||||
public static final boolean DEFAULT_ROUND = false;
|
||||
public static final int DEFAULT_PRECISION = 12;
|
||||
|
||||
static final Comparator<Hll> COMPARATOR = Comparator.nullsFirst(Comparator.comparingDouble(Hll::size));
|
||||
|
||||
protected final String name;
|
||||
protected final String fieldName;
|
||||
protected final int precision;
|
||||
protected final boolean round;
|
||||
|
||||
public HllAggregatorFactory(
|
||||
@JsonProperty("name") final String name,
|
||||
@JsonProperty("fieldName") final String fieldName,
|
||||
@JsonProperty("precision") @Nullable final Integer precision,
|
||||
@JsonProperty("round") @Nullable final Boolean round
|
||||
) {
|
||||
if (name == null) {
|
||||
throw new IAE("Must have a valid, non-null aggregator name");
|
||||
}
|
||||
if (fieldName == null) {
|
||||
throw new IAE("Parameter fieldName must be specified");
|
||||
}
|
||||
this.name = name;
|
||||
this.fieldName = fieldName;
|
||||
this.precision = precision == null ? DEFAULT_PRECISION : precision;
|
||||
this.round = round == null ? DEFAULT_ROUND : round;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Aggregator factorize(ColumnSelectorFactory columnSelectorFactory) {
|
||||
final ColumnValueSelector<Object> selector = columnSelectorFactory.makeColumnValueSelector(fieldName);
|
||||
return new HllAggregator(selector, precision);
|
||||
}
|
||||
|
||||
@Override
|
||||
public BufferAggregator factorizeBuffered(ColumnSelectorFactory columnSelectorFactory) {
|
||||
final ColumnValueSelector<Object> selector = columnSelectorFactory.makeColumnValueSelector(fieldName);
|
||||
return new HllBufferAggregator(
|
||||
selector,
|
||||
precision
|
||||
);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Comparator getComparator() {
|
||||
return COMPARATOR;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Object combine(Object lhs, Object rhs) {
|
||||
if(lhs == null){
|
||||
return rhs;
|
||||
}else if(rhs == null){
|
||||
return lhs;
|
||||
}else{
|
||||
final HllUnion union = new HllUnion(precision);
|
||||
union.update((Hll) lhs);
|
||||
union.update((Hll) rhs);
|
||||
Hll result = union.getResult();
|
||||
return result;
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public AggregateCombiner makeAggregateCombiner() {
|
||||
return new ObjectAggregateCombiner<Hll>() {
|
||||
private HllUnion union = null;
|
||||
|
||||
@Override
|
||||
public void reset(ColumnValueSelector selector) {
|
||||
//LOG.error("HllAggregateCombiner reset:" + "-" + Thread.currentThread().getId() + "-" + this);
|
||||
//union.reset();
|
||||
union = null;
|
||||
fold(selector);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void fold(ColumnValueSelector selector) {
|
||||
//LOG.error("HllAggregateCombiner fold:" + "-" + Thread.currentThread().getId() + "-" + this);
|
||||
final Hll hll = (Hll) selector.getObject();
|
||||
if(hll != null){
|
||||
if(union == null){
|
||||
union = new HllUnion(precision);
|
||||
}
|
||||
union.update(hll);
|
||||
}else{
|
||||
//LOG.error("HllAggregateCombiner fold_null:" + "-" + Thread.currentThread().getId() + "-" + this);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public Class<Hll> classOfObject() {
|
||||
return Hll.class;
|
||||
}
|
||||
|
||||
@Nullable
|
||||
@Override
|
||||
public Hll getObject() {
|
||||
//LOG.error("HllAggregateCombiner get:" + "-" + Thread.currentThread().getId() + "-" + this);
|
||||
if(union == null){
|
||||
return null;
|
||||
}else{
|
||||
Hll result = union.getResult();
|
||||
/*if(result.size() == 0){
|
||||
return null;
|
||||
}*/
|
||||
return result;
|
||||
}
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
@Override
|
||||
public AggregatorFactory getCombiningFactory() {
|
||||
// 千万不能写错，好大一个坑
|
||||
return new HllMergeAggregatorFactory(name, name, precision, round);
|
||||
}
|
||||
|
||||
@Override
|
||||
public AggregatorFactory getMergingFactory(AggregatorFactory other) throws AggregatorFactoryNotMergeableException {
|
||||
if (other.getName().equals(this.getName()) && other instanceof HllAggregatorFactory) {
|
||||
HllAggregatorFactory castedOther = (HllAggregatorFactory) other;
|
||||
|
||||
return new HllMergeAggregatorFactory(name, name,
|
||||
Math.max(precision, castedOther.precision),
|
||||
round || castedOther.round
|
||||
);
|
||||
} else {
|
||||
throw new AggregatorFactoryNotMergeableException(this, other);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<AggregatorFactory> getRequiredColumns() {
|
||||
return Collections.singletonList(
|
||||
new HllAggregatorFactory(fieldName, fieldName, precision, round)
|
||||
);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Object deserialize(Object object) {
|
||||
return HllUtils.deserializeHll(object);
|
||||
}
|
||||
|
||||
@Nullable
|
||||
@Override
|
||||
public Object finalizeComputation(@Nullable Object object) {
|
||||
if (object == null) {
|
||||
return null;
|
||||
}
|
||||
final Hll hll = (Hll) object;
|
||||
final double estimate = hll.size();
|
||||
|
||||
if (round) {
|
||||
return Math.round(estimate);
|
||||
} else {
|
||||
return estimate;
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
@JsonProperty
|
||||
public String getName() {
|
||||
return name;
|
||||
}
|
||||
|
||||
@JsonProperty
|
||||
public String getFieldName() {
|
||||
return fieldName;
|
||||
}
|
||||
|
||||
@JsonProperty
|
||||
public int getPrecision() {
|
||||
return precision;
|
||||
}
|
||||
|
||||
@JsonProperty
|
||||
public boolean isRound() {
|
||||
return round;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getTypeName() {
|
||||
return HllModule.HLLD_BUILD_TYPE_NAME;
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<String> requiredFields() {
|
||||
return Collections.singletonList(fieldName);
|
||||
}
|
||||
|
||||
@Override
|
||||
public int getMaxIntermediateSize() {
|
||||
return Hll.getUpdatableSerializationBytes(precision);
|
||||
}
|
||||
|
||||
@Override
|
||||
public byte[] getCacheKey() {
|
||||
return new CacheKeyBuilder(HllModule.CACHE_TYPE_ID_OFFSET).appendByte(HllModule.HLLD_BUILD_CACHE_TYPE_ID)
|
||||
.appendString(name).appendString(fieldName)
|
||||
.appendInt(precision).appendBoolean(round)
|
||||
.build();
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean equals(final Object o){
|
||||
if (this == o) {
|
||||
return true;
|
||||
}
|
||||
if (o == null || !getClass().equals(o.getClass())) {
|
||||
return false;
|
||||
}
|
||||
|
||||
HllAggregatorFactory that = (HllAggregatorFactory) o;
|
||||
return name.equals(that.name) && fieldName.equals(that.fieldName) &&
|
||||
precision == that.precision &&
|
||||
round == that.round
|
||||
;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int hashCode(){
|
||||
return Objects.hash(name, fieldName, precision, round);
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return getClass().getSimpleName() + "{" +
|
||||
"name='" + name + '\'' +
|
||||
", fieldName='" + fieldName + '\'' +
|
||||
", precision=" + precision +
|
||||
", round=" + round +
|
||||
'}';
|
||||
}
|
||||
}
|
||||
package org.apache.druid.query.aggregation.sketch.hlld;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||
import com.zdjz.galaxy.sketch.hlld.Hll;
|
||||
import com.zdjz.galaxy.sketch.hlld.HllUnion;
|
||||
import org.apache.druid.java.util.common.IAE;
|
||||
import org.apache.druid.java.util.common.logger.Logger;
|
||||
import org.apache.druid.query.aggregation.*;
|
||||
import org.apache.druid.query.cache.CacheKeyBuilder;
|
||||
import org.apache.druid.segment.ColumnSelectorFactory;
|
||||
import org.apache.druid.segment.ColumnValueSelector;
|
||||
import org.apache.druid.segment.column.ColumnType;
|
||||
|
||||
import javax.annotation.Nullable;
|
||||
import java.util.Collections;
|
||||
import java.util.Comparator;
|
||||
import java.util.List;
|
||||
import java.util.Objects;
|
||||
|
||||
public class HllAggregatorFactory extends AggregatorFactory {
|
||||
private static final Logger LOG = new Logger(HllAggregatorFactory.class);
|
||||
public static final boolean DEFAULT_ROUND = false;
|
||||
public static final int DEFAULT_PRECISION = 12;
|
||||
|
||||
static final Comparator<Hll> COMPARATOR = Comparator.nullsFirst(Comparator.comparingDouble(Hll::size));
|
||||
|
||||
protected final String name;
|
||||
protected final String fieldName;
|
||||
protected final int precision;
|
||||
protected final boolean round;
|
||||
protected final int updatableSerializationBytes;
|
||||
|
||||
public HllAggregatorFactory(
|
||||
@JsonProperty("name") final String name,
|
||||
@JsonProperty("fieldName") final String fieldName,
|
||||
@JsonProperty("precision") @Nullable final Integer precision,
|
||||
@JsonProperty("round") @Nullable final Boolean round
|
||||
) {
|
||||
if (name == null) {
|
||||
throw new IAE("Must have a valid, non-null aggregator name");
|
||||
}
|
||||
if (fieldName == null) {
|
||||
throw new IAE("Parameter fieldName must be specified");
|
||||
}
|
||||
this.name = name;
|
||||
this.fieldName = fieldName;
|
||||
this.precision = precision == null ? DEFAULT_PRECISION : precision;
|
||||
this.round = round == null ? DEFAULT_ROUND : round;
|
||||
this.updatableSerializationBytes = getUpdatableSerializationBytes();
|
||||
}
|
||||
|
||||
@Override
|
||||
public Aggregator factorize(ColumnSelectorFactory columnSelectorFactory) {
|
||||
final ColumnValueSelector<Object> selector = columnSelectorFactory.makeColumnValueSelector(fieldName);
|
||||
return new HllAggregator(selector, precision);
|
||||
}
|
||||
|
||||
@Override
|
||||
public BufferAggregator factorizeBuffered(ColumnSelectorFactory columnSelectorFactory) {
|
||||
final ColumnValueSelector<Object> selector = columnSelectorFactory.makeColumnValueSelector(fieldName);
|
||||
return new HllBufferAggregator(
|
||||
selector,
|
||||
precision
|
||||
);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Comparator getComparator() {
|
||||
return COMPARATOR;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Object combine(Object lhs, Object rhs) {
|
||||
if(lhs == null){
|
||||
return rhs;
|
||||
}else if(rhs == null){
|
||||
return lhs;
|
||||
}else{
|
||||
final HllUnion union = new HllUnion(precision);
|
||||
union.update((Hll) lhs);
|
||||
union.update((Hll) rhs);
|
||||
Hll result = union.getResult();
|
||||
return result;
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public AggregateCombiner makeAggregateCombiner() {
|
||||
return new ObjectAggregateCombiner<Hll>() {
|
||||
private HllUnion union = null;
|
||||
|
||||
@Override
|
||||
public void reset(ColumnValueSelector selector) {
|
||||
//LOG.error("HllAggregateCombiner reset:" + "-" + Thread.currentThread().getId() + "-" + this);
|
||||
//union.reset();
|
||||
union = null;
|
||||
fold(selector);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void fold(ColumnValueSelector selector) {
|
||||
//LOG.error("HllAggregateCombiner fold:" + "-" + Thread.currentThread().getId() + "-" + this);
|
||||
final Hll hll = (Hll) selector.getObject();
|
||||
if(hll != null){
|
||||
if(union == null){
|
||||
union = new HllUnion(precision);
|
||||
}
|
||||
union.update(hll);
|
||||
}else{
|
||||
//LOG.error("HllAggregateCombiner fold_null:" + "-" + Thread.currentThread().getId() + "-" + this);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public Class<Hll> classOfObject() {
|
||||
return Hll.class;
|
||||
}
|
||||
|
||||
@Nullable
|
||||
@Override
|
||||
public Hll getObject() {
|
||||
//LOG.error("HllAggregateCombiner get:" + "-" + Thread.currentThread().getId() + "-" + this);
|
||||
if(union == null){
|
||||
return null;
|
||||
}else{
|
||||
Hll result = union.getResult();
|
||||
/*if(result.size() == 0){
|
||||
return null;
|
||||
}*/
|
||||
return result;
|
||||
}
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
@Override
|
||||
public AggregatorFactory getCombiningFactory() {
|
||||
// 千万不能写错，好大一个坑
|
||||
return new HllMergeAggregatorFactory(name, name, precision, round);
|
||||
}
|
||||
|
||||
@Override
|
||||
public AggregatorFactory getMergingFactory(AggregatorFactory other) throws AggregatorFactoryNotMergeableException {
|
||||
if (other.getName().equals(this.getName()) && other instanceof HllAggregatorFactory) {
|
||||
HllAggregatorFactory castedOther = (HllAggregatorFactory) other;
|
||||
|
||||
return new HllMergeAggregatorFactory(name, name,
|
||||
Math.max(precision, castedOther.precision),
|
||||
round || castedOther.round
|
||||
);
|
||||
}
|
||||
|
||||
throw new AggregatorFactoryNotMergeableException(this, other);
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<AggregatorFactory> getRequiredColumns() {
|
||||
return Collections.singletonList(
|
||||
new HllAggregatorFactory(fieldName, fieldName, precision, round)
|
||||
);
|
||||
}
|
||||
|
||||
@Override
|
||||
public AggregatorFactory withName(String newName) {
|
||||
return new HllAggregatorFactory(newName, fieldName, precision, round);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Object deserialize(Object object) {
|
||||
if (object == null) {
|
||||
return null;
|
||||
}
|
||||
return HllUtils.deserializeHll(object);
|
||||
}
|
||||
|
||||
@Override
|
||||
public ColumnType getResultType() {
|
||||
//return round ? ColumnType.LONG : ColumnType.DOUBLE;
|
||||
return getIntermediateType();
|
||||
}
|
||||
|
||||
@Nullable
|
||||
@Override
|
||||
public Object finalizeComputation(@Nullable Object object) {
|
||||
if (object == null) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return object;
|
||||
|
||||
/*final Hll hll = (Hll) object;
|
||||
final double estimate = hll.size();
|
||||
|
||||
if (round) {
|
||||
return Math.round(estimate);
|
||||
} else {
|
||||
return estimate;
|
||||
}*/
|
||||
}
|
||||
|
||||
@Override
|
||||
@JsonProperty
|
||||
public String getName() {
|
||||
return name;
|
||||
}
|
||||
|
||||
@JsonProperty
|
||||
public String getFieldName() {
|
||||
return fieldName;
|
||||
}
|
||||
|
||||
@JsonProperty
|
||||
public int getPrecision() {
|
||||
return precision;
|
||||
}
|
||||
|
||||
@JsonProperty
|
||||
public boolean isRound() {
|
||||
return round;
|
||||
}
|
||||
|
||||
/*
|
||||
没这个方法了, 新版本需要实现getIntermediateType方法
|
||||
@Override
|
||||
public String getTypeName() {
|
||||
return HllModule.HLLD_BUILD_TYPE_NAME;
|
||||
}*/
|
||||
|
||||
@Override
|
||||
public ColumnType getIntermediateType() {
|
||||
return HllModule.BUILD_TYPE;
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<String> requiredFields() {
|
||||
return Collections.singletonList(fieldName);
|
||||
}
|
||||
|
||||
@Override
|
||||
public int getMaxIntermediateSize() {
|
||||
return updatableSerializationBytes == 0? getUpdatableSerializationBytes():updatableSerializationBytes;
|
||||
}
|
||||
|
||||
protected int getUpdatableSerializationBytes(){
|
||||
return Hll.getUpdatableSerializationBytes(precision);
|
||||
}
|
||||
|
||||
@Override
|
||||
public byte[] getCacheKey() {
|
||||
return new CacheKeyBuilder(HllModule.CACHE_TYPE_ID_OFFSET).appendByte(HllModule.HLLD_BUILD_CACHE_TYPE_ID)
|
||||
.appendString(name).appendString(fieldName)
|
||||
.appendInt(precision).appendBoolean(round)
|
||||
.build();
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean equals(final Object o){
|
||||
if (this == o) {
|
||||
return true;
|
||||
}
|
||||
if (o == null || !getClass().equals(o.getClass())) {
|
||||
return false;
|
||||
}
|
||||
|
||||
HllAggregatorFactory that = (HllAggregatorFactory) o;
|
||||
return name.equals(that.name) && fieldName.equals(that.fieldName) &&
|
||||
precision == that.precision &&
|
||||
round == that.round
|
||||
;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int hashCode(){
|
||||
return Objects.hash(name, fieldName, precision, round);
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return getClass().getSimpleName() + "{" +
|
||||
"name='" + name + '\'' +
|
||||
", fieldName='" + fieldName + '\'' +
|
||||
", precision=" + precision +
|
||||
", round=" + round +
|
||||
'}';
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,59 +1,73 @@
|
||||
package org.apache.druid.query.aggregation.sketch.hlld;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||
import com.zdjz.galaxy.sketch.hlld.Hll;
|
||||
import com.zdjz.galaxy.sketch.hlld.HllUnion;
|
||||
import org.apache.druid.query.aggregation.Aggregator;
|
||||
import org.apache.druid.query.aggregation.BufferAggregator;
|
||||
import org.apache.druid.query.cache.CacheKeyBuilder;
|
||||
import org.apache.druid.segment.ColumnSelectorFactory;
|
||||
import org.apache.druid.segment.ColumnValueSelector;
|
||||
|
||||
import javax.annotation.Nullable;
|
||||
|
||||
public class HllMergeAggregatorFactory extends HllAggregatorFactory{
|
||||
public HllMergeAggregatorFactory(
|
||||
@JsonProperty("name") final String name,
|
||||
@JsonProperty("fieldName") final String fieldName,
|
||||
@JsonProperty("precision") @Nullable final Integer precision,
|
||||
@JsonProperty("round") @Nullable final Boolean round
|
||||
) {
|
||||
super(name, fieldName, precision, round);
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getTypeName(){
|
||||
return HllModule.HLLD_TYPE_NAME;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Aggregator factorize(ColumnSelectorFactory metricFactory) {
|
||||
final ColumnValueSelector<Hll> selector = metricFactory.makeColumnValueSelector(getFieldName());
|
||||
return new HllMergeAggregator(
|
||||
selector,
|
||||
precision
|
||||
);
|
||||
}
|
||||
|
||||
@Override
|
||||
public BufferAggregator factorizeBuffered(ColumnSelectorFactory columnSelectorFactory) {
|
||||
final ColumnValueSelector<Hll> selector = columnSelectorFactory.makeColumnValueSelector(getFieldName());
|
||||
return new HllMergeBufferAggregator(
|
||||
selector,
|
||||
precision
|
||||
);
|
||||
}
|
||||
|
||||
@Override
|
||||
public byte[] getCacheKey() {
|
||||
return new CacheKeyBuilder(HllModule.CACHE_TYPE_ID_OFFSET).appendByte(HllModule.HLLD_MERGE_CACHE_TYPE_ID)
|
||||
.appendString(name).appendString(fieldName)
|
||||
.appendInt(precision).appendBoolean(round)
|
||||
.build();
|
||||
}
|
||||
|
||||
@Override
|
||||
public int getMaxIntermediateSize() {
|
||||
return HllUnion.getUpdatableSerializationBytes(precision);
|
||||
}
|
||||
}
|
||||
package org.apache.druid.query.aggregation.sketch.hlld;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||
import com.zdjz.galaxy.sketch.hlld.Hll;
|
||||
import com.zdjz.galaxy.sketch.hlld.HllUnion;
|
||||
import org.apache.druid.query.aggregation.Aggregator;
|
||||
import org.apache.druid.query.aggregation.AggregatorFactory;
|
||||
import org.apache.druid.query.aggregation.BufferAggregator;
|
||||
import org.apache.druid.query.cache.CacheKeyBuilder;
|
||||
import org.apache.druid.segment.ColumnSelectorFactory;
|
||||
import org.apache.druid.segment.ColumnValueSelector;
|
||||
import org.apache.druid.segment.column.ColumnType;
|
||||
|
||||
import javax.annotation.Nullable;
|
||||
|
||||
public class HllMergeAggregatorFactory extends HllAggregatorFactory{
|
||||
public HllMergeAggregatorFactory(
|
||||
@JsonProperty("name") final String name,
|
||||
@JsonProperty("fieldName") final String fieldName,
|
||||
@JsonProperty("precision") @Nullable final Integer precision,
|
||||
@JsonProperty("round") @Nullable final Boolean round
|
||||
) {
|
||||
super(name, fieldName, precision, round);
|
||||
}
|
||||
|
||||
/*
|
||||
没这个方法了, 新版本需要实现getIntermediateType方法
|
||||
@Override
|
||||
public String getTypeName(){
|
||||
return HllModule.HLLD_TYPE_NAME;
|
||||
}*/
|
||||
|
||||
@Override
|
||||
public ColumnType getIntermediateType() {
|
||||
return HllModule.TYPE;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Aggregator factorize(ColumnSelectorFactory metricFactory) {
|
||||
final ColumnValueSelector<Hll> selector = metricFactory.makeColumnValueSelector(getFieldName());
|
||||
return new HllMergeAggregator(
|
||||
selector,
|
||||
precision
|
||||
);
|
||||
}
|
||||
|
||||
@Override
|
||||
public BufferAggregator factorizeBuffered(ColumnSelectorFactory columnSelectorFactory) {
|
||||
final ColumnValueSelector<Hll> selector = columnSelectorFactory.makeColumnValueSelector(getFieldName());
|
||||
return new HllMergeBufferAggregator(
|
||||
selector,
|
||||
precision
|
||||
);
|
||||
}
|
||||
|
||||
@Override
|
||||
public AggregatorFactory withName(String newName) {
|
||||
return new HllMergeAggregatorFactory(newName, fieldName, precision, round);
|
||||
}
|
||||
|
||||
@Override
|
||||
public byte[] getCacheKey() {
|
||||
return new CacheKeyBuilder(HllModule.CACHE_TYPE_ID_OFFSET).appendByte(HllModule.HLLD_MERGE_CACHE_TYPE_ID)
|
||||
.appendString(name).appendString(fieldName)
|
||||
.appendInt(precision).appendBoolean(round)
|
||||
.build();
|
||||
}
|
||||
|
||||
@Override
|
||||
protected int getUpdatableSerializationBytes() {
|
||||
return HllUnion.getUpdatableSerializationBytes(precision);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -10,6 +10,7 @@ import org.apache.druid.initialization.DruidModule;
|
||||
import org.apache.druid.query.aggregation.sketch.hlld.sql.HllApproxCountDistinctSqlAggregator;
|
||||
import org.apache.druid.query.aggregation.sketch.hlld.sql.HllEstimateOperatorConversion;
|
||||
import org.apache.druid.query.aggregation.sketch.hlld.sql.HllObjectSqlAggregator;
|
||||
import org.apache.druid.segment.column.ColumnType;
|
||||
import org.apache.druid.segment.serde.ComplexMetrics;
|
||||
import org.apache.druid.sql.guice.SqlBindings;
|
||||
|
||||
@@ -24,6 +25,9 @@ public class HllModule implements DruidModule {
|
||||
|
||||
public static final String HLLD_TYPE_NAME = "HLLDSketch";
|
||||
public static final String HLLD_BUILD_TYPE_NAME = "HLLDSketchBuild";
|
||||
public static final ColumnType TYPE = ColumnType.ofComplex(HLLD_TYPE_NAME);
|
||||
public static final ColumnType BUILD_TYPE = ColumnType.ofComplex(HLLD_BUILD_TYPE_NAME);
|
||||
|
||||
|
||||
@Override
|
||||
public void configure(Binder binder) {
|
||||
|
||||
@@ -1,103 +1,114 @@
|
||||
package org.apache.druid.query.aggregation.sketch.hlld;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonCreator;
|
||||
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||
import com.zdjz.galaxy.sketch.hlld.Hll;
|
||||
import org.apache.druid.query.aggregation.AggregatorFactory;
|
||||
import org.apache.druid.query.aggregation.PostAggregator;
|
||||
import org.apache.druid.query.aggregation.post.ArithmeticPostAggregator;
|
||||
import org.apache.druid.query.cache.CacheKeyBuilder;
|
||||
|
||||
import java.util.Comparator;
|
||||
import java.util.Map;
|
||||
import java.util.Objects;
|
||||
import java.util.Set;
|
||||
|
||||
public class HllToEstimatePostAggregator implements PostAggregator {
|
||||
private final String name;
|
||||
private final PostAggregator field;
|
||||
private final boolean round;
|
||||
|
||||
@JsonCreator
|
||||
public HllToEstimatePostAggregator(
|
||||
@JsonProperty("name") final String name,
|
||||
@JsonProperty("field") final PostAggregator field,
|
||||
@JsonProperty("round") boolean round
|
||||
) {
|
||||
this.name = name;
|
||||
this.field = field;
|
||||
this.round = round;
|
||||
}
|
||||
|
||||
@Override
|
||||
@JsonProperty
|
||||
public String getName() {
|
||||
return name;
|
||||
}
|
||||
|
||||
@JsonProperty
|
||||
public PostAggregator getField() {
|
||||
return field;
|
||||
}
|
||||
|
||||
@JsonProperty
|
||||
public boolean isRound() {
|
||||
return round;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Set<String> getDependentFields() {
|
||||
return field.getDependentFields();
|
||||
}
|
||||
|
||||
@Override
|
||||
public Comparator<Double> getComparator() {
|
||||
return ArithmeticPostAggregator.DEFAULT_COMPARATOR;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Object compute(final Map<String, Object> combinedAggregators) {
|
||||
final Hll sketch = (Hll) field.compute(combinedAggregators);
|
||||
return round ? Math.round(sketch.size()) : sketch.size();
|
||||
}
|
||||
|
||||
@Override
|
||||
public PostAggregator decorate(final Map<String, AggregatorFactory> aggregators) {
|
||||
return this;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return "HllToEstimatePostAggregator{" +
|
||||
"name='" + name + '\'' +
|
||||
", field=" + field +
|
||||
", round=" + round +
|
||||
'}';
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean equals(final Object o) {
|
||||
if (this == o) {
|
||||
return true;
|
||||
}
|
||||
if (!(o instanceof HllToEstimatePostAggregator)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
final HllToEstimatePostAggregator that = (HllToEstimatePostAggregator) o;
|
||||
return name.equals(that.name) && field.equals(that.field) && round == that.round;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int hashCode() {
|
||||
return Objects.hash(name, field, round);
|
||||
}
|
||||
|
||||
@Override
|
||||
public byte[] getCacheKey() {
|
||||
CacheKeyBuilder builder = new CacheKeyBuilder(HllModule.CACHE_TYPE_ID_OFFSET).appendByte(HllModule.HLLD_TO_ESTIMATE_CACHE_TYPE_ID)
|
||||
.appendCacheable(field).appendBoolean(round);
|
||||
return builder.build();
|
||||
}
|
||||
|
||||
}
|
||||
package org.apache.druid.query.aggregation.sketch.hlld;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonCreator;
|
||||
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||
import com.zdjz.galaxy.sketch.hlld.Hll;
|
||||
import org.apache.druid.query.aggregation.AggregatorFactory;
|
||||
import org.apache.druid.query.aggregation.PostAggregator;
|
||||
import org.apache.druid.query.aggregation.post.ArithmeticPostAggregator;
|
||||
import org.apache.druid.query.cache.CacheKeyBuilder;
|
||||
import org.apache.druid.segment.ColumnInspector;
|
||||
import org.apache.druid.segment.column.ColumnType;
|
||||
|
||||
import java.util.Comparator;
|
||||
import java.util.Map;
|
||||
import java.util.Objects;
|
||||
import java.util.Set;
|
||||
|
||||
public class HllToEstimatePostAggregator implements PostAggregator {
|
||||
private final String name;
|
||||
private final PostAggregator field;
|
||||
private final boolean round;
|
||||
|
||||
@JsonCreator
|
||||
public HllToEstimatePostAggregator(
|
||||
@JsonProperty("name") final String name,
|
||||
@JsonProperty("field") final PostAggregator field,
|
||||
@JsonProperty("round") boolean round
|
||||
) {
|
||||
this.name = name;
|
||||
this.field = field;
|
||||
this.round = round;
|
||||
}
|
||||
|
||||
// 新版本需要实现的方法
|
||||
@Override
|
||||
public ColumnType getType(ColumnInspector signature) {
|
||||
return round ? ColumnType.LONG : ColumnType.DOUBLE;
|
||||
}
|
||||
|
||||
@Override
|
||||
@JsonProperty
|
||||
public String getName() {
|
||||
return name;
|
||||
}
|
||||
|
||||
@JsonProperty
|
||||
public PostAggregator getField() {
|
||||
return field;
|
||||
}
|
||||
|
||||
@JsonProperty
|
||||
public boolean isRound() {
|
||||
return round;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Set<String> getDependentFields() {
|
||||
return field.getDependentFields();
|
||||
}
|
||||
|
||||
@Override
|
||||
public Comparator<Double> getComparator() {
|
||||
return ArithmeticPostAggregator.DEFAULT_COMPARATOR;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Object compute(final Map<String, Object> combinedAggregators) {
|
||||
final Hll sketch = (Hll) field.compute(combinedAggregators);
|
||||
if(sketch == null){
|
||||
return round ? 0L: 0D;
|
||||
}
|
||||
return round ? Math.round(sketch.size()) : sketch.size();
|
||||
}
|
||||
|
||||
@Override
|
||||
public PostAggregator decorate(final Map<String, AggregatorFactory> aggregators) {
|
||||
return this;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return "HllToEstimatePostAggregator{" +
|
||||
"name='" + name + '\'' +
|
||||
", field=" + field +
|
||||
", round=" + round +
|
||||
'}';
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean equals(final Object o) {
|
||||
if (this == o) {
|
||||
return true;
|
||||
}
|
||||
if (!(o instanceof HllToEstimatePostAggregator)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
final HllToEstimatePostAggregator that = (HllToEstimatePostAggregator) o;
|
||||
return name.equals(that.name) && field.equals(that.field) && round == that.round;
|
||||
}
|
||||
|
||||
@Override
|
||||
public int hashCode() {
|
||||
return Objects.hash(name, field, round);
|
||||
}
|
||||
|
||||
@Override
|
||||
public byte[] getCacheKey() {
|
||||
CacheKeyBuilder builder = new CacheKeyBuilder(HllModule.CACHE_TYPE_ID_OFFSET).appendByte(HllModule.HLLD_TO_ESTIMATE_CACHE_TYPE_ID)
|
||||
.appendCacheable(field).appendBoolean(round);
|
||||
return builder.build();
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -5,36 +5,44 @@ import org.apache.calcite.sql.SqlFunctionCategory;
|
||||
import org.apache.calcite.sql.SqlKind;
|
||||
import org.apache.calcite.sql.type.*;
|
||||
import org.apache.druid.query.aggregation.AggregatorFactory;
|
||||
import org.apache.druid.query.aggregation.post.FieldAccessPostAggregator;
|
||||
import org.apache.druid.query.aggregation.post.FinalizingFieldAccessPostAggregator;
|
||||
import org.apache.druid.segment.VirtualColumn;
|
||||
import org.apache.druid.query.aggregation.sketch.hlld.HllAggregatorFactory;
|
||||
import org.apache.druid.query.aggregation.sketch.hlld.HllToEstimatePostAggregator;
|
||||
import org.apache.druid.sql.calcite.aggregation.Aggregation;
|
||||
|
||||
import java.util.Collections;
|
||||
import java.util.List;
|
||||
|
||||
public class HllApproxCountDistinctSqlAggregator extends HllBaseSqlAggregator {
|
||||
private static final SqlAggFunction FUNCTION_INSTANCE = new CPCSketchApproxCountDistinctSqlAggFunction();
|
||||
private static final String NAME = "APPROX_COUNT_DISTINCT_HLLD";
|
||||
|
||||
public HllApproxCountDistinctSqlAggregator(){
|
||||
super(true);
|
||||
}
|
||||
|
||||
@Override
|
||||
public SqlAggFunction calciteFunction() {
|
||||
return FUNCTION_INSTANCE;
|
||||
}
|
||||
|
||||
// 新版本参数少了virtualColumns
|
||||
@Override
|
||||
protected Aggregation toAggregation(
|
||||
String name,
|
||||
boolean finalizeAggregations,
|
||||
List<VirtualColumn> virtualColumns,
|
||||
AggregatorFactory aggregatorFactory
|
||||
) {
|
||||
return Aggregation.create(
|
||||
virtualColumns,
|
||||
Collections.singletonList(aggregatorFactory),
|
||||
//感觉是否是最外层的函数吧
|
||||
finalizeAggregations ? new FinalizingFieldAccessPostAggregator(
|
||||
finalizeAggregations ? new HllToEstimatePostAggregator(
|
||||
name,
|
||||
aggregatorFactory.getName()
|
||||
new FieldAccessPostAggregator(
|
||||
aggregatorFactory.getName(),
|
||||
aggregatorFactory.getName()
|
||||
),
|
||||
((HllAggregatorFactory)aggregatorFactory).isRound()
|
||||
) : null
|
||||
);
|
||||
}
|
||||
|
||||
@@ -2,6 +2,7 @@ package org.apache.druid.query.aggregation.sketch.hlld.sql;
|
||||
|
||||
import org.apache.calcite.rel.core.AggregateCall;
|
||||
import org.apache.calcite.rel.core.Project;
|
||||
import org.apache.calcite.rel.type.RelDataType;
|
||||
import org.apache.calcite.rex.RexBuilder;
|
||||
import org.apache.calcite.rex.RexLiteral;
|
||||
import org.apache.calcite.rex.RexNode;
|
||||
@@ -14,6 +15,7 @@ import org.apache.druid.query.aggregation.sketch.hlld.HllMergeAggregatorFactory;
|
||||
import org.apache.druid.query.dimension.DefaultDimensionSpec;
|
||||
import org.apache.druid.query.dimension.DimensionSpec;
|
||||
import org.apache.druid.segment.VirtualColumn;
|
||||
import org.apache.druid.segment.column.ColumnType;
|
||||
import org.apache.druid.segment.column.RowSignature;
|
||||
import org.apache.druid.segment.column.ValueType;
|
||||
import org.apache.druid.sql.calcite.aggregation.Aggregation;
|
||||
@@ -29,6 +31,13 @@ import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
public abstract class HllBaseSqlAggregator implements SqlAggregator {
|
||||
|
||||
private final boolean finalizeSketch;
|
||||
|
||||
protected HllBaseSqlAggregator(boolean finalizeSketch){
|
||||
this.finalizeSketch = finalizeSketch;
|
||||
}
|
||||
|
||||
@Nullable
|
||||
@Override
|
||||
public Aggregation toDruidAggregation(
|
||||
@@ -93,13 +102,14 @@ public abstract class HllBaseSqlAggregator implements SqlAggregator {
|
||||
round = HllAggregatorFactory.DEFAULT_ROUND;
|
||||
}
|
||||
|
||||
final List<VirtualColumn> virtualColumns = new ArrayList<>();
|
||||
// 新版本删除了final List<VirtualColumn> virtualColumns = new ArrayList<>();
|
||||
final AggregatorFactory aggregatorFactory;
|
||||
final String aggregatorName = finalizeAggregations ? Calcites.makePrefixedName(name, "a") : name;
|
||||
//final String aggregatorName = finalizeAggregations ? Calcites.makePrefixedName(name, "a") : name;
|
||||
final String aggregatorName = finalizeSketch ? Calcites.makePrefixedName(name, "a") : name;
|
||||
|
||||
// 输入是Cpc，返回HllMergeAggregatorFactory
|
||||
// 输入是Hll，返回HllSketchMergeAggregatorFactory
|
||||
if (columnArg.isDirectColumnAccess()
|
||||
&& rowSignature.getColumnType(columnArg.getDirectColumn()).orElse(null) == ValueType.COMPLEX) {
|
||||
&& rowSignature.getColumnType(columnArg.getDirectColumn()).map(type -> type.is(ValueType.COMPLEX)).orElse(false)) {
|
||||
// 这就是具体的聚合函数吧
|
||||
aggregatorFactory = new HllMergeAggregatorFactory(
|
||||
aggregatorName,
|
||||
@@ -109,10 +119,10 @@ public abstract class HllBaseSqlAggregator implements SqlAggregator {
|
||||
);
|
||||
} else {
|
||||
// 输入是regular column，HllBuildAggregatorFactory
|
||||
final SqlTypeName sqlTypeName = columnRexNode.getType().getSqlTypeName();
|
||||
final ValueType inputType = Calcites.getValueTypeForSqlTypeName(sqlTypeName);
|
||||
final RelDataType dataType = columnRexNode.getType();
|
||||
final ColumnType inputType = Calcites.getColumnTypeForRelDataType(dataType);
|
||||
if (inputType == null) {
|
||||
throw new ISE("Cannot translate sqlTypeName[%s] to Druid type for field[%s]", sqlTypeName, aggregatorName);
|
||||
throw new ISE("Cannot translate sqlTypeName[%s] to Druid type for field[%s]", dataType.getSqlTypeName(), aggregatorName);
|
||||
}
|
||||
|
||||
final DimensionSpec dimensionSpec;
|
||||
@@ -120,27 +130,34 @@ public abstract class HllBaseSqlAggregator implements SqlAggregator {
|
||||
if (columnArg.isDirectColumnAccess()) {
|
||||
dimensionSpec = columnArg.getSimpleExtraction().toDimensionSpec(null, inputType);
|
||||
} else {
|
||||
VirtualColumn virtualColumn = virtualColumnRegistry.getOrCreateVirtualColumnForExpression(
|
||||
plannerContext,
|
||||
String virtualColumnName = virtualColumnRegistry.getOrCreateVirtualColumnForExpression(
|
||||
columnArg,
|
||||
sqlTypeName
|
||||
dataType
|
||||
);
|
||||
dimensionSpec = new DefaultDimensionSpec(virtualColumn.getOutputName(), null, inputType);
|
||||
virtualColumns.add(virtualColumn);
|
||||
dimensionSpec = new DefaultDimensionSpec(virtualColumnName, null, inputType);
|
||||
}
|
||||
|
||||
aggregatorFactory = new HllAggregatorFactory(
|
||||
aggregatorName,
|
||||
dimensionSpec.getDimension(),
|
||||
precision,
|
||||
round
|
||||
);
|
||||
// 新版本的判断，输入是Hll
|
||||
if (inputType.is(ValueType.COMPLEX)) {
|
||||
aggregatorFactory = new HllMergeAggregatorFactory(
|
||||
aggregatorName,
|
||||
dimensionSpec.getOutputName(),
|
||||
precision,
|
||||
round
|
||||
);
|
||||
} else {
|
||||
aggregatorFactory = new HllAggregatorFactory(
|
||||
aggregatorName,
|
||||
dimensionSpec.getDimension(),
|
||||
precision,
|
||||
round
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
return toAggregation(
|
||||
name,
|
||||
finalizeAggregations,
|
||||
virtualColumns,
|
||||
finalizeSketch,
|
||||
aggregatorFactory
|
||||
);
|
||||
}
|
||||
@@ -148,7 +165,6 @@ public abstract class HllBaseSqlAggregator implements SqlAggregator {
|
||||
protected abstract Aggregation toAggregation(
|
||||
String name,
|
||||
boolean finalizeAggregations,
|
||||
List<VirtualColumn> virtualColumns,
|
||||
AggregatorFactory aggregatorFactory
|
||||
);
|
||||
}
|
||||
|
||||
@@ -13,16 +13,15 @@ import org.apache.druid.query.aggregation.PostAggregator;
|
||||
import org.apache.druid.query.aggregation.sketch.hlld.HllAggregatorFactory;
|
||||
import org.apache.druid.query.aggregation.sketch.hlld.HllToEstimatePostAggregator;
|
||||
import org.apache.druid.segment.column.RowSignature;
|
||||
import org.apache.druid.sql.calcite.expression.DirectOperatorConversion;
|
||||
import org.apache.druid.sql.calcite.expression.DruidExpression;
|
||||
import org.apache.druid.sql.calcite.expression.OperatorConversions;
|
||||
import org.apache.druid.sql.calcite.expression.PostAggregatorVisitor;
|
||||
import org.apache.druid.sql.calcite.expression.*;
|
||||
import org.apache.druid.sql.calcite.planner.PlannerContext;
|
||||
|
||||
import javax.annotation.Nullable;
|
||||
import java.util.List;
|
||||
|
||||
public class HllEstimateOperatorConversion extends DirectOperatorConversion {
|
||||
// postAggregator, toDruidExpression返回null。相当于post udf和普通udf是不一样的。
|
||||
// 新版本直接修改了父类
|
||||
public class HllEstimateOperatorConversion implements SqlOperatorConversion {
|
||||
private static final String FUNCTION_NAME = "HLLD_ESTIMATE";
|
||||
private static final SqlFunction SQL_FUNCTION = OperatorConversions
|
||||
.operatorBuilder(StringUtils.toUpperCase(FUNCTION_NAME))
|
||||
@@ -32,9 +31,7 @@ public class HllEstimateOperatorConversion extends DirectOperatorConversion {
|
||||
.returnTypeInference(ReturnTypes.DOUBLE)
|
||||
.build();
|
||||
|
||||
public HllEstimateOperatorConversion() {
|
||||
super(SQL_FUNCTION, FUNCTION_NAME);
|
||||
}
|
||||
// 新版本少了构造函数
|
||||
|
||||
@Override
|
||||
public SqlOperator calciteOperator() {
|
||||
@@ -63,7 +60,8 @@ public class HllEstimateOperatorConversion extends DirectOperatorConversion {
|
||||
plannerContext,
|
||||
rowSignature,
|
||||
operands.get(0),
|
||||
postAggregatorVisitor
|
||||
postAggregatorVisitor,
|
||||
true // 新版本多了个参数
|
||||
);
|
||||
|
||||
if (firstOperand == null) {
|
||||
|
||||
@@ -5,16 +5,18 @@ import org.apache.calcite.sql.SqlFunctionCategory;
|
||||
import org.apache.calcite.sql.SqlKind;
|
||||
import org.apache.calcite.sql.type.*;
|
||||
import org.apache.druid.query.aggregation.AggregatorFactory;
|
||||
import org.apache.druid.segment.VirtualColumn;
|
||||
import org.apache.druid.sql.calcite.aggregation.Aggregation;
|
||||
|
||||
import java.util.Collections;
|
||||
import java.util.List;
|
||||
|
||||
public class HllObjectSqlAggregator extends HllBaseSqlAggregator {
|
||||
private static final SqlAggFunction FUNCTION_INSTANCE = new CpcSketchSqlAggFunction();
|
||||
private static final String NAME = "HLLD";
|
||||
|
||||
public HllObjectSqlAggregator(){
|
||||
super(false);
|
||||
}
|
||||
|
||||
@Override
|
||||
public SqlAggFunction calciteFunction() {
|
||||
return FUNCTION_INSTANCE;
|
||||
@@ -24,11 +26,9 @@ public class HllObjectSqlAggregator extends HllBaseSqlAggregator {
|
||||
protected Aggregation toAggregation(
|
||||
String name,
|
||||
boolean finalizeAggregations,
|
||||
List<VirtualColumn> virtualColumns,
|
||||
AggregatorFactory aggregatorFactory
|
||||
) {
|
||||
return Aggregation.create(
|
||||
virtualColumns,
|
||||
Collections.singletonList(aggregatorFactory),
|
||||
null
|
||||
);
|
||||
|
||||
@@ -1,311 +1,429 @@
|
||||
package org.apache.druid.query.aggregation.sketch.hlld.sql;
|
||||
|
||||
|
||||
import com.fasterxml.jackson.databind.Module;
|
||||
import com.google.common.collect.ImmutableMap;
|
||||
import com.google.common.collect.ImmutableSet;
|
||||
import org.apache.calcite.schema.SchemaPlus;
|
||||
import org.apache.druid.java.util.common.io.Closer;
|
||||
import org.apache.druid.query.QueryRunnerFactoryConglomerate;
|
||||
import org.apache.druid.query.aggregation.CountAggregatorFactory;
|
||||
import org.apache.druid.query.aggregation.DoubleSumAggregatorFactory;
|
||||
import org.apache.druid.query.aggregation.sketch.hlld.HllAggregatorFactory;
|
||||
import org.apache.druid.query.aggregation.sketch.hlld.HllModule;
|
||||
import org.apache.druid.segment.IndexBuilder;
|
||||
import org.apache.druid.segment.QueryableIndex;
|
||||
import org.apache.druid.segment.TestHelper;
|
||||
import org.apache.druid.segment.incremental.IncrementalIndexSchema;
|
||||
import org.apache.druid.segment.writeout.OffHeapMemorySegmentWriteOutMediumFactory;
|
||||
import org.apache.druid.server.QueryStackTests;
|
||||
import org.apache.druid.server.security.AuthTestUtils;
|
||||
import org.apache.druid.server.security.AuthenticationResult;
|
||||
import org.apache.druid.sql.SqlLifecycle;
|
||||
import org.apache.druid.sql.SqlLifecycleFactory;
|
||||
import org.apache.druid.sql.calcite.planner.DruidOperatorTable;
|
||||
import org.apache.druid.sql.calcite.planner.PlannerConfig;
|
||||
import org.apache.druid.sql.calcite.planner.PlannerContext;
|
||||
import org.apache.druid.sql.calcite.planner.PlannerFactory;
|
||||
import org.apache.druid.sql.calcite.util.CalciteTestBase;
|
||||
import org.apache.druid.sql.calcite.util.CalciteTests;
|
||||
import org.apache.druid.sql.calcite.util.QueryLogHook;
|
||||
import org.apache.druid.sql.calcite.util.SpecificSegmentsQuerySegmentWalker;
|
||||
import org.apache.druid.timeline.DataSegment;
|
||||
import org.apache.druid.timeline.partition.LinearShardSpec;
|
||||
import org.junit.*;
|
||||
import org.junit.rules.TemporaryFolder;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.Arrays;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
public class HllApproxCountDistinctSqlAggregatorTest extends CalciteTestBase {
|
||||
private static final String DATA_SOURCE = "foo";
|
||||
private static final boolean ROUND = true;
|
||||
private static final Map<String, Object> QUERY_CONTEXT_DEFAULT = ImmutableMap.of(
|
||||
PlannerContext.CTX_SQL_QUERY_ID, "dummy"
|
||||
);
|
||||
private static QueryRunnerFactoryConglomerate conglomerate;
|
||||
private static Closer resourceCloser;
|
||||
private static AuthenticationResult authenticationResult = CalciteTests.REGULAR_USER_AUTH_RESULT;
|
||||
|
||||
@Rule
|
||||
public TemporaryFolder temporaryFolder = new TemporaryFolder();
|
||||
|
||||
@Rule
|
||||
public QueryLogHook queryLogHook = QueryLogHook.create(TestHelper.JSON_MAPPER);
|
||||
|
||||
private SpecificSegmentsQuerySegmentWalker walker;
|
||||
private SqlLifecycleFactory sqlLifecycleFactory;
|
||||
|
||||
@BeforeClass
|
||||
public static void setUpClass() {
|
||||
resourceCloser = Closer.create();
|
||||
conglomerate = QueryStackTests.createQueryRunnerFactoryConglomerate(resourceCloser);
|
||||
}
|
||||
|
||||
@AfterClass
|
||||
public static void tearDownClass() throws IOException {
|
||||
resourceCloser.close();
|
||||
}
|
||||
|
||||
@Before
|
||||
public void setUp() throws Exception {
|
||||
HllModule.registerSerde();
|
||||
for (Module mod : new HllModule().getJacksonModules()) {
|
||||
CalciteTests.getJsonMapper().registerModule(mod);
|
||||
TestHelper.JSON_MAPPER.registerModule(mod);
|
||||
}
|
||||
|
||||
final QueryableIndex index = IndexBuilder.create()
|
||||
.tmpDir(temporaryFolder.newFolder())
|
||||
.segmentWriteOutMediumFactory(OffHeapMemorySegmentWriteOutMediumFactory.instance())
|
||||
.schema(
|
||||
new IncrementalIndexSchema.Builder()
|
||||
.withMetrics(
|
||||
new CountAggregatorFactory("cnt"),
|
||||
new DoubleSumAggregatorFactory("m1", "m1"),
|
||||
new HllAggregatorFactory(
|
||||
"hll_dim1",
|
||||
"dim1",
|
||||
null,
|
||||
ROUND
|
||||
)
|
||||
)
|
||||
.withRollup(false)
|
||||
.build()
|
||||
)
|
||||
.rows(CalciteTests.ROWS1)
|
||||
.buildMMappedIndex();
|
||||
|
||||
walker = new SpecificSegmentsQuerySegmentWalker(conglomerate).add(
|
||||
DataSegment.builder()
|
||||
.dataSource(DATA_SOURCE)
|
||||
.interval(index.getDataInterval())
|
||||
.version("1")
|
||||
.shardSpec(new LinearShardSpec(0))
|
||||
.size(0)
|
||||
.build(),
|
||||
index
|
||||
);
|
||||
|
||||
final PlannerConfig plannerConfig = new PlannerConfig();
|
||||
final DruidOperatorTable operatorTable = new DruidOperatorTable(
|
||||
ImmutableSet.of(
|
||||
new HllApproxCountDistinctSqlAggregator(),
|
||||
new HllObjectSqlAggregator()
|
||||
),
|
||||
ImmutableSet.of(
|
||||
new HllEstimateOperatorConversion()
|
||||
)
|
||||
);
|
||||
|
||||
SchemaPlus rootSchema = CalciteTests.createMockRootSchema(conglomerate, walker, plannerConfig, AuthTestUtils.TEST_AUTHORIZER_MAPPER);
|
||||
sqlLifecycleFactory = CalciteTests.createSqlLifecycleFactory(
|
||||
new PlannerFactory(
|
||||
rootSchema,
|
||||
CalciteTests.createMockQueryLifecycleFactory(walker, conglomerate),
|
||||
operatorTable,
|
||||
CalciteTests.createExprMacroTable(),
|
||||
plannerConfig,
|
||||
AuthTestUtils.TEST_AUTHORIZER_MAPPER,
|
||||
CalciteTests.getJsonMapper(),
|
||||
CalciteTests.DRUID_SCHEMA_NAME
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
@After
|
||||
public void tearDown() throws Exception {
|
||||
walker.close();
|
||||
walker = null;
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testSqlQuery() throws Exception {
|
||||
SqlLifecycle sqlLifecycle = sqlLifecycleFactory.factorize();
|
||||
String sql = "select * from druid.foo";
|
||||
final List<Object[]> results =
|
||||
sqlLifecycle.runSimple(sql, QUERY_CONTEXT_DEFAULT, DEFAULT_PARAMETERS, authenticationResult).toList();
|
||||
for (Object[] result : results) {
|
||||
System.out.println(Arrays.toString(result));
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testSqlQuery2() throws Exception {
|
||||
SqlLifecycle sqlLifecycle = sqlLifecycleFactory.factorize();
|
||||
String sql = "select HLLD_ESTIMATE(HLLD(hll_dim1)) from druid.foo where dim1 = ''";
|
||||
final List<Object[]> results =
|
||||
sqlLifecycle.runSimple(sql, QUERY_CONTEXT_DEFAULT, DEFAULT_PARAMETERS, authenticationResult).toList();
|
||||
for (Object[] result : results) {
|
||||
System.out.println(Arrays.toString(result));
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testAgg() throws Exception {
|
||||
SqlLifecycle sqlLifecycle = sqlLifecycleFactory.factorize();
|
||||
|
||||
final String sql = "SELECT\n"
|
||||
+ " SUM(cnt),\n"
|
||||
+ " APPROX_COUNT_DISTINCT_HLLD(hll_dim1)\n"
|
||||
+ "FROM druid.foo";
|
||||
|
||||
final List<Object[]> results =
|
||||
sqlLifecycle.runSimple(sql, QUERY_CONTEXT_DEFAULT, DEFAULT_PARAMETERS, authenticationResult).toList();
|
||||
for (Object[] result : results) {
|
||||
System.out.println(Arrays.toString(result));
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
@Test
|
||||
public void testDistinct() throws Exception {
|
||||
SqlLifecycle sqlLifecycle = sqlLifecycleFactory.factorize();
|
||||
|
||||
final String sql = "SELECT\n"
|
||||
+ " SUM(cnt),\n"
|
||||
+ " APPROX_COUNT_DISTINCT_HLLD(dim2),\n" // uppercase
|
||||
+ " APPROX_COUNT_DISTINCT_HLLD(dim2) FILTER(WHERE dim2 <> ''),\n" // lowercase; also, filtered
|
||||
+ " APPROX_COUNT_DISTINCT_HLLD(SUBSTRING(dim2, 1, 1)),\n" // on extractionFn
|
||||
+ " APPROX_COUNT_DISTINCT_HLLD(SUBSTRING(dim2, 1, 1) || 'x'),\n" // on expression
|
||||
+ " APPROX_COUNT_DISTINCT_HLLD(hll_dim1, 16),\n" // on native HllSketch column
|
||||
+ " APPROX_COUNT_DISTINCT_HLLD(hll_dim1)\n" // on native HllSketch column
|
||||
+ "FROM druid.foo";
|
||||
|
||||
final List<Object[]> results =
|
||||
sqlLifecycle.runSimple(sql, QUERY_CONTEXT_DEFAULT, DEFAULT_PARAMETERS, authenticationResult).toList();
|
||||
for (Object[] result : results) {
|
||||
System.out.println(Arrays.toString(result));
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testDistinct2() throws Exception {
|
||||
SqlLifecycle sqlLifecycle = sqlLifecycleFactory.factorize();
|
||||
|
||||
final String sql = "SELECT\n"
|
||||
+ " SUM(cnt),\n"
|
||||
+ " APPROX_COUNT_DISTINCT_HLLD(dim2),\n"
|
||||
+ " HLLD(dim2),\n"
|
||||
+ " HLLD(hll_dim1),\n"
|
||||
+ " HLLD_ESTIMATE(HLLD(dim2)),\n"
|
||||
+ " HLLD_ESTIMATE(HLLD(dim2), true),\n"
|
||||
+ " HLLD_ESTIMATE(HLLD(dim1), true),\n"
|
||||
+ " HLLD_ESTIMATE(HLLD(hll_dim1)),\n" // on native HllSketch column
|
||||
+ " APPROX_COUNT_DISTINCT_HLLD(hll_dim1)\n" // on native HllSketch column
|
||||
+ "FROM druid.foo";
|
||||
|
||||
final List<Object[]> results =
|
||||
sqlLifecycle.runSimple(sql, QUERY_CONTEXT_DEFAULT, DEFAULT_PARAMETERS, authenticationResult).toList();
|
||||
for (Object[] result : results) {
|
||||
System.out.println(Arrays.toString(result));
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testDistinctDebug() throws Exception {
|
||||
SqlLifecycle sqlLifecycle = sqlLifecycleFactory.factorize();
|
||||
|
||||
final String sql = "SELECT\n"
|
||||
+ " SUM(cnt),\n"
|
||||
+ " APPROX_COUNT_DISTINCT_HLLD(dim2)\n"
|
||||
+ "FROM druid.foo";
|
||||
|
||||
final List<Object[]> results =
|
||||
sqlLifecycle.runSimple(sql, QUERY_CONTEXT_DEFAULT, DEFAULT_PARAMETERS, authenticationResult).toList();
|
||||
for (Object[] result : results) {
|
||||
System.out.println(Arrays.toString(result));
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testDeser() throws Exception {
|
||||
SqlLifecycle sqlLifecycle = sqlLifecycleFactory.factorize();
|
||||
|
||||
final String sql = "SELECT\n"
|
||||
+ " APPROX_COUNT_DISTINCT_HLLD(hll_dim1) cnt\n"
|
||||
+ "FROM druid.foo";
|
||||
|
||||
final List<Object[]> results =
|
||||
sqlLifecycle.runSimple(sql, QUERY_CONTEXT_DEFAULT, DEFAULT_PARAMETERS, authenticationResult).toList();
|
||||
for (Object[] result : results) {
|
||||
System.out.println(Arrays.toString(result));
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
@Test
|
||||
public void testGroupBy() throws Exception {
|
||||
SqlLifecycle sqlLifecycle = sqlLifecycleFactory.factorize();
|
||||
|
||||
final String sql = "SELECT cnt,\n"
|
||||
+ " APPROX_COUNT_DISTINCT_HLLD(hll_dim1, 14) cnt2\n"
|
||||
+ "FROM druid.foo group by cnt";
|
||||
|
||||
final List<Object[]> results =
|
||||
sqlLifecycle.runSimple(sql, QUERY_CONTEXT_DEFAULT, DEFAULT_PARAMETERS, authenticationResult).toList();
|
||||
for (Object[] result : results) {
|
||||
System.out.println(Arrays.toString(result));
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGroupBy1() throws Exception {
|
||||
SqlLifecycle sqlLifecycle = sqlLifecycleFactory.factorize();
|
||||
|
||||
final String sql = "SELECT __time,\n"
|
||||
+ " APPROX_COUNT_DISTINCT_HLLD(hll_dim1, 14) cnt\n"
|
||||
+ "FROM druid.foo group by __time";
|
||||
|
||||
final List<Object[]> results =
|
||||
sqlLifecycle.runSimple(sql, QUERY_CONTEXT_DEFAULT, DEFAULT_PARAMETERS, authenticationResult).toList();
|
||||
for (Object[] result : results) {
|
||||
System.out.println(Arrays.toString(result));
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGroupBy2() throws Exception {
|
||||
SqlLifecycle sqlLifecycle = sqlLifecycleFactory.factorize();
|
||||
|
||||
final String sql = "SELECT __time,\n"
|
||||
+ " APPROX_COUNT_DISTINCT_HLLD(hll_dim1, 14) cnt\n"
|
||||
+ "FROM druid.foo group by __time order by cnt desc";
|
||||
|
||||
final List<Object[]> results =
|
||||
sqlLifecycle.runSimple(sql, QUERY_CONTEXT_DEFAULT, DEFAULT_PARAMETERS, authenticationResult).toList();
|
||||
for (Object[] result : results) {
|
||||
System.out.println(Arrays.toString(result));
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
package org.apache.druid.query.aggregation.sketch.hlld.sql;
|
||||
|
||||
|
||||
import com.alibaba.fastjson2.JSON;
|
||||
import com.fasterxml.jackson.databind.Module;
|
||||
import com.google.inject.Injector;
|
||||
import org.apache.druid.guice.DruidInjectorBuilder;
|
||||
import org.apache.druid.query.QueryRunnerFactoryConglomerate;
|
||||
import org.apache.druid.query.aggregation.sketch.hlld.HllModule;
|
||||
import org.apache.druid.segment.QueryableIndex;
|
||||
import org.apache.druid.segment.TestHelper;
|
||||
import org.apache.druid.segment.join.JoinableFactoryWrapper;
|
||||
import org.apache.druid.sql.calcite.BaseCalciteQueryTest;
|
||||
import org.apache.druid.sql.calcite.QueryTestBuilder;
|
||||
import org.apache.druid.sql.calcite.QueryTestRunner;
|
||||
import org.apache.druid.sql.calcite.util.CalciteTests;
|
||||
import org.apache.druid.sql.calcite.util.SpecificSegmentsQuerySegmentWalker;
|
||||
import org.apache.druid.timeline.DataSegment;
|
||||
import org.apache.druid.timeline.partition.LinearShardSpec;
|
||||
import org.junit.*;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.util.*;
|
||||
|
||||
// 新版本父类直接变了，实现更简单了
|
||||
public class HllApproxCountDistinctSqlAggregatorTest extends BaseCalciteQueryTest {
|
||||
private static final boolean ROUND = true;
|
||||
|
||||
@Override
|
||||
public void gatherProperties(Properties properties)
|
||||
{
|
||||
super.gatherProperties(properties);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void configureGuice(DruidInjectorBuilder builder)
|
||||
{
|
||||
super.configureGuice(builder);
|
||||
builder.addModule(new HllModule());
|
||||
}
|
||||
|
||||
|
||||
|
||||
@SuppressWarnings("resource")
|
||||
@Override
|
||||
public SpecificSegmentsQuerySegmentWalker createQuerySegmentWalker(
|
||||
final QueryRunnerFactoryConglomerate conglomerate,
|
||||
final JoinableFactoryWrapper joinableFactory,
|
||||
final Injector injector
|
||||
) throws IOException
|
||||
{
|
||||
HllModule.registerSerde();
|
||||
for (Module mod : new HllModule().getJacksonModules()) {
|
||||
CalciteTests.getJsonMapper().registerModule(mod);
|
||||
TestHelper.JSON_MAPPER.registerModule(mod);
|
||||
}
|
||||
|
||||
final QueryableIndex index = TestHelper.getTestIndexIO().loadIndex(new File("D:/doc/datas/testIndex-1369101812"));
|
||||
//final QueryableIndex index = TestHelper.getTestIndexIO().loadIndex(new File("D:/doc/datas/9_index"));
|
||||
/*final QueryableIndex index = IndexBuilder.create()
|
||||
.tmpDir(temporaryFolder.newFolder())
|
||||
.segmentWriteOutMediumFactory(OffHeapMemorySegmentWriteOutMediumFactory.instance())
|
||||
.schema(
|
||||
new IncrementalIndexSchema.Builder()
|
||||
.withMetrics(
|
||||
new CountAggregatorFactory("cnt"),
|
||||
new DoubleSumAggregatorFactory("m1", "m1"),
|
||||
new HllAggregatorFactory(
|
||||
"hll_dim1",
|
||||
"dim1",
|
||||
null,
|
||||
ROUND
|
||||
)
|
||||
)
|
||||
.withRollup(false)
|
||||
.build()
|
||||
)
|
||||
.rows(TestDataBuilder.ROWS1)
|
||||
.buildMMappedIndex();*/
|
||||
|
||||
return new SpecificSegmentsQuerySegmentWalker(conglomerate).add(
|
||||
DataSegment.builder()
|
||||
.dataSource(CalciteTests.DATASOURCE1)
|
||||
.interval(index.getDataInterval())
|
||||
.version("1")
|
||||
.shardSpec(new LinearShardSpec(0))
|
||||
.size(0)
|
||||
.build(),
|
||||
index
|
||||
);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testSqlQuery() throws Exception {
|
||||
// Can't vectorize due to SUBSTRING expression.
|
||||
cannotVectorize();
|
||||
|
||||
String[] columns = new String[]{"__time", "dim1", "dim2", "dim3", "cnt", "hll_dim1", "m1"};
|
||||
|
||||
String sql = "select " + String.join(",", columns) + " from druid.foo";
|
||||
QueryTestBuilder builder = testBuilder().sql(sql);
|
||||
builder.run();
|
||||
QueryTestRunner.QueryResults queryResults = builder.results();
|
||||
List<Object[]> results = queryResults.results;
|
||||
for (Object[] result : results) {
|
||||
Map row = new LinkedHashMap();
|
||||
for (int i = 0; i < result.length; i++) {
|
||||
row.put(columns[i], result[i]);
|
||||
}
|
||||
System.out.println(JSON.toJSONString(row));
|
||||
// System.out.println(Arrays.toString(result));
|
||||
}
|
||||
|
||||
for (int i = 0; i < columns.length; i++) {
|
||||
Object[] values = new Object[results.size()];
|
||||
for (int j = 0; j < results.size(); j++) {
|
||||
values[j] = results.get(j)[i];
|
||||
}
|
||||
System.out.println(columns[i] + ":" + Arrays.toString(values));
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testSqlQuery11() throws Exception {
|
||||
// Can't vectorize due to SUBSTRING expression.
|
||||
//cannotVectorize();
|
||||
|
||||
|
||||
String sql = "select HLLD(hll_dim1) hll_dim1 from (select hll_dim1 from druid.foo limit 5) t ";
|
||||
//sql = "select HLLD(hll_dim1) hll_dim1 from druid.foo t ";
|
||||
QueryTestBuilder builder = testBuilder().sql(sql).skipVectorize();;
|
||||
builder.run();
|
||||
QueryTestRunner.QueryResults queryResults = builder.results();
|
||||
List<Object[]> results = queryResults.results;
|
||||
for (Object[] result : results) {
|
||||
System.out.println(Arrays.toString(result));
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testSqlQuery12() throws Exception {
|
||||
// Can't vectorize due to SUBSTRING expression.
|
||||
cannotVectorize();
|
||||
|
||||
String sql = "select * from (select * from druid.foo limit 6) t where __time >= '1970-12-15 07:00:28' and __time < '2023-12-15 08:10:28' ";
|
||||
QueryTestBuilder builder = testBuilder().sql(sql);
|
||||
builder.run();
|
||||
QueryTestRunner.QueryResults queryResults = builder.results();
|
||||
List<Object[]> results = queryResults.results;
|
||||
for (Object[] result : results) {
|
||||
System.out.println(Arrays.toString(result));
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testSqlQuery1() throws Exception {
|
||||
// Can't vectorize due to SUBSTRING expression.
|
||||
cannotVectorize();
|
||||
|
||||
String sql = "select dim1 from druid.foo";
|
||||
QueryTestBuilder builder = testBuilder().sql(sql);
|
||||
builder.run();
|
||||
QueryTestRunner.QueryResults queryResults = builder.results();
|
||||
List<Object[]> results = queryResults.results;
|
||||
for (Object[] result : results) {
|
||||
System.out.println(Arrays.toString(result));
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testSqlQuery2() throws Exception {
|
||||
//cannotVectorize();
|
||||
//String sql = "select HLLD_ESTIMATE(HLLD(hll_dim1)) from druid.foo where dim1 = '1'";
|
||||
// Caused by: org.apache.calcite.sql.validate.SqlValidatorException: Aggregate expressions cannot be nested
|
||||
//String sql = "select HLLD_ESTIMATE(HLLD(hll_dim1)), APPROX_COUNT_DISTINCT_HLLD(HLLD(hll_dim1)), HLLD(hll_dim1) from druid.foo";
|
||||
String sql = "select HLLD_ESTIMATE(HLLD(hll_dim1)), APPROX_COUNT_DISTINCT_HLLD(hll_dim1), HLLD(hll_dim1) from (select HLLD(hll_dim1) hll_dim1 from druid.foo) t";
|
||||
QueryTestBuilder builder = testBuilder().sql(sql).skipVectorize();
|
||||
builder.run();
|
||||
QueryTestRunner.QueryResults queryResults = builder.results();
|
||||
List<Object[]> results = queryResults.results;
|
||||
for (Object[] result : results) {
|
||||
System.out.println(Arrays.toString(result));
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testSqlQuery3() throws Exception {
|
||||
//cannotVectorize();
|
||||
//String sql = "select HLLD_ESTIMATE(HLLD(hll_dim1)) from druid.foo where dim1 = ''";
|
||||
String sql = "select APPROX_COUNT_DISTINCT_HLLD(hll, 12) from (select HLLD(hll_dim1) hll from druid.foo where dim1 = '1') t ";
|
||||
QueryTestBuilder builder = testBuilder().sql(sql).skipVectorize();
|
||||
builder.run();
|
||||
QueryTestRunner.QueryResults queryResults = builder.results();
|
||||
List<Object[]> results = queryResults.results;
|
||||
for (Object[] result : results) {
|
||||
System.out.println(Arrays.toString(result));
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testSqlQuery4() throws Exception {
|
||||
//cannotVectorize();
|
||||
//String sql = "select HLLD_ESTIMATE(HLLD(hll_dim1)) from druid.foo where dim1 = ''";
|
||||
String sql = "select APPROX_COUNT_DISTINCT_HLLD(hll, 12) from (select HLLD(hll_dim1) hll from druid.foo where dim1 = '1') t ";
|
||||
QueryTestBuilder builder = testBuilder().sql(sql).skipVectorize();
|
||||
builder.run();
|
||||
QueryTestRunner.QueryResults queryResults = builder.results();
|
||||
List<Object[]> results = queryResults.results;
|
||||
for (Object[] result : results) {
|
||||
System.out.println(Arrays.toString(result));
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testSqlQuery5() throws Exception {
|
||||
//cannotVectorize();
|
||||
//String sql = "select HLLD_ESTIMATE(HLLD(hll_dim1)) from druid.foo where dim1 = ''";
|
||||
String sql = "select dim1,APPROX_COUNT_DISTINCT_HLLD(hll, 12) from (select dim1,HLLD(hll_dim1) hll from druid.foo where dim1 = '1' group by dim1) t group by dim1";
|
||||
QueryTestBuilder builder = testBuilder().sql(sql).skipVectorize();
|
||||
builder.run();
|
||||
QueryTestRunner.QueryResults queryResults = builder.results();
|
||||
List<Object[]> results = queryResults.results;
|
||||
for (Object[] result : results) {
|
||||
System.out.println(Arrays.toString(result));
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testSqlQuery6() throws Exception {
|
||||
//cannotVectorize();
|
||||
//String sql = "select HLLD_ESTIMATE(HLLD(hll_dim1)) from druid.foo where dim1 = ''";
|
||||
String sql = "select dim1,APPROX_COUNT_DISTINCT_HLLD(hll, 12) from (select dim1,HLLD(dim1) hll from druid.foo where dim1 = '1' group by dim1 limit 10) t group by dim1";
|
||||
//String sql = "select dim1,HLLD_ESTIMATE(HLLD(hll), false) from (select dim1,HLLD(dim1) hll from druid.foo where dim1 = '1' group by dim1 limit 10) t group by dim1";
|
||||
QueryTestBuilder builder = testBuilder().sql(sql).skipVectorize();
|
||||
builder.run();
|
||||
QueryTestRunner.QueryResults queryResults = builder.results();
|
||||
List<Object[]> results = queryResults.results;
|
||||
for (Object[] result : results) {
|
||||
System.out.println(Arrays.toString(result));
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testSqlQuery62() throws Exception {
|
||||
//cannotVectorize();
|
||||
//String sql = "select HLLD_ESTIMATE(HLLD(hll_dim1)) from druid.foo where dim1 = ''";
|
||||
String sql = "select dim1,APPROX_COUNT_DISTINCT_HLLD(hll) from (select dim1,HLLD(dim1) hll from druid.foo where dim1 = '1' group by dim1 limit 10) t group by dim1";
|
||||
QueryTestBuilder builder = testBuilder().sql(sql).skipVectorize();
|
||||
builder.run();
|
||||
QueryTestRunner.QueryResults queryResults = builder.results();
|
||||
List<Object[]> results = queryResults.results;
|
||||
for (Object[] result : results) {
|
||||
System.out.println(Arrays.toString(result));
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testSqlQuery7() throws Exception {
|
||||
//cannotVectorize();
|
||||
//String sql = "select HLLD_ESTIMATE(HLLD(hll_dim1)) from druid.foo where dim1 = ''";
|
||||
String sql = "select dim1,APPROX_COUNT_DISTINCT_HLLD(hll, 12) from (select dim1,HLLD(dim1) hll from druid.foo where dim1 = '1' group by dim1) t group by dim1 limit 10";
|
||||
QueryTestBuilder builder = testBuilder().sql(sql).skipVectorize();
|
||||
builder.run();
|
||||
QueryTestRunner.QueryResults queryResults = builder.results();
|
||||
List<Object[]> results = queryResults.results;
|
||||
for (Object[] result : results) {
|
||||
System.out.println(Arrays.toString(result));
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testAgg() throws Exception {
|
||||
final String sql = "SELECT\n"
|
||||
+ " SUM(cnt),\n"
|
||||
+ " APPROX_COUNT_DISTINCT_HLLD(hll_dim1)\n"
|
||||
+ "FROM druid.foo";
|
||||
|
||||
QueryTestBuilder builder = testBuilder().sql(sql).skipVectorize();
|
||||
builder.run();
|
||||
QueryTestRunner.QueryResults queryResults = builder.results();
|
||||
List<Object[]> results = queryResults.results;
|
||||
for (Object[] result : results) {
|
||||
System.out.println(Arrays.toString(result));
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testDistinct() throws Exception {
|
||||
final String sql = "SELECT\n"
|
||||
+ " SUM(cnt),\n"
|
||||
+ " APPROX_COUNT_DISTINCT_HLLD(dim2),\n" // uppercase
|
||||
+ " APPROX_COUNT_DISTINCT_HLLD(dim2) FILTER(WHERE dim2 <> ''),\n" // lowercase; also, filtered
|
||||
+ " APPROX_COUNT_DISTINCT_HLLD(SUBSTRING(dim2, 1, 1)),\n" // on extractionFn
|
||||
+ " APPROX_COUNT_DISTINCT_HLLD(SUBSTRING(dim2, 1, 1) || 'x'),\n" // on expression
|
||||
+ " APPROX_COUNT_DISTINCT_HLLD(hll_dim1, 16),\n" // on native HllSketch column
|
||||
+ " APPROX_COUNT_DISTINCT_HLLD(hll_dim1)\n" // on native HllSketch column
|
||||
+ "FROM druid.foo";
|
||||
|
||||
QueryTestBuilder builder = testBuilder().sql(sql).skipVectorize();
|
||||
builder.run();
|
||||
QueryTestRunner.QueryResults queryResults = builder.results();
|
||||
List<Object[]> results = queryResults.results;
|
||||
for (Object[] result : results) {
|
||||
System.out.println(Arrays.toString(result));
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testDistinct2() throws Exception {
|
||||
final String sql = "SELECT\n"
|
||||
+ " SUM(cnt),\n"
|
||||
+ " APPROX_COUNT_DISTINCT_HLLD(dim2),\n"
|
||||
+ " HLLD(dim2),\n"
|
||||
+ " HLLD(hll_dim1),\n"
|
||||
+ " HLLD_ESTIMATE(HLLD(dim2)),\n"
|
||||
+ " HLLD_ESTIMATE(HLLD(dim2), true),\n"
|
||||
+ " HLLD_ESTIMATE(HLLD(dim1), true),\n"
|
||||
+ " HLLD_ESTIMATE(HLLD(hll_dim1)),\n" // on native HllSketch column
|
||||
+ " APPROX_COUNT_DISTINCT_HLLD(hll_dim1)\n" // on native HllSketch column
|
||||
+ "FROM druid.foo";
|
||||
|
||||
QueryTestBuilder builder = testBuilder().sql(sql).skipVectorize();
|
||||
builder.run();
|
||||
QueryTestRunner.QueryResults queryResults = builder.results();
|
||||
List<Object[]> results = queryResults.results;
|
||||
for (Object[] result : results) {
|
||||
System.out.println(Arrays.toString(result));
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testDistinctDebug2() throws Exception {
|
||||
final String sql = "SELECT\n"
|
||||
+ " dim1, dim2\n"
|
||||
+ "FROM druid.foo";
|
||||
|
||||
QueryTestBuilder builder = testBuilder().sql(sql).skipVectorize();
|
||||
builder.run();
|
||||
QueryTestRunner.QueryResults queryResults = builder.results();
|
||||
List<Object[]> results = queryResults.results;
|
||||
for (Object[] result : results) {
|
||||
System.out.println(Arrays.toString(result));
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testDistinctDebug() throws Exception {
|
||||
final String sql = "SELECT\n"
|
||||
+ " SUM(cnt),\n"
|
||||
+ " APPROX_COUNT_DISTINCT_HLLD(dim2)\n"
|
||||
+ "FROM druid.foo";
|
||||
|
||||
QueryTestBuilder builder = testBuilder().sql(sql).skipVectorize();
|
||||
builder.run();
|
||||
QueryTestRunner.QueryResults queryResults = builder.results();
|
||||
List<Object[]> results = queryResults.results;
|
||||
for (Object[] result : results) {
|
||||
System.out.println(Arrays.toString(result));
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testDeser() throws Exception {
|
||||
final String sql = "SELECT\n"
|
||||
+ " APPROX_COUNT_DISTINCT_HLLD(hll_dim1) cnt\n"
|
||||
+ "FROM druid.foo";
|
||||
|
||||
QueryTestBuilder builder = testBuilder().sql(sql).skipVectorize();
|
||||
builder.run();
|
||||
QueryTestRunner.QueryResults queryResults = builder.results();
|
||||
List<Object[]> results = queryResults.results;
|
||||
for (Object[] result : results) {
|
||||
System.out.println(Arrays.toString(result));
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
@Test
|
||||
public void testGroupBy() throws Exception {
|
||||
final String sql = "SELECT cnt,\n"
|
||||
+ " APPROX_COUNT_DISTINCT_HLLD(hll_dim1, 14) cnt2\n"
|
||||
+ "FROM druid.foo group by cnt";
|
||||
|
||||
QueryTestBuilder builder = testBuilder().sql(sql).skipVectorize();
|
||||
builder.run();
|
||||
QueryTestRunner.QueryResults queryResults = builder.results();
|
||||
List<Object[]> results = queryResults.results;
|
||||
for (Object[] result : results) {
|
||||
System.out.println(Arrays.toString(result));
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGroupBy1() throws Exception {
|
||||
final String sql = "SELECT __time,\n"
|
||||
+ " APPROX_COUNT_DISTINCT_HLLD(hll_dim1, 14) cnt\n"
|
||||
+ "FROM druid.foo group by __time";
|
||||
|
||||
QueryTestBuilder builder = testBuilder().sql(sql).skipVectorize();
|
||||
builder.run();
|
||||
QueryTestRunner.QueryResults queryResults = builder.results();
|
||||
List<Object[]> results = queryResults.results;
|
||||
for (Object[] result : results) {
|
||||
System.out.println(Arrays.toString(result));
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGroupBy2() throws Exception {
|
||||
final String sql = "SELECT __time,\n"
|
||||
+ " APPROX_COUNT_DISTINCT_HLLD(hll_dim1, 14) cnt\n"
|
||||
+ "FROM druid.foo group by __time order by cnt desc";
|
||||
QueryTestBuilder builder = testBuilder().sql(sql).skipVectorize();
|
||||
builder.run();
|
||||
QueryTestRunner.QueryResults queryResults = builder.results();
|
||||
List<Object[]> results = queryResults.results;
|
||||
for (Object[] result : results) {
|
||||
System.out.println(Arrays.toString(result));
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
143
druid-udf/pom.xml
Normal file
143
druid-udf/pom.xml
Normal file
@@ -0,0 +1,143 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0"
|
||||
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
|
||||
<groupId>org.example</groupId>
|
||||
<artifactId>druid-udf_26.0.0</artifactId>
|
||||
<name>druid-udf</name>
|
||||
<version>1.0-SNAPSHOT</version>
|
||||
|
||||
<properties>
|
||||
<maven.compiler.source>11</maven.compiler.source>
|
||||
<maven.compiler.target>11</maven.compiler.target>
|
||||
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
|
||||
<druid.version>26.0.0</druid.version>
|
||||
</properties>
|
||||
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>org.apache.druid</groupId>
|
||||
<artifactId>druid-server</artifactId>
|
||||
<version>${druid.version}</version>
|
||||
<scope>provided</scope>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.apache.druid</groupId>
|
||||
<artifactId>druid-sql</artifactId>
|
||||
<version>${druid.version}</version>
|
||||
<scope>provided</scope>
|
||||
</dependency>
|
||||
|
||||
<!-- Tests -->
|
||||
|
||||
<dependency>
|
||||
<groupId>org.easymock</groupId>
|
||||
<artifactId>easymock</artifactId>
|
||||
<version>4.3</version>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.apache.druid</groupId>
|
||||
<artifactId>druid-processing</artifactId>
|
||||
<version>${druid.version}</version>
|
||||
<type>test-jar</type>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.druid</groupId>
|
||||
<artifactId>druid-server</artifactId>
|
||||
<version>${druid.version}</version>
|
||||
<scope>test</scope>
|
||||
<type>test-jar</type>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.druid</groupId>
|
||||
<artifactId>druid-sql</artifactId>
|
||||
<version>${druid.version}</version>
|
||||
<type>test-jar</type>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>junit</groupId>
|
||||
<artifactId>junit</artifactId>
|
||||
<version>4.12</version>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>com.alibaba.fastjson2</groupId>
|
||||
<artifactId>fastjson2</artifactId>
|
||||
<version>2.0.34</version>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
|
||||
<build>
|
||||
<plugins>
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-compiler-plugin</artifactId>
|
||||
<version>3.1</version>
|
||||
<configuration>
|
||||
<compilerArgument>-Xlint:unchecked</compilerArgument>
|
||||
<source>11</source>
|
||||
<target>11</target>
|
||||
</configuration>
|
||||
</plugin>
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-surefire-plugin</artifactId>
|
||||
<version>2.19.1</version>
|
||||
<configuration>
|
||||
<argLine>-Duser.timezone=UTC</argLine>
|
||||
<redirectTestOutputToFile>true</redirectTestOutputToFile>
|
||||
</configuration>
|
||||
</plugin>
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-assembly-plugin</artifactId>
|
||||
<version>2.5.5</version>
|
||||
<executions>
|
||||
<execution>
|
||||
<id>distro-assembly</id>
|
||||
<phase>package</phase>
|
||||
<goals>
|
||||
<goal>single</goal>
|
||||
</goals>
|
||||
<configuration>
|
||||
<finalName>${project.artifactId}-${project.version}</finalName>
|
||||
<tarLongFileMode>posix</tarLongFileMode>
|
||||
<descriptors>
|
||||
<descriptor>src/assembly/assembly.xml</descriptor>
|
||||
</descriptors>
|
||||
</configuration>
|
||||
</execution>
|
||||
</executions>
|
||||
</plugin>
|
||||
<plugin>
|
||||
<artifactId>maven-release-plugin</artifactId>
|
||||
<version>2.5.3</version>
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>org.apache.maven.scm</groupId>
|
||||
<artifactId>maven-scm-provider-gitexe</artifactId>
|
||||
<version>1.9.4</version>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
</plugin>
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-jar-plugin</artifactId>
|
||||
<version>3.0.2</version>
|
||||
<configuration>
|
||||
<archive>
|
||||
<addMavenDescriptor>false</addMavenDescriptor>
|
||||
</archive>
|
||||
</configuration>
|
||||
</plugin>
|
||||
</plugins>
|
||||
</build>
|
||||
</project>
|
||||
54
druid-udf/src/assembly/assembly.xml
Normal file
54
druid-udf/src/assembly/assembly.xml
Normal file
@@ -0,0 +1,54 @@
|
||||
<?xml version="1.0"?>
|
||||
<!--
|
||||
~ Copyright 2016 Imply Data, Inc.
|
||||
~
|
||||
~ Licensed under the Apache License, Version 2.0 (the "License");
|
||||
~ you may not use this file except in compliance with the License.
|
||||
~ You may obtain a copy of the License at
|
||||
~
|
||||
~ http://www.apache.org/licenses/LICENSE-2.0
|
||||
~
|
||||
~ Unless required by applicable law or agreed to in writing, software
|
||||
~ distributed under the License is distributed on an "AS IS" BASIS,
|
||||
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
~ See the License for the specific language governing permissions and
|
||||
~ limitations under the License.
|
||||
-->
|
||||
|
||||
<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.3"
|
||||
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.3 http://maven.apache.org/xsd/assembly-1.1.3.xsd">
|
||||
<id>bin</id>
|
||||
<formats>
|
||||
<format>tar.gz</format>
|
||||
</formats>
|
||||
|
||||
<baseDirectory>${project.name}</baseDirectory>
|
||||
|
||||
<dependencySets>
|
||||
<dependencySet>
|
||||
<useProjectArtifact>false</useProjectArtifact>
|
||||
<useTransitiveDependencies>true</useTransitiveDependencies>
|
||||
<outputDirectory>.</outputDirectory>
|
||||
<unpack>false</unpack>
|
||||
</dependencySet>
|
||||
</dependencySets>
|
||||
|
||||
<fileSets>
|
||||
<fileSet>
|
||||
<directory>.</directory>
|
||||
<outputDirectory/>
|
||||
<includes>
|
||||
<include>README.md</include>
|
||||
<include>LICENSE</include>
|
||||
</includes>
|
||||
</fileSet>
|
||||
<fileSet>
|
||||
<directory>${project.build.directory}</directory>
|
||||
<outputDirectory>.</outputDirectory>
|
||||
<includes>
|
||||
<include>*.jar</include>
|
||||
</includes>
|
||||
</fileSet>
|
||||
</fileSets>
|
||||
</assembly>
|
||||
@@ -0,0 +1,23 @@
|
||||
package org.apache.druid.query.udf;
|
||||
|
||||
import com.google.inject.Binder;
|
||||
import org.apache.druid.guice.ExpressionModule;
|
||||
import org.apache.druid.initialization.DruidModule;
|
||||
import org.apache.druid.query.udf.expressions.DimensionBucketExprMacro;
|
||||
import org.apache.druid.query.udf.sql.DimensionBucketOperatorConversion;
|
||||
import org.apache.druid.sql.guice.SqlBindings;
|
||||
|
||||
public class UdfModule implements DruidModule {
|
||||
@Override
|
||||
public void configure(Binder binder) {
|
||||
SqlBindings.addOperatorConversion(binder, DimensionBucketOperatorConversion.class);
|
||||
ExpressionModule.addExprMacro(binder, DimensionBucketExprMacro.class);
|
||||
}
|
||||
|
||||
/*@Override
|
||||
public List<? extends Module> getJacksonModules() {
|
||||
// Register Jackson module for any classes we need to be able to use in JSON queries or ingestion specs.
|
||||
return Collections.<Module>singletonList(new SimpleModule("UdfModule"));
|
||||
}*/
|
||||
|
||||
}
|
||||
@@ -0,0 +1,82 @@
|
||||
package org.apache.druid.query.udf.expressions;
|
||||
|
||||
import org.apache.druid.math.expr.*;
|
||||
import org.apache.druid.math.expr.ExprMacroTable.ExprMacro;
|
||||
|
||||
import javax.annotation.Nullable;
|
||||
import java.util.List;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
public class DimensionBucketExprMacro implements ExprMacro {
|
||||
private static final String NAME = "dimension_bucket";
|
||||
|
||||
@Override
|
||||
public String name() {
|
||||
return NAME;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Expr apply(List<Expr> args) {
|
||||
validationHelperCheckMinArgumentCount(args, 2);
|
||||
Expr bucketCnt = args.get(0);
|
||||
if(!bucketCnt.isLiteral()|| bucketCnt.eval(InputBindings.nilBindings()).asInt() <= 0) {
|
||||
throw validationFailed("first bucketCount argument must is int literal and > 0");
|
||||
}
|
||||
return new DimensionBucketExpr(args);
|
||||
}
|
||||
|
||||
static class DimensionBucketExpr extends ExprMacroTable.BaseScalarMacroFunctionExpr {
|
||||
private final int bucketCount;
|
||||
|
||||
public DimensionBucketExpr(List<Expr> args) {
|
||||
super(NAME, args);
|
||||
bucketCount = args.get(0).eval(InputBindings.nilBindings()).asInt();
|
||||
}
|
||||
|
||||
@Override
|
||||
public ExprEval eval(ObjectBinding bindings) {
|
||||
int result = 1;
|
||||
for (int i = 1; i < args.size(); i++) {
|
||||
ExprEval eval = args.get(i).eval(bindings);
|
||||
Object element = eval.value();
|
||||
if(element instanceof Object[]){
|
||||
for (Object ele : (Object[]) element) {
|
||||
result = 31 * result + (ele == null ? 0 : ele.hashCode());
|
||||
}
|
||||
}else{
|
||||
result = 31 * result + (element == null ? 0 : element.hashCode());
|
||||
}
|
||||
|
||||
/*else if (element instanceof Number) {
|
||||
//result = 31 * result + Integer.hashCode(((Number)element).intValue());
|
||||
result = 31 * result + Long.hashCode(((Number)element).longValue());
|
||||
}*/
|
||||
}
|
||||
|
||||
int bucket = Math.abs(result) % bucketCount;
|
||||
return ExprEval.of(IntToHexUtil.uInt16ToHexStringFast(bucket));
|
||||
}
|
||||
|
||||
@Override
|
||||
public Expr visit(Shuttle shuttle) {
|
||||
List<Expr> newArgs = args.stream().map(x -> x.visit(shuttle)).collect(Collectors.toList());
|
||||
return shuttle.visit(new DimensionBucketExpr(newArgs));
|
||||
}
|
||||
|
||||
@Override
|
||||
public BindingAnalysis analyzeInputs() {
|
||||
return super.analyzeInputs();
|
||||
}
|
||||
|
||||
@Nullable
|
||||
@Override
|
||||
public ExpressionType getOutputType(InputBindingInspector inspector) {
|
||||
return ExpressionType.STRING;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean canVectorize(InputBindingInspector inspector) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,45 @@
|
||||
package org.apache.druid.query.udf.expressions;
|
||||
|
||||
import java.nio.charset.StandardCharsets;
|
||||
|
||||
public class IntToHexUtil {
|
||||
static final byte[] digits = {
|
||||
'0' , '1' , '2' , '3' , '4' , '5' ,
|
||||
'6' , '7' , '8' , '9' , 'a' , 'b' ,
|
||||
'c' , 'd' , 'e' , 'f' , 'g' , 'h' ,
|
||||
'i' , 'j' , 'k' , 'l' , 'm' , 'n' ,
|
||||
'o' , 'p' , 'q' , 'r' , 's' , 't' ,
|
||||
'u' , 'v' , 'w' , 'x' , 'y' , 'z'
|
||||
};
|
||||
static final String[] uInt16HexsCache;
|
||||
static final int uInt16HexsCacheSize = 8192;
|
||||
|
||||
static{
|
||||
uInt16HexsCache = new String[uInt16HexsCacheSize];
|
||||
for (int i = 0; i < uInt16HexsCacheSize; i++) {
|
||||
uInt16HexsCache[i] = uInt16ToHexString(i);
|
||||
}
|
||||
}
|
||||
|
||||
public static String uInt16ToHexStringFast(int i){
|
||||
if(i < uInt16HexsCacheSize){
|
||||
return uInt16HexsCache[i];
|
||||
}else{
|
||||
return uInt16ToHexString(i);
|
||||
}
|
||||
}
|
||||
|
||||
private static String uInt16ToHexString(int i){
|
||||
byte[] bytes = new byte[4];
|
||||
int mask = 15; // 16 - 1
|
||||
int value = i;
|
||||
bytes[3] = digits[value & mask];
|
||||
value >>>= 4;
|
||||
bytes[2] = digits[value & mask];
|
||||
value >>>= 4;
|
||||
bytes[1] = digits[value & mask];
|
||||
value >>>= 4;
|
||||
bytes[0] = digits[value & mask];
|
||||
return new String(bytes, StandardCharsets.US_ASCII);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,43 @@
|
||||
package org.apache.druid.query.udf.sql;
|
||||
|
||||
import org.apache.calcite.rex.RexNode;
|
||||
import org.apache.calcite.sql.SqlFunction;
|
||||
import org.apache.calcite.sql.SqlFunctionCategory;
|
||||
import org.apache.calcite.sql.SqlKind;
|
||||
import org.apache.calcite.sql.SqlOperator;
|
||||
import org.apache.calcite.sql.type.OperandTypes;
|
||||
import org.apache.calcite.sql.type.ReturnTypes;
|
||||
import org.apache.calcite.sql.type.SqlOperandCountRanges;
|
||||
import org.apache.calcite.sql.type.SqlTypeName;
|
||||
import org.apache.druid.segment.column.RowSignature;
|
||||
import org.apache.druid.sql.calcite.expression.DruidExpression;
|
||||
import org.apache.druid.sql.calcite.expression.OperatorConversions;
|
||||
import org.apache.druid.sql.calcite.expression.SqlOperatorConversion;
|
||||
import org.apache.druid.sql.calcite.planner.Calcites;
|
||||
import org.apache.druid.sql.calcite.planner.PlannerContext;
|
||||
|
||||
import javax.annotation.Nullable;
|
||||
|
||||
public class DimensionBucketOperatorConversion implements SqlOperatorConversion {
|
||||
private static final SqlFunction SQL_FUNCTION = new SqlFunction(
|
||||
"DIMENSION_BUCKET",
|
||||
SqlKind.OTHER_FUNCTION,
|
||||
ReturnTypes.explicit(
|
||||
factory -> Calcites.createSqlTypeWithNullability(factory, SqlTypeName.VARCHAR, true)
|
||||
),
|
||||
null,
|
||||
OperandTypes.variadic(SqlOperandCountRanges.from(2)),
|
||||
SqlFunctionCategory.USER_DEFINED_FUNCTION
|
||||
);
|
||||
|
||||
@Override
|
||||
public SqlOperator calciteOperator() {
|
||||
return SQL_FUNCTION;
|
||||
}
|
||||
|
||||
@Nullable
|
||||
@Override
|
||||
public DruidExpression toDruidExpression(PlannerContext plannerContext, RowSignature rowSignature, RexNode rexNode) {
|
||||
return OperatorConversions.convertDirectCall(plannerContext, rowSignature, rexNode, "dimension_bucket");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
org.apache.druid.query.udf.UdfModule
|
||||
@@ -0,0 +1,146 @@
|
||||
package org.apache.druid.query.udf.expressions;
|
||||
|
||||
import com.google.common.collect.ImmutableMap;
|
||||
import org.apache.druid.math.expr.*;
|
||||
import org.apache.druid.testing.InitializedNullHandlingTest;
|
||||
import org.junit.Test;
|
||||
|
||||
import java.util.Collections;
|
||||
|
||||
public class DimensionBucketExprTest extends InitializedNullHandlingTest {
|
||||
private final ExprMacroTable exprMacroTable = new ExprMacroTable(Collections.singletonList(new DimensionBucketExprMacro()));
|
||||
Expr.ObjectBinding inputBindings = InputBindings.forInputSuppliers(
|
||||
new ImmutableMap.Builder<String, InputBindings.InputSupplier>()
|
||||
.put("string", InputBindings.inputSupplier(ExpressionType.STRING, () -> "abcdef"))
|
||||
.put("long", InputBindings.inputSupplier(ExpressionType.LONG, () -> 1234L))
|
||||
.put("double", InputBindings.inputSupplier(ExpressionType.DOUBLE, () -> 1.234))
|
||||
.put("array1", InputBindings.inputSupplier(ExpressionType.STRING_ARRAY, () -> new Object[]{"1", "2", "3"}))
|
||||
.put("array2", InputBindings.inputSupplier(ExpressionType.STRING_ARRAY, () -> new String[]{"1", "2", "3"}))
|
||||
.put("nullString", InputBindings.inputSupplier(ExpressionType.STRING, () -> null))
|
||||
.put("nullLong", InputBindings.inputSupplier(ExpressionType.LONG, () -> null))
|
||||
.put("nullDouble", InputBindings.inputSupplier(ExpressionType.DOUBLE, () -> null))
|
||||
.build()
|
||||
);
|
||||
|
||||
Expr.ObjectBinding[] inputBindingArray = new Expr.ObjectBinding[]{
|
||||
InputBindings.forInputSuppliers(
|
||||
new ImmutableMap.Builder<String, InputBindings.InputSupplier>()
|
||||
.put("device_id", InputBindings.inputSupplier(ExpressionType.STRING, () -> "1"))
|
||||
.put("rule_id", InputBindings.inputSupplier(ExpressionType.LONG, () -> 81))
|
||||
.put("template_id", InputBindings.inputSupplier(ExpressionType.LONG, () -> 81))
|
||||
.put("chart_id", InputBindings.inputSupplier(ExpressionType.LONG, () -> 81))
|
||||
.put("version", InputBindings.inputSupplier(ExpressionType.LONG, () -> 1))
|
||||
.put("client_ip_object", InputBindings.inputSupplier(ExpressionType.STRING_ARRAY, () -> null))
|
||||
.put("server_ip_object", InputBindings.inputSupplier(ExpressionType.STRING_ARRAY, () -> null))
|
||||
.put("fqdn_category", InputBindings.inputSupplier(ExpressionType.STRING_ARRAY, () -> null))
|
||||
.put("client_ip", InputBindings.inputSupplier(ExpressionType.STRING, () -> null))
|
||||
.put("server_ip", InputBindings.inputSupplier(ExpressionType.STRING, () -> null))
|
||||
.put("server_fqdn", InputBindings.inputSupplier(ExpressionType.STRING, () -> null))
|
||||
.put("server_domain", InputBindings.inputSupplier(ExpressionType.STRING, () -> null))
|
||||
.put("application", InputBindings.inputSupplier(ExpressionType.STRING, () -> null))
|
||||
.build()
|
||||
),
|
||||
InputBindings.forInputSuppliers(
|
||||
new ImmutableMap.Builder<String, InputBindings.InputSupplier>()
|
||||
.put("device_id", InputBindings.inputSupplier(ExpressionType.STRING, () -> "1"))
|
||||
.put("rule_id", InputBindings.inputSupplier(ExpressionType.LONG, () -> 101))
|
||||
.put("template_id", InputBindings.inputSupplier(ExpressionType.LONG, () -> 101))
|
||||
.put("chart_id", InputBindings.inputSupplier(ExpressionType.LONG, () -> 101))
|
||||
.put("version", InputBindings.inputSupplier(ExpressionType.LONG, () -> 1))
|
||||
.put("client_ip_object", InputBindings.inputSupplier(ExpressionType.STRING_ARRAY, () -> new Object[]{"5","7","8"}))
|
||||
.put("server_ip_object", InputBindings.inputSupplier(ExpressionType.STRING_ARRAY, () -> null))
|
||||
.put("fqdn_category", InputBindings.inputSupplier(ExpressionType.STRING_ARRAY, () -> null))
|
||||
.put("client_ip", InputBindings.inputSupplier(ExpressionType.STRING, () -> null))
|
||||
.put("server_ip", InputBindings.inputSupplier(ExpressionType.STRING, () -> null))
|
||||
.put("server_fqdn", InputBindings.inputSupplier(ExpressionType.STRING, () -> null))
|
||||
.put("server_domain", InputBindings.inputSupplier(ExpressionType.STRING, () -> null))
|
||||
.put("application", InputBindings.inputSupplier(ExpressionType.STRING, () -> null))
|
||||
.build()
|
||||
),
|
||||
InputBindings.forInputSuppliers(
|
||||
new ImmutableMap.Builder<String, InputBindings.InputSupplier>()
|
||||
.put("device_id", InputBindings.inputSupplier(ExpressionType.STRING, () -> "1"))
|
||||
.put("rule_id", InputBindings.inputSupplier(ExpressionType.LONG, () -> 271L))
|
||||
.put("template_id", InputBindings.inputSupplier(ExpressionType.LONG, () -> 271L))
|
||||
.put("chart_id", InputBindings.inputSupplier(ExpressionType.LONG, () -> 271L))
|
||||
.put("version", InputBindings.inputSupplier(ExpressionType.LONG, () -> 1L))
|
||||
.put("client_ip_object", InputBindings.inputSupplier(ExpressionType.STRING_ARRAY, () -> null))
|
||||
.put("server_ip_object", InputBindings.inputSupplier(ExpressionType.STRING_ARRAY, () -> null))
|
||||
.put("fqdn_category", InputBindings.inputSupplier(ExpressionType.STRING_ARRAY, () -> null))
|
||||
.put("client_ip", InputBindings.inputSupplier(ExpressionType.STRING, () -> null))
|
||||
.put("server_ip", InputBindings.inputSupplier(ExpressionType.STRING, () -> "5.245.228.51"))
|
||||
.put("server_fqdn", InputBindings.inputSupplier(ExpressionType.STRING, () -> null))
|
||||
.put("server_domain", InputBindings.inputSupplier(ExpressionType.STRING, () -> null))
|
||||
.put("application", InputBindings.inputSupplier(ExpressionType.STRING, () -> null))
|
||||
.build()
|
||||
),
|
||||
// ...
|
||||
InputBindings.forInputSuppliers(
|
||||
new ImmutableMap.Builder<String, InputBindings.InputSupplier>()
|
||||
.put("device_id", InputBindings.inputSupplier(ExpressionType.STRING, () -> "1"))
|
||||
.put("rule_id", InputBindings.inputSupplier(ExpressionType.LONG, () -> 81))
|
||||
.put("template_id", InputBindings.inputSupplier(ExpressionType.LONG, () -> 81))
|
||||
.put("chart_id", InputBindings.inputSupplier(ExpressionType.LONG, () -> 81))
|
||||
.put("version", InputBindings.inputSupplier(ExpressionType.LONG, () -> 1))
|
||||
.put("client_ip_object", InputBindings.inputSupplier(ExpressionType.STRING, () -> null))
|
||||
.put("server_ip_object", InputBindings.inputSupplier(ExpressionType.STRING, () -> null))
|
||||
.put("fqdn_category", InputBindings.inputSupplier(ExpressionType.STRING, () -> null))
|
||||
.put("client_ip", InputBindings.inputSupplier(ExpressionType.STRING, () -> null))
|
||||
.put("server_ip", InputBindings.inputSupplier(ExpressionType.STRING, () -> null))
|
||||
.put("server_fqdn", InputBindings.inputSupplier(ExpressionType.STRING, () -> null))
|
||||
.put("server_domain", InputBindings.inputSupplier(ExpressionType.STRING, () -> null))
|
||||
.put("application", InputBindings.inputSupplier(ExpressionType.STRING, () -> null))
|
||||
.build()
|
||||
),
|
||||
InputBindings.forInputSuppliers(
|
||||
new ImmutableMap.Builder<String, InputBindings.InputSupplier>()
|
||||
.put("device_id", InputBindings.inputSupplier(ExpressionType.STRING, () -> "1"))
|
||||
.put("rule_id", InputBindings.inputSupplier(ExpressionType.LONG, () -> 101))
|
||||
.put("template_id", InputBindings.inputSupplier(ExpressionType.LONG, () -> 101))
|
||||
.put("chart_id", InputBindings.inputSupplier(ExpressionType.LONG, () -> 101))
|
||||
.put("version", InputBindings.inputSupplier(ExpressionType.LONG, () -> 1))
|
||||
.put("client_ip_object", InputBindings.inputSupplier(ExpressionType.STRING, () -> "5,7,8"))
|
||||
.put("server_ip_object", InputBindings.inputSupplier(ExpressionType.STRING, () -> null))
|
||||
.put("fqdn_category", InputBindings.inputSupplier(ExpressionType.STRING, () -> null))
|
||||
.put("client_ip", InputBindings.inputSupplier(ExpressionType.STRING, () -> null))
|
||||
.put("server_ip", InputBindings.inputSupplier(ExpressionType.STRING, () -> null))
|
||||
.put("server_fqdn", InputBindings.inputSupplier(ExpressionType.STRING, () -> null))
|
||||
.put("server_domain", InputBindings.inputSupplier(ExpressionType.STRING, () -> null))
|
||||
.put("application", InputBindings.inputSupplier(ExpressionType.STRING, () -> null))
|
||||
.build()
|
||||
),
|
||||
InputBindings.forInputSuppliers(
|
||||
new ImmutableMap.Builder<String, InputBindings.InputSupplier>()
|
||||
.put("device_id", InputBindings.inputSupplier(ExpressionType.STRING, () -> "1"))
|
||||
.put("rule_id", InputBindings.inputSupplier(ExpressionType.LONG, () -> 271L))
|
||||
.put("template_id", InputBindings.inputSupplier(ExpressionType.LONG, () -> 271L))
|
||||
.put("chart_id", InputBindings.inputSupplier(ExpressionType.LONG, () -> 271L))
|
||||
.put("version", InputBindings.inputSupplier(ExpressionType.LONG, () -> 1L))
|
||||
.put("client_ip_object", InputBindings.inputSupplier(ExpressionType.STRING, () -> null))
|
||||
.put("server_ip_object", InputBindings.inputSupplier(ExpressionType.STRING, () -> null))
|
||||
.put("fqdn_category", InputBindings.inputSupplier(ExpressionType.STRING, () -> null))
|
||||
.put("client_ip", InputBindings.inputSupplier(ExpressionType.STRING, () -> null))
|
||||
.put("server_ip", InputBindings.inputSupplier(ExpressionType.STRING, () -> "5.245.228.51"))
|
||||
.put("server_fqdn", InputBindings.inputSupplier(ExpressionType.STRING, () -> null))
|
||||
.put("server_domain", InputBindings.inputSupplier(ExpressionType.STRING, () -> null))
|
||||
.put("application", InputBindings.inputSupplier(ExpressionType.STRING, () -> null))
|
||||
.build()
|
||||
),
|
||||
};
|
||||
|
||||
@Test
|
||||
public void test() {
|
||||
Expr expr = Parser.parse("dimension_bucket(1024, 100, 'aaa', string,long,double,array1, array2, nullString, nullLong)", exprMacroTable);
|
||||
ExprEval eval = expr.eval(inputBindings);
|
||||
System.out.println(eval.value());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void test2() {
|
||||
for (Expr.ObjectBinding objectBinding : inputBindingArray) {
|
||||
Expr expr = Parser.parse("dimension_bucket(1024, device_id, rule_id, template_id, chart_id, version, client_ip_object, server_ip_object, fqdn_category, client_ip, server_ip, server_fqdn, server_domain, application)", exprMacroTable);
|
||||
ExprEval eval = expr.eval(objectBinding);
|
||||
System.out.println(objectBinding.get("rule_id") + ", bucket_id:" + eval.value());
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user