Merge branch 'develop' into 'main'
Develop See merge request galaxy/platform/algorithm/druid-extensions!4
This commit is contained in:
李奉超
@@ -1,361 +1,388 @@
|
|||||||
package org.HdrHistogram; /**
|
package org.HdrHistogram; /**
|
||||||
* Written by Gil Tene of Azul Systems, and released to the public domain,
|
* Written by Gil Tene of Azul Systems, and released to the public domain,
|
||||||
* as explained at http://creativecommons.org/publicdomain/zero/1.0/
|
* as explained at http://creativecommons.org/publicdomain/zero/1.0/
|
||||||
*
|
*
|
||||||
* @author Gil Tene
|
* @author Gil Tene
|
||||||
*/
|
*/
|
||||||
|
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
import java.io.ObjectInputStream;
|
import java.io.ObjectInputStream;
|
||||||
import java.nio.ByteBuffer;
|
import java.nio.ByteBuffer;
|
||||||
import java.util.ArrayList;
|
import java.util.*;
|
||||||
import java.util.Arrays;
|
import java.util.zip.DataFormatException;
|
||||||
import java.util.List;
|
|
||||||
import java.util.zip.DataFormatException;
|
/**
|
||||||
|
* <h3>A High Dynamic Range (HDR) Histogram</h3>
|
||||||
/**
|
* <p>
|
||||||
* <h3>A High Dynamic Range (HDR) Histogram</h3>
|
* {@link ArrayHistogram} supports the recording and analyzing sampled data value counts across a configurable integer value
|
||||||
* <p>
|
* range with configurable value precision within the range. Value precision is expressed as the number of significant
|
||||||
* {@link ArrayHistogram} supports the recording and analyzing sampled data value counts across a configurable integer value
|
* digits in the value recording, and provides control over value quantization behavior across the value range and the
|
||||||
* range with configurable value precision within the range. Value precision is expressed as the number of significant
|
* subsequent value resolution at any given level.
|
||||||
* digits in the value recording, and provides control over value quantization behavior across the value range and the
|
* <p>
|
||||||
* subsequent value resolution at any given level.
|
* For example, a Histogram could be configured to track the counts of observed integer values between 0 and
|
||||||
* <p>
|
* 3,600,000,000 while maintaining a value precision of 3 significant digits across that range. Value quantization
|
||||||
* For example, a Histogram could be configured to track the counts of observed integer values between 0 and
|
* within the range will thus be no larger than 1/1,000th (or 0.1%) of any value. This example Histogram could
|
||||||
* 3,600,000,000 while maintaining a value precision of 3 significant digits across that range. Value quantization
|
* be used to track and analyze the counts of observed response times ranging between 1 microsecond and 1 hour
|
||||||
* within the range will thus be no larger than 1/1,000th (or 0.1%) of any value. This example Histogram could
|
* in magnitude, while maintaining a value resolution of 1 microsecond up to 1 millisecond, a resolution of
|
||||||
* be used to track and analyze the counts of observed response times ranging between 1 microsecond and 1 hour
|
* 1 millisecond (or better) up to one second, and a resolution of 1 second (or better) up to 1,000 seconds. At its
|
||||||
* in magnitude, while maintaining a value resolution of 1 microsecond up to 1 millisecond, a resolution of
|
* maximum tracked value (1 hour), it would still maintain a resolution of 3.6 seconds (or better).
|
||||||
* 1 millisecond (or better) up to one second, and a resolution of 1 second (or better) up to 1,000 seconds. At its
|
* <p>
|
||||||
* maximum tracked value (1 hour), it would still maintain a resolution of 3.6 seconds (or better).
|
* Histogram tracks value counts in <b><code>long</code></b> fields. Smaller field types are available in the
|
||||||
* <p>
|
* {@link IntCountsHistogram} and {@link ShortCountsHistogram} implementations of
|
||||||
* Histogram tracks value counts in <b><code>long</code></b> fields. Smaller field types are available in the
|
* {@link AbstractHistogram}.
|
||||||
* {@link IntCountsHistogram} and {@link ShortCountsHistogram} implementations of
|
* <p>
|
||||||
* {@link AbstractHistogram}.
|
* Auto-resizing: When constructed with no specified value range range (or when auto-resize is turned on with {@link
|
||||||
* <p>
|
* ArrayHistogram#setAutoResize}) a {@link ArrayHistogram} will auto-resize its dynamic range to include recorded values as
|
||||||
* Auto-resizing: When constructed with no specified value range range (or when auto-resize is turned on with {@link
|
* they are encountered. Note that recording calls that cause auto-resizing may take longer to execute, as resizing
|
||||||
* ArrayHistogram#setAutoResize}) a {@link ArrayHistogram} will auto-resize its dynamic range to include recorded values as
|
* incurs allocation and copying of internal data structures.
|
||||||
* they are encountered. Note that recording calls that cause auto-resizing may take longer to execute, as resizing
|
* <p>
|
||||||
* incurs allocation and copying of internal data structures.
|
* See package description for {@link org.HdrHistogram} for details.
|
||||||
* <p>
|
*/
|
||||||
* See package description for {@link org.HdrHistogram} for details.
|
|
||||||
*/
|
public class ArrayHistogram extends AbstractHistogram implements Histogramer{
|
||||||
|
long totalCount;
|
||||||
public class ArrayHistogram extends AbstractHistogram implements Histogramer{
|
long[] counts;
|
||||||
long totalCount;
|
int normalizingIndexOffset;
|
||||||
long[] counts;
|
|
||||||
int normalizingIndexOffset;
|
@Override
|
||||||
|
long getCountAtIndex(final int index) {
|
||||||
@Override
|
return counts[normalizeIndex(index, normalizingIndexOffset, countsArrayLength)];
|
||||||
long getCountAtIndex(final int index) {
|
}
|
||||||
return counts[normalizeIndex(index, normalizingIndexOffset, countsArrayLength)];
|
|
||||||
}
|
@Override
|
||||||
|
long getCountAtNormalizedIndex(final int index) {
|
||||||
@Override
|
return counts[index];
|
||||||
long getCountAtNormalizedIndex(final int index) {
|
}
|
||||||
return counts[index];
|
|
||||||
}
|
@Override
|
||||||
|
void incrementCountAtIndex(final int index) {
|
||||||
@Override
|
counts[normalizeIndex(index, normalizingIndexOffset, countsArrayLength)]++;
|
||||||
void incrementCountAtIndex(final int index) {
|
}
|
||||||
counts[normalizeIndex(index, normalizingIndexOffset, countsArrayLength)]++;
|
|
||||||
}
|
@Override
|
||||||
|
void addToCountAtIndex(final int index, final long value) {
|
||||||
@Override
|
// 正常情况下normalizingIndexOffset = 0, index不用偏移
|
||||||
void addToCountAtIndex(final int index, final long value) {
|
counts[normalizeIndex(index, normalizingIndexOffset, countsArrayLength)] += value;
|
||||||
// 正常情况下normalizingIndexOffset = 0, index不用偏移
|
}
|
||||||
counts[normalizeIndex(index, normalizingIndexOffset, countsArrayLength)] += value;
|
|
||||||
}
|
@Override
|
||||||
|
void setCountAtIndex(int index, long value) {
|
||||||
@Override
|
counts[normalizeIndex(index, normalizingIndexOffset, countsArrayLength)] = value;
|
||||||
void setCountAtIndex(int index, long value) {
|
}
|
||||||
counts[normalizeIndex(index, normalizingIndexOffset, countsArrayLength)] = value;
|
|
||||||
}
|
@Override
|
||||||
|
void setCountAtNormalizedIndex(int index, long value) {
|
||||||
@Override
|
counts[index] = value;
|
||||||
void setCountAtNormalizedIndex(int index, long value) {
|
}
|
||||||
counts[index] = value;
|
|
||||||
}
|
@Override
|
||||||
|
int getNormalizingIndexOffset() {
|
||||||
@Override
|
return normalizingIndexOffset;
|
||||||
int getNormalizingIndexOffset() {
|
}
|
||||||
return normalizingIndexOffset;
|
|
||||||
}
|
@Override
|
||||||
|
void setNormalizingIndexOffset(int normalizingIndexOffset) {
|
||||||
@Override
|
this.normalizingIndexOffset = normalizingIndexOffset;
|
||||||
void setNormalizingIndexOffset(int normalizingIndexOffset) {
|
}
|
||||||
this.normalizingIndexOffset = normalizingIndexOffset;
|
|
||||||
}
|
@Override
|
||||||
|
void setIntegerToDoubleValueConversionRatio(double integerToDoubleValueConversionRatio) {
|
||||||
@Override
|
nonConcurrentSetIntegerToDoubleValueConversionRatio(integerToDoubleValueConversionRatio);
|
||||||
void setIntegerToDoubleValueConversionRatio(double integerToDoubleValueConversionRatio) {
|
}
|
||||||
nonConcurrentSetIntegerToDoubleValueConversionRatio(integerToDoubleValueConversionRatio);
|
|
||||||
}
|
@Override
|
||||||
|
void shiftNormalizingIndexByOffset(int offsetToAdd,
|
||||||
@Override
|
boolean lowestHalfBucketPopulated,
|
||||||
void shiftNormalizingIndexByOffset(int offsetToAdd,
|
double newIntegerToDoubleValueConversionRatio) {
|
||||||
boolean lowestHalfBucketPopulated,
|
nonConcurrentNormalizingIndexShift(offsetToAdd, lowestHalfBucketPopulated);
|
||||||
double newIntegerToDoubleValueConversionRatio) {
|
}
|
||||||
nonConcurrentNormalizingIndexShift(offsetToAdd, lowestHalfBucketPopulated);
|
|
||||||
}
|
@Override
|
||||||
|
void clearCounts() {
|
||||||
@Override
|
Arrays.fill(counts, 0);
|
||||||
void clearCounts() {
|
totalCount = 0;
|
||||||
Arrays.fill(counts, 0);
|
}
|
||||||
totalCount = 0;
|
|
||||||
}
|
@Override
|
||||||
|
public Histogramer makeCopy() {
|
||||||
@Override
|
return miniCopy();
|
||||||
public Histogramer makeCopy() {
|
}
|
||||||
return miniCopy();
|
|
||||||
}
|
@Override
|
||||||
|
public ArrayHistogram copy() {
|
||||||
@Override
|
ArrayHistogram copy = new ArrayHistogram(this);
|
||||||
public ArrayHistogram copy() {
|
copy.add(this);
|
||||||
ArrayHistogram copy = new ArrayHistogram(this);
|
return copy;
|
||||||
copy.add(this);
|
}
|
||||||
return copy;
|
|
||||||
}
|
public ArrayHistogram miniCopy() {
|
||||||
|
ArrayHistogram copy = new ArrayHistogram(lowestDiscernibleValue, maxValue < highestTrackableValue ? Math.max(maxValue, lowestDiscernibleValue * 2) : highestTrackableValue, numberOfSignificantValueDigits);
|
||||||
public ArrayHistogram miniCopy() {
|
copy.add(this);
|
||||||
ArrayHistogram copy = new ArrayHistogram(lowestDiscernibleValue, maxValue < highestTrackableValue ? Math.max(maxValue, lowestDiscernibleValue * 2) : highestTrackableValue, numberOfSignificantValueDigits);
|
return copy;
|
||||||
copy.add(this);
|
}
|
||||||
return copy;
|
|
||||||
}
|
@Override
|
||||||
|
public ArrayHistogram copyCorrectedForCoordinatedOmission(final long expectedIntervalBetweenValueSamples) {
|
||||||
@Override
|
ArrayHistogram copy = new ArrayHistogram(this);
|
||||||
public ArrayHistogram copyCorrectedForCoordinatedOmission(final long expectedIntervalBetweenValueSamples) {
|
copy.addWhileCorrectingForCoordinatedOmission(this, expectedIntervalBetweenValueSamples);
|
||||||
ArrayHistogram copy = new ArrayHistogram(this);
|
return copy;
|
||||||
copy.addWhileCorrectingForCoordinatedOmission(this, expectedIntervalBetweenValueSamples);
|
}
|
||||||
return copy;
|
|
||||||
}
|
@Override
|
||||||
|
public long getTotalCount() {
|
||||||
@Override
|
return totalCount;
|
||||||
public long getTotalCount() {
|
}
|
||||||
return totalCount;
|
|
||||||
}
|
@Override
|
||||||
|
void setTotalCount(final long totalCount) {
|
||||||
@Override
|
this.totalCount = totalCount;
|
||||||
void setTotalCount(final long totalCount) {
|
}
|
||||||
this.totalCount = totalCount;
|
|
||||||
}
|
@Override
|
||||||
|
void incrementTotalCount() {
|
||||||
@Override
|
totalCount++;
|
||||||
void incrementTotalCount() {
|
}
|
||||||
totalCount++;
|
|
||||||
}
|
@Override
|
||||||
|
void addToTotalCount(final long value) {
|
||||||
@Override
|
totalCount += value;
|
||||||
void addToTotalCount(final long value) {
|
}
|
||||||
totalCount += value;
|
|
||||||
}
|
@Override
|
||||||
|
int _getEstimatedFootprintInBytes() {
|
||||||
@Override
|
return (512 + (8 * counts.length));
|
||||||
int _getEstimatedFootprintInBytes() {
|
}
|
||||||
return (512 + (8 * counts.length));
|
|
||||||
}
|
@Override
|
||||||
|
void resize(long newHighestTrackableValue) {
|
||||||
@Override
|
int oldNormalizedZeroIndex = normalizeIndex(0, normalizingIndexOffset, countsArrayLength);
|
||||||
void resize(long newHighestTrackableValue) {
|
|
||||||
int oldNormalizedZeroIndex = normalizeIndex(0, normalizingIndexOffset, countsArrayLength);
|
establishSize(newHighestTrackableValue);
|
||||||
|
|
||||||
establishSize(newHighestTrackableValue);
|
int countsDelta = countsArrayLength - counts.length;
|
||||||
|
|
||||||
int countsDelta = countsArrayLength - counts.length;
|
counts = Arrays.copyOf(counts, countsArrayLength);
|
||||||
|
|
||||||
counts = Arrays.copyOf(counts, countsArrayLength);
|
if (oldNormalizedZeroIndex != 0) {
|
||||||
|
// We need to shift the stuff from the zero index and up to the end of the array:
|
||||||
if (oldNormalizedZeroIndex != 0) {
|
int newNormalizedZeroIndex = oldNormalizedZeroIndex + countsDelta;
|
||||||
// We need to shift the stuff from the zero index and up to the end of the array:
|
int lengthToCopy = (countsArrayLength - countsDelta) - oldNormalizedZeroIndex;
|
||||||
int newNormalizedZeroIndex = oldNormalizedZeroIndex + countsDelta;
|
System.arraycopy(counts, oldNormalizedZeroIndex, counts, newNormalizedZeroIndex, lengthToCopy);
|
||||||
int lengthToCopy = (countsArrayLength - countsDelta) - oldNormalizedZeroIndex;
|
Arrays.fill(counts, oldNormalizedZeroIndex, newNormalizedZeroIndex, 0);
|
||||||
System.arraycopy(counts, oldNormalizedZeroIndex, counts, newNormalizedZeroIndex, lengthToCopy);
|
}
|
||||||
Arrays.fill(counts, oldNormalizedZeroIndex, newNormalizedZeroIndex, 0);
|
}
|
||||||
}
|
|
||||||
}
|
/**
|
||||||
|
* Construct an auto-resizing histogram with a lowest discernible value of 1 and an auto-adjusting
|
||||||
/**
|
* highestTrackableValue. Can auto-resize up to track values up to (Long.MAX_VALUE / 2).
|
||||||
* Construct an auto-resizing histogram with a lowest discernible value of 1 and an auto-adjusting
|
*
|
||||||
* highestTrackableValue. Can auto-resize up to track values up to (Long.MAX_VALUE / 2).
|
* @param numberOfSignificantValueDigits Specifies the precision to use. This is the number of significant
|
||||||
*
|
* decimal digits to which the histogram will maintain value resolution
|
||||||
* @param numberOfSignificantValueDigits Specifies the precision to use. This is the number of significant
|
* and separation. Must be a non-negative integer between 0 and 5.
|
||||||
* decimal digits to which the histogram will maintain value resolution
|
*/
|
||||||
* and separation. Must be a non-negative integer between 0 and 5.
|
public ArrayHistogram(final int numberOfSignificantValueDigits) {
|
||||||
*/
|
this(1, 2, numberOfSignificantValueDigits);
|
||||||
public ArrayHistogram(final int numberOfSignificantValueDigits) {
|
setAutoResize(true);
|
||||||
this(1, 2, numberOfSignificantValueDigits);
|
}
|
||||||
setAutoResize(true);
|
|
||||||
}
|
/**
|
||||||
|
* Construct a Histogram given the Highest value to be tracked and a number of significant decimal digits. The
|
||||||
/**
|
* histogram will be constructed to implicitly track (distinguish from 0) values as low as 1.
|
||||||
* Construct a Histogram given the Highest value to be tracked and a number of significant decimal digits. The
|
*
|
||||||
* histogram will be constructed to implicitly track (distinguish from 0) values as low as 1.
|
* @param highestTrackableValue The highest value to be tracked by the histogram. Must be a positive
|
||||||
*
|
* integer that is {@literal >=} 2.
|
||||||
* @param highestTrackableValue The highest value to be tracked by the histogram. Must be a positive
|
* @param numberOfSignificantValueDigits Specifies the precision to use. This is the number of significant
|
||||||
* integer that is {@literal >=} 2.
|
* decimal digits to which the histogram will maintain value resolution
|
||||||
* @param numberOfSignificantValueDigits Specifies the precision to use. This is the number of significant
|
* and separation. Must be a non-negative integer between 0 and 5.
|
||||||
* decimal digits to which the histogram will maintain value resolution
|
*/
|
||||||
* and separation. Must be a non-negative integer between 0 and 5.
|
public ArrayHistogram(final long highestTrackableValue, final int numberOfSignificantValueDigits) {
|
||||||
*/
|
this(1, highestTrackableValue, numberOfSignificantValueDigits);
|
||||||
public ArrayHistogram(final long highestTrackableValue, final int numberOfSignificantValueDigits) {
|
}
|
||||||
this(1, highestTrackableValue, numberOfSignificantValueDigits);
|
|
||||||
}
|
/**
|
||||||
|
* Construct a Histogram given the Lowest and Highest values to be tracked and a number of significant
|
||||||
/**
|
* decimal digits. Providing a lowestDiscernibleValue is useful is situations where the units used
|
||||||
* Construct a Histogram given the Lowest and Highest values to be tracked and a number of significant
|
* for the histogram's values are much smaller that the minimal accuracy required. E.g. when tracking
|
||||||
* decimal digits. Providing a lowestDiscernibleValue is useful is situations where the units used
|
* time values stated in nanosecond units, where the minimal accuracy required is a microsecond, the
|
||||||
* for the histogram's values are much smaller that the minimal accuracy required. E.g. when tracking
|
* proper value for lowestDiscernibleValue would be 1000.
|
||||||
* time values stated in nanosecond units, where the minimal accuracy required is a microsecond, the
|
*
|
||||||
* proper value for lowestDiscernibleValue would be 1000.
|
* @param lowestDiscernibleValue The lowest value that can be discerned (distinguished from 0) by the
|
||||||
*
|
* histogram. Must be a positive integer that is {@literal >=} 1. May be
|
||||||
* @param lowestDiscernibleValue The lowest value that can be discerned (distinguished from 0) by the
|
* internally rounded down to nearest power of 2.
|
||||||
* histogram. Must be a positive integer that is {@literal >=} 1. May be
|
* @param highestTrackableValue The highest value to be tracked by the histogram. Must be a positive
|
||||||
* internally rounded down to nearest power of 2.
|
* integer that is {@literal >=} (2 * lowestDiscernibleValue).
|
||||||
* @param highestTrackableValue The highest value to be tracked by the histogram. Must be a positive
|
* @param numberOfSignificantValueDigits Specifies the precision to use. This is the number of significant
|
||||||
* integer that is {@literal >=} (2 * lowestDiscernibleValue).
|
* decimal digits to which the histogram will maintain value resolution
|
||||||
* @param numberOfSignificantValueDigits Specifies the precision to use. This is the number of significant
|
* and separation. Must be a non-negative integer between 0 and 5.
|
||||||
* decimal digits to which the histogram will maintain value resolution
|
*/
|
||||||
* and separation. Must be a non-negative integer between 0 and 5.
|
public ArrayHistogram(final long lowestDiscernibleValue, final long highestTrackableValue,
|
||||||
*/
|
final int numberOfSignificantValueDigits) {
|
||||||
public ArrayHistogram(final long lowestDiscernibleValue, final long highestTrackableValue,
|
this(lowestDiscernibleValue, highestTrackableValue, numberOfSignificantValueDigits, true);
|
||||||
final int numberOfSignificantValueDigits) {
|
}
|
||||||
this(lowestDiscernibleValue, highestTrackableValue, numberOfSignificantValueDigits, true);
|
|
||||||
}
|
/**
|
||||||
|
* Construct a histogram with the same range settings as a given source histogram,
|
||||||
/**
|
* duplicating the source's start/end timestamps (but NOT its contents)
|
||||||
* Construct a histogram with the same range settings as a given source histogram,
|
* @param source The source histogram to duplicate
|
||||||
* duplicating the source's start/end timestamps (but NOT its contents)
|
*/
|
||||||
* @param source The source histogram to duplicate
|
public ArrayHistogram(final AbstractHistogram source) {
|
||||||
*/
|
this(source, true);
|
||||||
public ArrayHistogram(final AbstractHistogram source) {
|
}
|
||||||
this(source, true);
|
|
||||||
}
|
ArrayHistogram(final AbstractHistogram source, boolean allocateCountsArray) {
|
||||||
|
super(source);
|
||||||
ArrayHistogram(final AbstractHistogram source, boolean allocateCountsArray) {
|
if (allocateCountsArray) {
|
||||||
super(source);
|
counts = new long[countsArrayLength];
|
||||||
if (allocateCountsArray) {
|
}
|
||||||
counts = new long[countsArrayLength];
|
wordSizeInBytes = 8;
|
||||||
}
|
}
|
||||||
wordSizeInBytes = 8;
|
|
||||||
}
|
ArrayHistogram(final long lowestDiscernibleValue, final long highestTrackableValue,
|
||||||
|
final int numberOfSignificantValueDigits, boolean allocateCountsArray) {
|
||||||
ArrayHistogram(final long lowestDiscernibleValue, final long highestTrackableValue,
|
super(lowestDiscernibleValue, highestTrackableValue, numberOfSignificantValueDigits);
|
||||||
final int numberOfSignificantValueDigits, boolean allocateCountsArray) {
|
if (allocateCountsArray) {
|
||||||
super(lowestDiscernibleValue, highestTrackableValue, numberOfSignificantValueDigits);
|
counts = new long[countsArrayLength];
|
||||||
if (allocateCountsArray) {
|
}
|
||||||
counts = new long[countsArrayLength];
|
// 写死 = 8
|
||||||
}
|
wordSizeInBytes = 8;
|
||||||
// 写死 = 8
|
}
|
||||||
wordSizeInBytes = 8;
|
|
||||||
}
|
/**
|
||||||
|
* Construct a new histogram by decoding it from a ByteBuffer.
|
||||||
/**
|
* @param buffer The buffer to decode from
|
||||||
* Construct a new histogram by decoding it from a ByteBuffer.
|
* @param minBarForHighestTrackableValue Force highestTrackableValue to be set at least this high
|
||||||
* @param buffer The buffer to decode from
|
* @return The newly constructed histogram
|
||||||
* @param minBarForHighestTrackableValue Force highestTrackableValue to be set at least this high
|
*/
|
||||||
* @return The newly constructed histogram
|
public static ArrayHistogram decodeFromByteBuffer(final ByteBuffer buffer,
|
||||||
*/
|
final long minBarForHighestTrackableValue) {
|
||||||
public static ArrayHistogram decodeFromByteBuffer(final ByteBuffer buffer,
|
return decodeFromByteBuffer(buffer, ArrayHistogram.class, minBarForHighestTrackableValue);
|
||||||
final long minBarForHighestTrackableValue) {
|
}
|
||||||
return decodeFromByteBuffer(buffer, ArrayHistogram.class, minBarForHighestTrackableValue);
|
|
||||||
}
|
/**
|
||||||
|
* Construct a new histogram by decoding it from a compressed form in a ByteBuffer.
|
||||||
/**
|
* @param buffer The buffer to decode from
|
||||||
* Construct a new histogram by decoding it from a compressed form in a ByteBuffer.
|
* @param minBarForHighestTrackableValue Force highestTrackableValue to be set at least this high
|
||||||
* @param buffer The buffer to decode from
|
* @return The newly constructed histogram
|
||||||
* @param minBarForHighestTrackableValue Force highestTrackableValue to be set at least this high
|
* @throws DataFormatException on error parsing/decompressing the buffer
|
||||||
* @return The newly constructed histogram
|
*/
|
||||||
* @throws DataFormatException on error parsing/decompressing the buffer
|
public static ArrayHistogram decodeFromCompressedByteBuffer(final ByteBuffer buffer,
|
||||||
*/
|
final long minBarForHighestTrackableValue)
|
||||||
public static ArrayHistogram decodeFromCompressedByteBuffer(final ByteBuffer buffer,
|
throws DataFormatException {
|
||||||
final long minBarForHighestTrackableValue)
|
return decodeFromCompressedByteBuffer(buffer, ArrayHistogram.class, minBarForHighestTrackableValue);
|
||||||
throws DataFormatException {
|
}
|
||||||
return decodeFromCompressedByteBuffer(buffer, ArrayHistogram.class, minBarForHighestTrackableValue);
|
|
||||||
}
|
private void readObject(final ObjectInputStream o)
|
||||||
|
throws IOException, ClassNotFoundException {
|
||||||
private void readObject(final ObjectInputStream o)
|
o.defaultReadObject();
|
||||||
throws IOException, ClassNotFoundException {
|
}
|
||||||
o.defaultReadObject();
|
|
||||||
}
|
/**
|
||||||
|
* Construct a new Histogram by decoding it from a String containing a base64 encoded
|
||||||
/**
|
* compressed histogram representation.
|
||||||
* Construct a new Histogram by decoding it from a String containing a base64 encoded
|
*
|
||||||
* compressed histogram representation.
|
* @param base64CompressedHistogramString A string containing a base64 encoding of a compressed histogram
|
||||||
*
|
* @return A Histogream decoded from the string
|
||||||
* @param base64CompressedHistogramString A string containing a base64 encoding of a compressed histogram
|
* @throws DataFormatException on error parsing/decompressing the input
|
||||||
* @return A Histogream decoded from the string
|
*/
|
||||||
* @throws DataFormatException on error parsing/decompressing the input
|
public static ArrayHistogram fromString(final String base64CompressedHistogramString)
|
||||||
*/
|
throws DataFormatException {
|
||||||
public static ArrayHistogram fromString(final String base64CompressedHistogramString)
|
// 这还有个base64字符串的解析
|
||||||
throws DataFormatException {
|
return decodeFromCompressedByteBuffer(
|
||||||
// 这还有个base64字符串的解析
|
ByteBuffer.wrap(Base64Helper.parseBase64Binary(base64CompressedHistogramString)),
|
||||||
return decodeFromCompressedByteBuffer(
|
0);
|
||||||
ByteBuffer.wrap(Base64Helper.parseBase64Binary(base64CompressedHistogramString)),
|
}
|
||||||
0);
|
|
||||||
}
|
@Override
|
||||||
|
public List<Percentile> percentileList(int percentileTicksPerHalfDistance) {
|
||||||
@Override
|
List<Percentile> percentiles = new ArrayList<>();
|
||||||
public List<Percentile> percentileList(int percentileTicksPerHalfDistance) {
|
for (HistogramIterationValue percentile : this.percentiles(percentileTicksPerHalfDistance)) {
|
||||||
List<Percentile> percentiles = new ArrayList<>();
|
if(percentile.getCountAddedInThisIterationStep() > 0){
|
||||||
for (HistogramIterationValue percentile : this.percentiles(percentileTicksPerHalfDistance)) {
|
percentiles.add(new Percentile(percentile.getValueIteratedTo(), percentile.getCountAddedInThisIterationStep(), percentile.getPercentile()));
|
||||||
if(percentile.getCountAddedInThisIterationStep() > 0){
|
}
|
||||||
percentiles.add(new Percentile(percentile.getValueIteratedTo(), percentile.getCountAddedInThisIterationStep(), percentile.getPercentile()));
|
}
|
||||||
}
|
return percentiles;
|
||||||
}
|
}
|
||||||
return percentiles;
|
|
||||||
}
|
@Override
|
||||||
|
public Map<String, Object> describe() {
|
||||||
@Override
|
long min = getMinValue();
|
||||||
public Histogramer resetHistogram() {
|
long max = getMaxValue(); // max = this.maxValue;
|
||||||
if(isAutoResize()){
|
long count = getTotalCount();
|
||||||
return new ArrayHistogram(this.numberOfSignificantValueDigits);
|
double mean = getMean();
|
||||||
}else{
|
long sum = (long) (mean * count);
|
||||||
this.reset();
|
mean = Math.round(mean * 100.0) / 100.0;
|
||||||
return this;
|
long p25 = getValueAtPercentile(25);
|
||||||
}
|
long p50 = getValueAtPercentile(50);
|
||||||
}
|
long p75 = getValueAtPercentile(75);
|
||||||
|
long p90 = getValueAtPercentile(90);
|
||||||
@Override
|
long p95 = getValueAtPercentile(95);
|
||||||
public Histogramer merge(Histogramer histogram) {
|
long p99 = getValueAtPercentile(99);
|
||||||
if(histogram instanceof AbstractHistogram){
|
Map<String, Object> rst = new LinkedHashMap<>();
|
||||||
this.add((AbstractHistogram)histogram);
|
rst.put("count", count);
|
||||||
return this;
|
rst.put("mean", mean);
|
||||||
}else if(histogram instanceof DirectMapHistogram){
|
rst.put("sum", sum);
|
||||||
try {
|
rst.put("min", min);
|
||||||
((DirectMapHistogram)histogram).mergeInto(this);
|
rst.put("p25", p25);
|
||||||
return this;
|
rst.put("p50", p50);
|
||||||
} catch (Exception e) {
|
rst.put("p75", p75);
|
||||||
throw new RuntimeException(e);
|
rst.put("p90", p90);
|
||||||
}
|
rst.put("p95", p95);
|
||||||
}else{
|
rst.put("p99", p99);
|
||||||
throw new UnsupportedOperationException("unsupported method");
|
rst.put("max", max);
|
||||||
}
|
return rst;
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public byte[] toBytes() {
|
public Histogramer resetHistogram() {
|
||||||
ByteBuffer byteBuffer = ByteBuffer.allocate(this.getNeededByteBufferCapacity());
|
if(isAutoResize()){
|
||||||
this.encodeIntoByteBuffer(byteBuffer);
|
return new ArrayHistogram(this.numberOfSignificantValueDigits);
|
||||||
return byteBuffer2Bytes(byteBuffer);
|
}else{
|
||||||
}
|
this.reset();
|
||||||
|
return this;
|
||||||
public static ArrayHistogram fromBytes(byte[] bytes) {
|
}
|
||||||
ByteBuffer byteBuffer = ByteBuffer.wrap(bytes);
|
}
|
||||||
return fromByteBuffer(byteBuffer);
|
|
||||||
}
|
@Override
|
||||||
|
public Histogramer merge(Histogramer histogram) {
|
||||||
public static ArrayHistogram fromByteBuffer(ByteBuffer byteBuffer) {
|
if(histogram instanceof AbstractHistogram){
|
||||||
int initPosition = byteBuffer.position();
|
this.add((AbstractHistogram)histogram);
|
||||||
int cookie = byteBuffer.getInt(initPosition);
|
return this;
|
||||||
if(DirectMapHistogram.getCookieBase(cookie) == DirectMapHistogram.V2CompressedEncodingCookieBase){
|
}else if(histogram instanceof DirectMapHistogram){
|
||||||
try {
|
try {
|
||||||
return ArrayHistogram.decodeFromCompressedByteBuffer(byteBuffer, 2);
|
((DirectMapHistogram)histogram).mergeInto(this);
|
||||||
} catch (DataFormatException e) {
|
return this;
|
||||||
throw new RuntimeException(e);
|
} catch (Exception e) {
|
||||||
}
|
throw new RuntimeException(e);
|
||||||
}else if(DirectMapHistogram.getCookieBase(cookie) == DirectMapHistogram.V2EncodingCookieBase){
|
}
|
||||||
return ArrayHistogram.decodeFromByteBuffer(byteBuffer, 2);
|
}else{
|
||||||
}
|
throw new UnsupportedOperationException("unsupported method");
|
||||||
throw new UnsupportedOperationException("unsupported method");
|
}
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
@Override
|
||||||
|
public byte[] toBytes() {
|
||||||
|
ByteBuffer byteBuffer = ByteBuffer.allocate(this.getNeededByteBufferCapacity());
|
||||||
|
this.encodeIntoByteBuffer(byteBuffer);
|
||||||
|
return byteBuffer2Bytes(byteBuffer);
|
||||||
|
}
|
||||||
|
|
||||||
|
public static ArrayHistogram fromBytes(byte[] bytes) {
|
||||||
|
ByteBuffer byteBuffer = ByteBuffer.wrap(bytes);
|
||||||
|
return fromByteBuffer(byteBuffer);
|
||||||
|
}
|
||||||
|
|
||||||
|
public static ArrayHistogram fromByteBuffer(ByteBuffer byteBuffer) {
|
||||||
|
int initPosition = byteBuffer.position();
|
||||||
|
int cookie = byteBuffer.getInt(initPosition);
|
||||||
|
if(DirectMapHistogram.getCookieBase(cookie) == DirectMapHistogram.V2CompressedEncodingCookieBase){
|
||||||
|
try {
|
||||||
|
return ArrayHistogram.decodeFromCompressedByteBuffer(byteBuffer, 2);
|
||||||
|
} catch (DataFormatException e) {
|
||||||
|
throw new RuntimeException(e);
|
||||||
|
}
|
||||||
|
}else if(DirectMapHistogram.getCookieBase(cookie) == DirectMapHistogram.V2EncodingCookieBase){
|
||||||
|
return ArrayHistogram.decodeFromByteBuffer(byteBuffer, 2);
|
||||||
|
}
|
||||||
|
throw new UnsupportedOperationException("unsupported method");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -1,203 +1,234 @@
|
|||||||
package org.HdrHistogram;
|
package org.HdrHistogram;
|
||||||
|
|
||||||
import java.nio.ByteBuffer;
|
import java.nio.ByteBuffer;
|
||||||
import java.util.ArrayList;
|
import java.util.ArrayList;
|
||||||
import java.util.List;
|
import java.util.LinkedHashMap;
|
||||||
|
import java.util.List;
|
||||||
public class DirectArrayHistogram extends AbstractHistogram implements Histogramer{
|
import java.util.Map;
|
||||||
long totalCount;
|
|
||||||
int normalizingIndexOffset;
|
public class DirectArrayHistogram extends AbstractHistogram implements Histogramer{
|
||||||
private ByteBuffer byteBuffer;
|
long totalCount;
|
||||||
private int initPosition;
|
int normalizingIndexOffset;
|
||||||
|
private ByteBuffer byteBuffer;
|
||||||
public DirectArrayHistogram(final long lowestDiscernibleValue, final long highestTrackableValue,
|
private int initPosition;
|
||||||
final int numberOfSignificantValueDigits, ByteBuffer byteBuffer) {
|
|
||||||
super(lowestDiscernibleValue, highestTrackableValue, numberOfSignificantValueDigits);
|
public DirectArrayHistogram(final long lowestDiscernibleValue, final long highestTrackableValue,
|
||||||
this.byteBuffer = byteBuffer;
|
final int numberOfSignificantValueDigits, ByteBuffer byteBuffer) {
|
||||||
this.initPosition = byteBuffer.position();
|
super(lowestDiscernibleValue, highestTrackableValue, numberOfSignificantValueDigits);
|
||||||
wordSizeInBytes = 8;
|
this.byteBuffer = byteBuffer;
|
||||||
}
|
this.initPosition = byteBuffer.position();
|
||||||
|
wordSizeInBytes = 8;
|
||||||
// druid内部使用
|
}
|
||||||
public void resetByteBuffer(ByteBuffer byteBuffer){
|
|
||||||
this.byteBuffer = byteBuffer;
|
// druid内部使用
|
||||||
this.initPosition = byteBuffer.position();
|
public void resetByteBuffer(ByteBuffer byteBuffer){
|
||||||
}
|
this.byteBuffer = byteBuffer;
|
||||||
|
this.initPosition = byteBuffer.position();
|
||||||
@Override
|
}
|
||||||
long getCountAtIndex(int index) {
|
|
||||||
int i = normalizeIndex(index, normalizingIndexOffset, countsArrayLength);
|
@Override
|
||||||
return byteBuffer.getLong(initPosition + i * 8);
|
long getCountAtIndex(int index) {
|
||||||
}
|
int i = normalizeIndex(index, normalizingIndexOffset, countsArrayLength);
|
||||||
|
return byteBuffer.getLong(initPosition + i * 8);
|
||||||
@Override
|
}
|
||||||
long getCountAtNormalizedIndex(int index) {
|
|
||||||
return byteBuffer.getLong(initPosition + index * 8);
|
@Override
|
||||||
}
|
long getCountAtNormalizedIndex(int index) {
|
||||||
|
return byteBuffer.getLong(initPosition + index * 8);
|
||||||
@Override
|
}
|
||||||
void incrementCountAtIndex(int index) {
|
|
||||||
int i = normalizeIndex(index, normalizingIndexOffset, countsArrayLength);
|
@Override
|
||||||
int pos = initPosition + i * 8;
|
void incrementCountAtIndex(int index) {
|
||||||
long val = byteBuffer.getLong(pos);
|
int i = normalizeIndex(index, normalizingIndexOffset, countsArrayLength);
|
||||||
byteBuffer.putLong(pos, val + 1);
|
int pos = initPosition + i * 8;
|
||||||
}
|
long val = byteBuffer.getLong(pos);
|
||||||
|
byteBuffer.putLong(pos, val + 1);
|
||||||
@Override
|
}
|
||||||
void addToCountAtIndex(int index, long value) {
|
|
||||||
int i = normalizeIndex(index, normalizingIndexOffset, countsArrayLength);
|
@Override
|
||||||
int pos = initPosition + i * 8;
|
void addToCountAtIndex(int index, long value) {
|
||||||
long val = byteBuffer.getLong(pos);
|
int i = normalizeIndex(index, normalizingIndexOffset, countsArrayLength);
|
||||||
byteBuffer.putLong(pos, val + value);
|
int pos = initPosition + i * 8;
|
||||||
}
|
long val = byteBuffer.getLong(pos);
|
||||||
|
byteBuffer.putLong(pos, val + value);
|
||||||
@Override
|
}
|
||||||
void setCountAtIndex(int index, long value) {
|
|
||||||
int i = normalizeIndex(index, normalizingIndexOffset, countsArrayLength);
|
@Override
|
||||||
int pos = initPosition + i * 8;
|
void setCountAtIndex(int index, long value) {
|
||||||
byteBuffer.putLong(pos, value);
|
int i = normalizeIndex(index, normalizingIndexOffset, countsArrayLength);
|
||||||
}
|
int pos = initPosition + i * 8;
|
||||||
|
byteBuffer.putLong(pos, value);
|
||||||
@Override
|
}
|
||||||
void setCountAtNormalizedIndex(int index, long value) {
|
|
||||||
int pos = initPosition + index * 8;
|
@Override
|
||||||
byteBuffer.putLong(pos, value);
|
void setCountAtNormalizedIndex(int index, long value) {
|
||||||
}
|
int pos = initPosition + index * 8;
|
||||||
|
byteBuffer.putLong(pos, value);
|
||||||
@Override
|
}
|
||||||
int getNormalizingIndexOffset() {
|
|
||||||
return normalizingIndexOffset;
|
@Override
|
||||||
}
|
int getNormalizingIndexOffset() {
|
||||||
|
return normalizingIndexOffset;
|
||||||
@Override
|
}
|
||||||
void setNormalizingIndexOffset(int normalizingIndexOffset) {
|
|
||||||
if(normalizingIndexOffset == 0){
|
@Override
|
||||||
this.normalizingIndexOffset = normalizingIndexOffset;
|
void setNormalizingIndexOffset(int normalizingIndexOffset) {
|
||||||
}else{
|
if(normalizingIndexOffset == 0){
|
||||||
throw new RuntimeException("cant not setNormalizingIndexOffset");
|
this.normalizingIndexOffset = normalizingIndexOffset;
|
||||||
}
|
}else{
|
||||||
}
|
throw new RuntimeException("cant not setNormalizingIndexOffset");
|
||||||
|
}
|
||||||
@Override
|
}
|
||||||
void setIntegerToDoubleValueConversionRatio(double integerToDoubleValueConversionRatio) {
|
|
||||||
nonConcurrentSetIntegerToDoubleValueConversionRatio(integerToDoubleValueConversionRatio);
|
@Override
|
||||||
}
|
void setIntegerToDoubleValueConversionRatio(double integerToDoubleValueConversionRatio) {
|
||||||
|
nonConcurrentSetIntegerToDoubleValueConversionRatio(integerToDoubleValueConversionRatio);
|
||||||
@Override
|
}
|
||||||
void shiftNormalizingIndexByOffset(int offsetToAdd, boolean lowestHalfBucketPopulated, double newIntegerToDoubleValueConversionRatio) {
|
|
||||||
nonConcurrentNormalizingIndexShift(offsetToAdd, lowestHalfBucketPopulated);
|
@Override
|
||||||
}
|
void shiftNormalizingIndexByOffset(int offsetToAdd, boolean lowestHalfBucketPopulated, double newIntegerToDoubleValueConversionRatio) {
|
||||||
|
nonConcurrentNormalizingIndexShift(offsetToAdd, lowestHalfBucketPopulated);
|
||||||
@Override
|
}
|
||||||
void clearCounts() {
|
|
||||||
for (int i = 0; i < countsArrayLength; i++) {
|
@Override
|
||||||
byteBuffer.putLong(initPosition + i * 8, 0L);
|
void clearCounts() {
|
||||||
}
|
for (int i = 0; i < countsArrayLength; i++) {
|
||||||
totalCount = 0;
|
byteBuffer.putLong(initPosition + i * 8, 0L);
|
||||||
}
|
}
|
||||||
|
totalCount = 0;
|
||||||
@Override
|
}
|
||||||
public Histogramer makeCopy() {
|
|
||||||
return miniCopy();
|
@Override
|
||||||
}
|
public Histogramer makeCopy() {
|
||||||
|
return miniCopy();
|
||||||
@Override
|
}
|
||||||
public ArrayHistogram copy() {
|
|
||||||
ArrayHistogram copy = new ArrayHistogram(this);
|
@Override
|
||||||
copy.add(this);
|
public ArrayHistogram copy() {
|
||||||
return copy;
|
ArrayHistogram copy = new ArrayHistogram(this);
|
||||||
}
|
copy.add(this);
|
||||||
|
return copy;
|
||||||
public ArrayHistogram miniCopy() {
|
}
|
||||||
ArrayHistogram copy = new ArrayHistogram(lowestDiscernibleValue, maxValue < highestTrackableValue ? Math.max(maxValue, lowestDiscernibleValue * 2) : highestTrackableValue, numberOfSignificantValueDigits);
|
|
||||||
copy.add(this);
|
public ArrayHistogram miniCopy() {
|
||||||
return copy;
|
ArrayHistogram copy = new ArrayHistogram(lowestDiscernibleValue, maxValue < highestTrackableValue ? Math.max(maxValue, lowestDiscernibleValue * 2) : highestTrackableValue, numberOfSignificantValueDigits);
|
||||||
}
|
copy.add(this);
|
||||||
|
return copy;
|
||||||
@Override
|
}
|
||||||
public AbstractHistogram copyCorrectedForCoordinatedOmission(long expectedIntervalBetweenValueSamples) {
|
|
||||||
Histogram copy = new Histogram(this);
|
@Override
|
||||||
copy.addWhileCorrectingForCoordinatedOmission(this, expectedIntervalBetweenValueSamples);
|
public AbstractHistogram copyCorrectedForCoordinatedOmission(long expectedIntervalBetweenValueSamples) {
|
||||||
return copy;
|
Histogram copy = new Histogram(this);
|
||||||
}
|
copy.addWhileCorrectingForCoordinatedOmission(this, expectedIntervalBetweenValueSamples);
|
||||||
|
return copy;
|
||||||
@Override
|
}
|
||||||
public long getTotalCount() {
|
|
||||||
return totalCount;
|
@Override
|
||||||
}
|
public long getTotalCount() {
|
||||||
|
return totalCount;
|
||||||
@Override
|
}
|
||||||
void setTotalCount(final long totalCount) {
|
|
||||||
this.totalCount = totalCount;
|
@Override
|
||||||
}
|
void setTotalCount(final long totalCount) {
|
||||||
|
this.totalCount = totalCount;
|
||||||
@Override
|
}
|
||||||
void incrementTotalCount() {
|
|
||||||
totalCount++;
|
@Override
|
||||||
}
|
void incrementTotalCount() {
|
||||||
|
totalCount++;
|
||||||
@Override
|
}
|
||||||
void addToTotalCount(long value) {
|
|
||||||
totalCount += value;
|
@Override
|
||||||
}
|
void addToTotalCount(long value) {
|
||||||
|
totalCount += value;
|
||||||
|
}
|
||||||
@Override
|
|
||||||
int _getEstimatedFootprintInBytes() {
|
|
||||||
return (512 + (8 * countsArrayLength));
|
@Override
|
||||||
}
|
int _getEstimatedFootprintInBytes() {
|
||||||
|
return (512 + (8 * countsArrayLength));
|
||||||
@Override
|
}
|
||||||
void resize(long newHighestTrackableValue) {
|
|
||||||
throw new RuntimeException("cant not resize");
|
@Override
|
||||||
}
|
void resize(long newHighestTrackableValue) {
|
||||||
|
throw new RuntimeException("cant not resize");
|
||||||
public static int getCountsArrayLength(long lowestDiscernibleValue, long highestTrackableValue, int numberOfSignificantValueDigits){
|
}
|
||||||
Histogram his = new Histogram(lowestDiscernibleValue, highestTrackableValue, numberOfSignificantValueDigits, false);
|
|
||||||
return his.countsArrayLength;
|
public static int getCountsArrayLength(long lowestDiscernibleValue, long highestTrackableValue, int numberOfSignificantValueDigits){
|
||||||
}
|
Histogram his = new Histogram(lowestDiscernibleValue, highestTrackableValue, numberOfSignificantValueDigits, false);
|
||||||
|
return his.countsArrayLength;
|
||||||
public static final int getUpdatableSerializationBytes(long lowestDiscernibleValue, long highestTrackableValue, int numberOfSignificantValueDigits){
|
}
|
||||||
return getCountsArrayLength(lowestDiscernibleValue, highestTrackableValue, numberOfSignificantValueDigits) * 8;
|
|
||||||
}
|
public static final int getUpdatableSerializationBytes(long lowestDiscernibleValue, long highestTrackableValue, int numberOfSignificantValueDigits){
|
||||||
|
return getCountsArrayLength(lowestDiscernibleValue, highestTrackableValue, numberOfSignificantValueDigits) * 8;
|
||||||
@Override
|
}
|
||||||
public List<Percentile> percentileList(int percentileTicksPerHalfDistance) {
|
|
||||||
List<Percentile> percentiles = new ArrayList<>();
|
@Override
|
||||||
for (HistogramIterationValue percentile : this.percentiles(percentileTicksPerHalfDistance)) {
|
public List<Percentile> percentileList(int percentileTicksPerHalfDistance) {
|
||||||
if(percentile.getCountAddedInThisIterationStep() > 0){
|
List<Percentile> percentiles = new ArrayList<>();
|
||||||
percentiles.add(new Percentile(percentile.getValueIteratedTo(), percentile.getCountAddedInThisIterationStep(), percentile.getPercentile()));
|
for (HistogramIterationValue percentile : this.percentiles(percentileTicksPerHalfDistance)) {
|
||||||
}
|
if(percentile.getCountAddedInThisIterationStep() > 0){
|
||||||
}
|
percentiles.add(new Percentile(percentile.getValueIteratedTo(), percentile.getCountAddedInThisIterationStep(), percentile.getPercentile()));
|
||||||
return percentiles;
|
}
|
||||||
}
|
}
|
||||||
|
return percentiles;
|
||||||
@Override
|
}
|
||||||
public Histogramer resetHistogram() {
|
|
||||||
throw new UnsupportedOperationException("unsupported method");
|
@Override
|
||||||
}
|
public Map<String, Object> describe() {
|
||||||
|
long min = getMinValue();
|
||||||
@Override
|
long max = getMaxValue(); // max = this.maxValue;
|
||||||
public Histogramer merge(Histogramer histogram) {
|
long count = getTotalCount();
|
||||||
if(histogram instanceof AbstractHistogram){
|
double mean = getMean();
|
||||||
this.add((AbstractHistogram)histogram);
|
long sum = (long) (mean * count);
|
||||||
return this;
|
mean = Math.round(mean * 100.0) / 100.0;
|
||||||
}else if(histogram instanceof DirectMapHistogram){
|
long p25 = getValueAtPercentile(25);
|
||||||
try {
|
long p50 = getValueAtPercentile(50);
|
||||||
((DirectMapHistogram)histogram).mergeInto(this);
|
long p75 = getValueAtPercentile(75);
|
||||||
return this;
|
long p90 = getValueAtPercentile(90);
|
||||||
} catch (Exception e) {
|
long p95 = getValueAtPercentile(95);
|
||||||
throw new RuntimeException(e);
|
long p99 = getValueAtPercentile(99);
|
||||||
}
|
Map<String, Object> rst = new LinkedHashMap<>();
|
||||||
}else{
|
rst.put("count", count);
|
||||||
throw new UnsupportedOperationException("unsupported method");
|
rst.put("mean", mean);
|
||||||
}
|
rst.put("sum", sum);
|
||||||
}
|
rst.put("min", min);
|
||||||
|
rst.put("p25", p25);
|
||||||
@Override
|
rst.put("p50", p50);
|
||||||
public byte[] toBytes() {
|
rst.put("p75", p75);
|
||||||
ByteBuffer byteBuffer = ByteBuffer.allocate(this.getNeededByteBufferCapacity());
|
rst.put("p90", p90);
|
||||||
this.encodeIntoByteBuffer(byteBuffer);
|
rst.put("p95", p95);
|
||||||
return byteBuffer2Bytes(byteBuffer);
|
rst.put("p99", p99);
|
||||||
}
|
rst.put("max", max);
|
||||||
}
|
return rst;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public Histogramer resetHistogram() {
|
||||||
|
throw new UnsupportedOperationException("unsupported method");
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public Histogramer merge(Histogramer histogram) {
|
||||||
|
if(histogram instanceof AbstractHistogram){
|
||||||
|
this.add((AbstractHistogram)histogram);
|
||||||
|
return this;
|
||||||
|
}else if(histogram instanceof DirectMapHistogram){
|
||||||
|
try {
|
||||||
|
((DirectMapHistogram)histogram).mergeInto(this);
|
||||||
|
return this;
|
||||||
|
} catch (Exception e) {
|
||||||
|
throw new RuntimeException(e);
|
||||||
|
}
|
||||||
|
}else{
|
||||||
|
throw new UnsupportedOperationException("unsupported method");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public byte[] toBytes() {
|
||||||
|
ByteBuffer byteBuffer = ByteBuffer.allocate(this.getNeededByteBufferCapacity());
|
||||||
|
this.encodeIntoByteBuffer(byteBuffer);
|
||||||
|
return byteBuffer2Bytes(byteBuffer);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -1,486 +1,492 @@
|
|||||||
package org.HdrHistogram;
|
package org.HdrHistogram;
|
||||||
|
|
||||||
import java.nio.ByteBuffer;
|
import java.nio.ByteBuffer;
|
||||||
import java.nio.ByteOrder;
|
import java.nio.ByteOrder;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
import java.util.zip.DataFormatException;
|
import java.util.Map;
|
||||||
import java.util.zip.Inflater;
|
import java.util.zip.DataFormatException;
|
||||||
|
import java.util.zip.Inflater;
|
||||||
import static java.nio.ByteOrder.BIG_ENDIAN;
|
|
||||||
|
import static java.nio.ByteOrder.BIG_ENDIAN;
|
||||||
/**
|
|
||||||
* 直接映射字节数组到Histogram，只读的Histogram，用于druid查询，减少gc减少计算，序列化后的是稀疏数组的形式
|
/**
|
||||||
*/
|
* 直接映射字节数组到Histogram，只读的Histogram，用于druid查询，减少gc减少计算，序列化后的是稀疏数组的形式
|
||||||
public class DirectMapHistogram implements Histogramer{
|
*/
|
||||||
static final int V2maxWordSizeInBytes = 9; // LEB128-64b9B + ZigZag require up to 9 bytes per word
|
public class DirectMapHistogram implements Histogramer{
|
||||||
static final int V2EncodingCookieBase = 0x1c849303;
|
static final int V2maxWordSizeInBytes = 9; // LEB128-64b9B + ZigZag require up to 9 bytes per word
|
||||||
static final int V2CompressedEncodingCookieBase = 0x1c849304;
|
static final int V2EncodingCookieBase = 0x1c849303;
|
||||||
|
static final int V2CompressedEncodingCookieBase = 0x1c849304;
|
||||||
final ByteBuffer byteBuffer;
|
|
||||||
final int initPosition;
|
final ByteBuffer byteBuffer;
|
||||||
long totalCount;
|
final int initPosition;
|
||||||
|
long totalCount;
|
||||||
private DirectMapHistogram(ByteBuffer byteBuffer) {
|
|
||||||
int initPosition = byteBuffer.position();
|
private DirectMapHistogram(ByteBuffer byteBuffer) {
|
||||||
this.byteBuffer = byteBuffer;
|
int initPosition = byteBuffer.position();
|
||||||
this.initPosition = initPosition;
|
this.byteBuffer = byteBuffer;
|
||||||
this.totalCount = -1;
|
this.initPosition = initPosition;
|
||||||
}
|
this.totalCount = -1;
|
||||||
|
}
|
||||||
public static boolean byteBufferCanToDirectMapHistogram(ByteBuffer byteBuffer) {
|
|
||||||
int initPosition = byteBuffer.position();
|
public static boolean byteBufferCanToDirectMapHistogram(ByteBuffer byteBuffer) {
|
||||||
int cookie = byteBuffer.getInt(initPosition);
|
int initPosition = byteBuffer.position();
|
||||||
return getCookieBase(cookie) == V2EncodingCookieBase || getCookieBase(cookie) == V2CompressedEncodingCookieBase;
|
int cookie = byteBuffer.getInt(initPosition);
|
||||||
}
|
return getCookieBase(cookie) == V2EncodingCookieBase || getCookieBase(cookie) == V2CompressedEncodingCookieBase;
|
||||||
|
}
|
||||||
public static DirectMapHistogram wrapBytes(byte[] bytes) {
|
|
||||||
ByteBuffer byteBuffer = ByteBuffer.wrap(bytes);
|
public static DirectMapHistogram wrapBytes(byte[] bytes) {
|
||||||
return wrapByteBuffer(byteBuffer);
|
ByteBuffer byteBuffer = ByteBuffer.wrap(bytes);
|
||||||
}
|
return wrapByteBuffer(byteBuffer);
|
||||||
|
}
|
||||||
public static DirectMapHistogram wrapByteBuffer(ByteBuffer byteBuffer) {
|
|
||||||
if(byteBufferCanToDirectMapHistogram(byteBuffer)){
|
public static DirectMapHistogram wrapByteBuffer(ByteBuffer byteBuffer) {
|
||||||
DirectMapHistogram hll = new DirectMapHistogram(byteBuffer);
|
if(byteBufferCanToDirectMapHistogram(byteBuffer)){
|
||||||
return hll;
|
DirectMapHistogram hll = new DirectMapHistogram(byteBuffer);
|
||||||
}
|
return hll;
|
||||||
throw new RuntimeException("can not wrapByteBuffer");
|
}
|
||||||
}
|
throw new RuntimeException("can not wrapByteBuffer");
|
||||||
|
}
|
||||||
public void mergeInto(AbstractHistogram histogram) throws Exception{
|
|
||||||
int cookie = byteBuffer.getInt(initPosition);
|
public void mergeInto(AbstractHistogram histogram) throws Exception{
|
||||||
if(getCookieBase(cookie) == V2CompressedEncodingCookieBase){
|
int cookie = byteBuffer.getInt(initPosition);
|
||||||
final int lengthOfCompressedContents = byteBuffer.getInt(initPosition + 4);
|
if(getCookieBase(cookie) == V2CompressedEncodingCookieBase){
|
||||||
final Inflater decompressor = new Inflater();
|
final int lengthOfCompressedContents = byteBuffer.getInt(initPosition + 4);
|
||||||
|
final Inflater decompressor = new Inflater();
|
||||||
if (byteBuffer.hasArray()) {
|
|
||||||
decompressor.setInput(byteBuffer.array(), initPosition + 8, lengthOfCompressedContents);
|
if (byteBuffer.hasArray()) {
|
||||||
} else {
|
decompressor.setInput(byteBuffer.array(), initPosition + 8, lengthOfCompressedContents);
|
||||||
byte[] compressedContents = new byte[lengthOfCompressedContents];
|
} else {
|
||||||
byteBuffer.position(initPosition + 8);
|
byte[] compressedContents = new byte[lengthOfCompressedContents];
|
||||||
try {
|
byteBuffer.position(initPosition + 8);
|
||||||
byteBuffer.get(compressedContents);
|
try {
|
||||||
decompressor.setInput(compressedContents);
|
byteBuffer.get(compressedContents);
|
||||||
}finally {
|
decompressor.setInput(compressedContents);
|
||||||
byteBuffer.position(initPosition);
|
}finally {
|
||||||
}
|
byteBuffer.position(initPosition);
|
||||||
}
|
}
|
||||||
final int headerSize = 40;
|
}
|
||||||
final ByteBuffer headerBuffer = ByteBuffer.allocate(headerSize).order(BIG_ENDIAN);
|
final int headerSize = 40;
|
||||||
decompressor.inflate(headerBuffer.array());
|
final ByteBuffer headerBuffer = ByteBuffer.allocate(headerSize).order(BIG_ENDIAN);
|
||||||
|
decompressor.inflate(headerBuffer.array());
|
||||||
cookie = headerBuffer.getInt();
|
|
||||||
final int payloadLengthInBytes;
|
cookie = headerBuffer.getInt();
|
||||||
final int normalizingIndexOffset;
|
final int payloadLengthInBytes;
|
||||||
final int numberOfSignificantValueDigits;
|
final int normalizingIndexOffset;
|
||||||
final long lowestTrackableUnitValue;
|
final int numberOfSignificantValueDigits;
|
||||||
long highestTrackableValue;
|
final long lowestTrackableUnitValue;
|
||||||
final double integerToDoubleValueConversionRatio;
|
long highestTrackableValue;
|
||||||
|
final double integerToDoubleValueConversionRatio;
|
||||||
assert getCookieBase(cookie) == V2EncodingCookieBase;
|
|
||||||
|
assert getCookieBase(cookie) == V2EncodingCookieBase;
|
||||||
payloadLengthInBytes = headerBuffer.getInt(4);
|
|
||||||
normalizingIndexOffset = headerBuffer.getInt(8);
|
payloadLengthInBytes = headerBuffer.getInt(4);
|
||||||
numberOfSignificantValueDigits = headerBuffer.getInt( 12);
|
normalizingIndexOffset = headerBuffer.getInt(8);
|
||||||
lowestTrackableUnitValue = headerBuffer.getLong(16);
|
numberOfSignificantValueDigits = headerBuffer.getInt( 12);
|
||||||
highestTrackableValue = headerBuffer.getLong(24);
|
lowestTrackableUnitValue = headerBuffer.getLong(16);
|
||||||
integerToDoubleValueConversionRatio = headerBuffer.getDouble(32);
|
highestTrackableValue = headerBuffer.getLong(24);
|
||||||
|
integerToDoubleValueConversionRatio = headerBuffer.getDouble(32);
|
||||||
highestTrackableValue = Math.max(highestTrackableValue, 2);
|
|
||||||
|
highestTrackableValue = Math.max(highestTrackableValue, 2);
|
||||||
final long largestValueWithSingleUnitResolution = 2 * (long) Math.pow(10, numberOfSignificantValueDigits);
|
|
||||||
final int unitMagnitude = (int) (Math.log(lowestTrackableUnitValue)/Math.log(2));
|
final long largestValueWithSingleUnitResolution = 2 * (long) Math.pow(10, numberOfSignificantValueDigits);
|
||||||
final long unitMagnitudeMask = (1 << unitMagnitude) - 1;
|
final int unitMagnitude = (int) (Math.log(lowestTrackableUnitValue)/Math.log(2));
|
||||||
int subBucketCountMagnitude = (int) Math.ceil(Math.log(largestValueWithSingleUnitResolution)/Math.log(2));
|
final long unitMagnitudeMask = (1 << unitMagnitude) - 1;
|
||||||
final int subBucketHalfCountMagnitude = subBucketCountMagnitude - 1;
|
int subBucketCountMagnitude = (int) Math.ceil(Math.log(largestValueWithSingleUnitResolution)/Math.log(2));
|
||||||
final int subBucketCount = 1 << subBucketCountMagnitude;
|
final int subBucketHalfCountMagnitude = subBucketCountMagnitude - 1;
|
||||||
final int subBucketHalfCount = subBucketCount / 2;
|
final int subBucketCount = 1 << subBucketCountMagnitude;
|
||||||
final long subBucketMask = ((long)subBucketCount - 1) << unitMagnitude;
|
final int subBucketHalfCount = subBucketCount / 2;
|
||||||
if (subBucketCountMagnitude + unitMagnitude > 62) {
|
final long subBucketMask = ((long)subBucketCount - 1) << unitMagnitude;
|
||||||
// subBucketCount entries can't be represented, with unitMagnitude applied, in a positive long.
|
if (subBucketCountMagnitude + unitMagnitude > 62) {
|
||||||
// Technically it still sort of works if their sum is 63: you can represent all but the last number
|
// subBucketCount entries can't be represented, with unitMagnitude applied, in a positive long.
|
||||||
// in the shifted subBucketCount. However, the utility of such a histogram vs ones whose magnitude here
|
// Technically it still sort of works if their sum is 63: you can represent all but the last number
|
||||||
// fits in 62 bits is debatable, and it makes it harder to work through the logic.
|
// in the shifted subBucketCount. However, the utility of such a histogram vs ones whose magnitude here
|
||||||
// Sums larger than 64 are totally broken as leadingZeroCountBase would go negative.
|
// fits in 62 bits is debatable, and it makes it harder to work through the logic.
|
||||||
throw new IllegalArgumentException("Cannot represent numberOfSignificantValueDigits worth of values " +
|
// Sums larger than 64 are totally broken as leadingZeroCountBase would go negative.
|
||||||
"beyond lowestDiscernibleValue");
|
throw new IllegalArgumentException("Cannot represent numberOfSignificantValueDigits worth of values " +
|
||||||
}
|
"beyond lowestDiscernibleValue");
|
||||||
|
}
|
||||||
final int expectedCapacity = payloadLengthInBytes;
|
|
||||||
|
final int expectedCapacity = payloadLengthInBytes;
|
||||||
ByteBuffer sourceBuffer = ByteBuffer.allocate(expectedCapacity).order(BIG_ENDIAN);
|
|
||||||
int decompressedByteCount = decompressor.inflate(sourceBuffer.array());
|
ByteBuffer sourceBuffer = ByteBuffer.allocate(expectedCapacity).order(BIG_ENDIAN);
|
||||||
decompressor.end(); // 必须手动调用，否则快速调用可能内存溢出(堆外内存)
|
int decompressedByteCount = decompressor.inflate(sourceBuffer.array());
|
||||||
if ((payloadLengthInBytes != Integer.MAX_VALUE) && (decompressedByteCount < payloadLengthInBytes)) {
|
decompressor.end(); // 必须手动调用，否则快速调用可能内存溢出(堆外内存)
|
||||||
throw new IllegalArgumentException("The buffer does not contain the indicated payload amount");
|
if ((payloadLengthInBytes != Integer.MAX_VALUE) && (decompressedByteCount < payloadLengthInBytes)) {
|
||||||
}
|
throw new IllegalArgumentException("The buffer does not contain the indicated payload amount");
|
||||||
assert decompressedByteCount == expectedCapacity;
|
}
|
||||||
|
assert decompressedByteCount == expectedCapacity;
|
||||||
int dstIndex = 0;
|
|
||||||
int endPosition = sourceBuffer.position() + expectedCapacity; //期望的结束读取的索引
|
int dstIndex = 0;
|
||||||
while (sourceBuffer.position() < endPosition) {
|
int endPosition = sourceBuffer.position() + expectedCapacity; //期望的结束读取的索引
|
||||||
long count;
|
while (sourceBuffer.position() < endPosition) {
|
||||||
int zerosCount = 0;
|
long count;
|
||||||
// V2 encoding format uses a long encoded in a ZigZag LEB128 format (up to V2maxWordSizeInBytes):
|
int zerosCount = 0;
|
||||||
count = ZigZagEncoding.getLong(sourceBuffer);
|
// V2 encoding format uses a long encoded in a ZigZag LEB128 format (up to V2maxWordSizeInBytes):
|
||||||
if (count < 0) {
|
count = ZigZagEncoding.getLong(sourceBuffer);
|
||||||
long zc = -count; // 0值的连续个数
|
if (count < 0) {
|
||||||
if (zc > Integer.MAX_VALUE) {
|
long zc = -count; // 0值的连续个数
|
||||||
throw new IllegalArgumentException(
|
if (zc > Integer.MAX_VALUE) {
|
||||||
"An encoded zero count of > Integer.MAX_VALUE was encountered in the source");
|
throw new IllegalArgumentException(
|
||||||
}
|
"An encoded zero count of > Integer.MAX_VALUE was encountered in the source");
|
||||||
zerosCount = (int) zc;
|
}
|
||||||
}
|
zerosCount = (int) zc;
|
||||||
if (zerosCount > 0) {
|
}
|
||||||
dstIndex += zerosCount; // No need to set zeros in array. Just skip them.
|
if (zerosCount > 0) {
|
||||||
} else {
|
dstIndex += zerosCount; // No need to set zeros in array. Just skip them.
|
||||||
// 单个非连续的0也会被输出
|
} else {
|
||||||
if(count > 0){
|
// 单个非连续的0也会被输出
|
||||||
long value = valueFromIndex(dstIndex, subBucketHalfCountMagnitude, subBucketHalfCount, unitMagnitude);
|
if(count > 0){
|
||||||
histogram.recordValueWithCount(value, count);
|
long value = valueFromIndex(dstIndex, subBucketHalfCountMagnitude, subBucketHalfCount, unitMagnitude);
|
||||||
}
|
histogram.recordValueWithCount(value, count);
|
||||||
dstIndex++;
|
}
|
||||||
}
|
dstIndex++;
|
||||||
}
|
}
|
||||||
|
}
|
||||||
}else if(getCookieBase(cookie) == V2EncodingCookieBase){
|
|
||||||
final int payloadLengthInBytes;
|
}else if(getCookieBase(cookie) == V2EncodingCookieBase){
|
||||||
final int normalizingIndexOffset;
|
final int payloadLengthInBytes;
|
||||||
final int numberOfSignificantValueDigits;
|
final int normalizingIndexOffset;
|
||||||
final long lowestTrackableUnitValue;
|
final int numberOfSignificantValueDigits;
|
||||||
long highestTrackableValue;
|
final long lowestTrackableUnitValue;
|
||||||
final double integerToDoubleValueConversionRatio;
|
long highestTrackableValue;
|
||||||
|
final double integerToDoubleValueConversionRatio;
|
||||||
payloadLengthInBytes = byteBuffer.getInt(initPosition + 4);
|
|
||||||
normalizingIndexOffset = byteBuffer.getInt(initPosition + 8);
|
payloadLengthInBytes = byteBuffer.getInt(initPosition + 4);
|
||||||
numberOfSignificantValueDigits = byteBuffer.getInt(initPosition + 12);
|
normalizingIndexOffset = byteBuffer.getInt(initPosition + 8);
|
||||||
lowestTrackableUnitValue = byteBuffer.getLong(initPosition + 16);
|
numberOfSignificantValueDigits = byteBuffer.getInt(initPosition + 12);
|
||||||
highestTrackableValue = byteBuffer.getLong(initPosition + 24);
|
lowestTrackableUnitValue = byteBuffer.getLong(initPosition + 16);
|
||||||
integerToDoubleValueConversionRatio = byteBuffer.getDouble(initPosition + 32);
|
highestTrackableValue = byteBuffer.getLong(initPosition + 24);
|
||||||
|
integerToDoubleValueConversionRatio = byteBuffer.getDouble(initPosition + 32);
|
||||||
highestTrackableValue = Math.max(highestTrackableValue, 2);
|
|
||||||
|
highestTrackableValue = Math.max(highestTrackableValue, 2);
|
||||||
final long largestValueWithSingleUnitResolution = 2 * (long) Math.pow(10, numberOfSignificantValueDigits);
|
|
||||||
final int unitMagnitude = (int) (Math.log(lowestTrackableUnitValue)/Math.log(2));
|
final long largestValueWithSingleUnitResolution = 2 * (long) Math.pow(10, numberOfSignificantValueDigits);
|
||||||
final long unitMagnitudeMask = (1 << unitMagnitude) - 1;
|
final int unitMagnitude = (int) (Math.log(lowestTrackableUnitValue)/Math.log(2));
|
||||||
int subBucketCountMagnitude = (int) Math.ceil(Math.log(largestValueWithSingleUnitResolution)/Math.log(2));
|
final long unitMagnitudeMask = (1 << unitMagnitude) - 1;
|
||||||
final int subBucketHalfCountMagnitude = subBucketCountMagnitude - 1;
|
int subBucketCountMagnitude = (int) Math.ceil(Math.log(largestValueWithSingleUnitResolution)/Math.log(2));
|
||||||
final int subBucketCount = 1 << subBucketCountMagnitude;
|
final int subBucketHalfCountMagnitude = subBucketCountMagnitude - 1;
|
||||||
final int subBucketHalfCount = subBucketCount / 2;
|
final int subBucketCount = 1 << subBucketCountMagnitude;
|
||||||
final long subBucketMask = ((long)subBucketCount - 1) << unitMagnitude;
|
final int subBucketHalfCount = subBucketCount / 2;
|
||||||
if (subBucketCountMagnitude + unitMagnitude > 62) {
|
final long subBucketMask = ((long)subBucketCount - 1) << unitMagnitude;
|
||||||
// subBucketCount entries can't be represented, with unitMagnitude applied, in a positive long.
|
if (subBucketCountMagnitude + unitMagnitude > 62) {
|
||||||
// Technically it still sort of works if their sum is 63: you can represent all but the last number
|
// subBucketCount entries can't be represented, with unitMagnitude applied, in a positive long.
|
||||||
// in the shifted subBucketCount. However, the utility of such a histogram vs ones whose magnitude here
|
// Technically it still sort of works if their sum is 63: you can represent all but the last number
|
||||||
// fits in 62 bits is debatable, and it makes it harder to work through the logic.
|
// in the shifted subBucketCount. However, the utility of such a histogram vs ones whose magnitude here
|
||||||
// Sums larger than 64 are totally broken as leadingZeroCountBase would go negative.
|
// fits in 62 bits is debatable, and it makes it harder to work through the logic.
|
||||||
throw new IllegalArgumentException("Cannot represent numberOfSignificantValueDigits worth of values " +
|
// Sums larger than 64 are totally broken as leadingZeroCountBase would go negative.
|
||||||
"beyond lowestDiscernibleValue");
|
throw new IllegalArgumentException("Cannot represent numberOfSignificantValueDigits worth of values " +
|
||||||
}
|
"beyond lowestDiscernibleValue");
|
||||||
|
}
|
||||||
final int expectedCapacity =payloadLengthInBytes;
|
|
||||||
assert expectedCapacity == payloadLengthInBytes;
|
final int expectedCapacity =payloadLengthInBytes;
|
||||||
if(expectedCapacity > byteBuffer.limit() - 40){
|
assert expectedCapacity == payloadLengthInBytes;
|
||||||
throw new IllegalArgumentException("The buffer does not contain the full Histogram payload");
|
if(expectedCapacity > byteBuffer.limit() - 40){
|
||||||
}
|
throw new IllegalArgumentException("The buffer does not contain the full Histogram payload");
|
||||||
final int position = initPosition + 40;
|
}
|
||||||
final int lengthInBytes = expectedCapacity;
|
final int position = initPosition + 40;
|
||||||
final int wordSizeInBytes = V2maxWordSizeInBytes;
|
final int lengthInBytes = expectedCapacity;
|
||||||
// fillCountsArrayFromSourceBuffer
|
final int wordSizeInBytes = V2maxWordSizeInBytes;
|
||||||
|
// fillCountsArrayFromSourceBuffer
|
||||||
ByteBuffer sourceBuffer = byteBuffer.duplicate();
|
|
||||||
sourceBuffer.position(position);
|
ByteBuffer sourceBuffer = byteBuffer.duplicate();
|
||||||
final long maxAllowableCountInHistigram = Long.MAX_VALUE;
|
sourceBuffer.position(position);
|
||||||
int dstIndex = 0;
|
final long maxAllowableCountInHistigram = Long.MAX_VALUE;
|
||||||
int endPosition = sourceBuffer.position() + lengthInBytes; //期望的结束读取的索引
|
int dstIndex = 0;
|
||||||
while (sourceBuffer.position() < endPosition) {
|
int endPosition = sourceBuffer.position() + lengthInBytes; //期望的结束读取的索引
|
||||||
long count;
|
while (sourceBuffer.position() < endPosition) {
|
||||||
int zerosCount = 0;
|
long count;
|
||||||
// V2 encoding format uses a long encoded in a ZigZag LEB128 format (up to V2maxWordSizeInBytes):
|
int zerosCount = 0;
|
||||||
count = ZigZagEncoding.getLong(sourceBuffer);
|
// V2 encoding format uses a long encoded in a ZigZag LEB128 format (up to V2maxWordSizeInBytes):
|
||||||
if (count < 0) {
|
count = ZigZagEncoding.getLong(sourceBuffer);
|
||||||
long zc = -count; // 0值的连续个数
|
if (count < 0) {
|
||||||
if (zc > Integer.MAX_VALUE) {
|
long zc = -count; // 0值的连续个数
|
||||||
throw new IllegalArgumentException(
|
if (zc > Integer.MAX_VALUE) {
|
||||||
"An encoded zero count of > Integer.MAX_VALUE was encountered in the source");
|
throw new IllegalArgumentException(
|
||||||
}
|
"An encoded zero count of > Integer.MAX_VALUE was encountered in the source");
|
||||||
zerosCount = (int) zc;
|
}
|
||||||
}
|
zerosCount = (int) zc;
|
||||||
if (zerosCount > 0) {
|
}
|
||||||
dstIndex += zerosCount; // No need to set zeros in array. Just skip them.
|
if (zerosCount > 0) {
|
||||||
} else {
|
dstIndex += zerosCount; // No need to set zeros in array. Just skip them.
|
||||||
// 单个非连续的0也会被输出
|
} else {
|
||||||
if(count > 0){
|
// 单个非连续的0也会被输出
|
||||||
long value = valueFromIndex(dstIndex, subBucketHalfCountMagnitude, subBucketHalfCount, unitMagnitude);
|
if(count > 0){
|
||||||
histogram.recordValueWithCount(value, count);
|
long value = valueFromIndex(dstIndex, subBucketHalfCountMagnitude, subBucketHalfCount, unitMagnitude);
|
||||||
}
|
histogram.recordValueWithCount(value, count);
|
||||||
dstIndex++;
|
}
|
||||||
}
|
dstIndex++;
|
||||||
}
|
}
|
||||||
}else{
|
}
|
||||||
throw new RuntimeException("can not wrapByteBuffer");
|
}else{
|
||||||
}
|
throw new RuntimeException("can not wrapByteBuffer");
|
||||||
}
|
}
|
||||||
|
}
|
||||||
final long valueFromIndex(final int index, int subBucketHalfCountMagnitude, int subBucketHalfCount, int unitMagnitude) {
|
|
||||||
int bucketIndex = (index >> subBucketHalfCountMagnitude) - 1;
|
final long valueFromIndex(final int index, int subBucketHalfCountMagnitude, int subBucketHalfCount, int unitMagnitude) {
|
||||||
int subBucketIndex = (index & (subBucketHalfCount - 1)) + subBucketHalfCount;
|
int bucketIndex = (index >> subBucketHalfCountMagnitude) - 1;
|
||||||
if (bucketIndex < 0) {
|
int subBucketIndex = (index & (subBucketHalfCount - 1)) + subBucketHalfCount;
|
||||||
subBucketIndex -= subBucketHalfCount;
|
if (bucketIndex < 0) {
|
||||||
bucketIndex = 0;
|
subBucketIndex -= subBucketHalfCount;
|
||||||
}
|
bucketIndex = 0;
|
||||||
return valueFromIndex(bucketIndex, subBucketIndex, unitMagnitude);
|
}
|
||||||
}
|
return valueFromIndex(bucketIndex, subBucketIndex, unitMagnitude);
|
||||||
|
}
|
||||||
private long valueFromIndex(final int bucketIndex, final int subBucketIndex, int unitMagnitude) {
|
|
||||||
return ((long) subBucketIndex) << (bucketIndex + unitMagnitude);
|
private long valueFromIndex(final int bucketIndex, final int subBucketIndex, int unitMagnitude) {
|
||||||
}
|
return ((long) subBucketIndex) << (bucketIndex + unitMagnitude);
|
||||||
|
}
|
||||||
static int getCookieBase(final int cookie) {
|
|
||||||
return (cookie & ~0xf0);
|
static int getCookieBase(final int cookie) {
|
||||||
}
|
return (cookie & ~0xf0);
|
||||||
|
}
|
||||||
@Override
|
|
||||||
public long getTotalCount() {
|
@Override
|
||||||
if(totalCount >= 0){
|
public long getTotalCount() {
|
||||||
return totalCount;
|
if(totalCount >= 0){
|
||||||
}
|
return totalCount;
|
||||||
try {
|
}
|
||||||
totalCount = 0;
|
try {
|
||||||
int cookie = byteBuffer.getInt(initPosition);
|
totalCount = 0;
|
||||||
if(getCookieBase(cookie) == V2CompressedEncodingCookieBase){
|
int cookie = byteBuffer.getInt(initPosition);
|
||||||
final int lengthOfCompressedContents = byteBuffer.getInt(initPosition + 4);
|
if(getCookieBase(cookie) == V2CompressedEncodingCookieBase){
|
||||||
final Inflater decompressor = new Inflater();
|
final int lengthOfCompressedContents = byteBuffer.getInt(initPosition + 4);
|
||||||
|
final Inflater decompressor = new Inflater();
|
||||||
if (byteBuffer.hasArray()) {
|
|
||||||
decompressor.setInput(byteBuffer.array(), initPosition + 8, lengthOfCompressedContents);
|
if (byteBuffer.hasArray()) {
|
||||||
} else {
|
decompressor.setInput(byteBuffer.array(), initPosition + 8, lengthOfCompressedContents);
|
||||||
byte[] compressedContents = new byte[lengthOfCompressedContents];
|
} else {
|
||||||
byteBuffer.position(initPosition + 8);
|
byte[] compressedContents = new byte[lengthOfCompressedContents];
|
||||||
try {
|
byteBuffer.position(initPosition + 8);
|
||||||
byteBuffer.get(compressedContents);
|
try {
|
||||||
decompressor.setInput(compressedContents);
|
byteBuffer.get(compressedContents);
|
||||||
}finally {
|
decompressor.setInput(compressedContents);
|
||||||
byteBuffer.position(initPosition);
|
}finally {
|
||||||
}
|
byteBuffer.position(initPosition);
|
||||||
}
|
}
|
||||||
final int headerSize = 40;
|
}
|
||||||
final ByteBuffer headerBuffer = ByteBuffer.allocate(headerSize).order(BIG_ENDIAN);
|
final int headerSize = 40;
|
||||||
decompressor.inflate(headerBuffer.array());
|
final ByteBuffer headerBuffer = ByteBuffer.allocate(headerSize).order(BIG_ENDIAN);
|
||||||
|
decompressor.inflate(headerBuffer.array());
|
||||||
cookie = headerBuffer.getInt();
|
|
||||||
final int payloadLengthInBytes;
|
cookie = headerBuffer.getInt();
|
||||||
final int normalizingIndexOffset;
|
final int payloadLengthInBytes;
|
||||||
final int numberOfSignificantValueDigits;
|
final int normalizingIndexOffset;
|
||||||
final long lowestTrackableUnitValue;
|
final int numberOfSignificantValueDigits;
|
||||||
long highestTrackableValue;
|
final long lowestTrackableUnitValue;
|
||||||
final double integerToDoubleValueConversionRatio;
|
long highestTrackableValue;
|
||||||
|
final double integerToDoubleValueConversionRatio;
|
||||||
assert getCookieBase(cookie) == V2EncodingCookieBase;
|
|
||||||
|
assert getCookieBase(cookie) == V2EncodingCookieBase;
|
||||||
payloadLengthInBytes = headerBuffer.getInt(4);
|
|
||||||
normalizingIndexOffset = headerBuffer.getInt(8);
|
payloadLengthInBytes = headerBuffer.getInt(4);
|
||||||
numberOfSignificantValueDigits = headerBuffer.getInt( 12);
|
normalizingIndexOffset = headerBuffer.getInt(8);
|
||||||
lowestTrackableUnitValue = headerBuffer.getLong(16);
|
numberOfSignificantValueDigits = headerBuffer.getInt( 12);
|
||||||
highestTrackableValue = headerBuffer.getLong(24);
|
lowestTrackableUnitValue = headerBuffer.getLong(16);
|
||||||
integerToDoubleValueConversionRatio = headerBuffer.getDouble(32);
|
highestTrackableValue = headerBuffer.getLong(24);
|
||||||
|
integerToDoubleValueConversionRatio = headerBuffer.getDouble(32);
|
||||||
highestTrackableValue = Math.max(highestTrackableValue, 2);
|
|
||||||
|
highestTrackableValue = Math.max(highestTrackableValue, 2);
|
||||||
final long largestValueWithSingleUnitResolution = 2 * (long) Math.pow(10, numberOfSignificantValueDigits);
|
|
||||||
final int unitMagnitude = (int) (Math.log(lowestTrackableUnitValue)/Math.log(2));
|
final long largestValueWithSingleUnitResolution = 2 * (long) Math.pow(10, numberOfSignificantValueDigits);
|
||||||
final long unitMagnitudeMask = (1 << unitMagnitude) - 1;
|
final int unitMagnitude = (int) (Math.log(lowestTrackableUnitValue)/Math.log(2));
|
||||||
int subBucketCountMagnitude = (int) Math.ceil(Math.log(largestValueWithSingleUnitResolution)/Math.log(2));
|
final long unitMagnitudeMask = (1 << unitMagnitude) - 1;
|
||||||
final int subBucketHalfCountMagnitude = subBucketCountMagnitude - 1;
|
int subBucketCountMagnitude = (int) Math.ceil(Math.log(largestValueWithSingleUnitResolution)/Math.log(2));
|
||||||
final int subBucketCount = 1 << subBucketCountMagnitude;
|
final int subBucketHalfCountMagnitude = subBucketCountMagnitude - 1;
|
||||||
final int subBucketHalfCount = subBucketCount / 2;
|
final int subBucketCount = 1 << subBucketCountMagnitude;
|
||||||
final long subBucketMask = ((long)subBucketCount - 1) << unitMagnitude;
|
final int subBucketHalfCount = subBucketCount / 2;
|
||||||
if (subBucketCountMagnitude + unitMagnitude > 62) {
|
final long subBucketMask = ((long)subBucketCount - 1) << unitMagnitude;
|
||||||
// subBucketCount entries can't be represented, with unitMagnitude applied, in a positive long.
|
if (subBucketCountMagnitude + unitMagnitude > 62) {
|
||||||
// Technically it still sort of works if their sum is 63: you can represent all but the last number
|
// subBucketCount entries can't be represented, with unitMagnitude applied, in a positive long.
|
||||||
// in the shifted subBucketCount. However, the utility of such a histogram vs ones whose magnitude here
|
// Technically it still sort of works if their sum is 63: you can represent all but the last number
|
||||||
// fits in 62 bits is debatable, and it makes it harder to work through the logic.
|
// in the shifted subBucketCount. However, the utility of such a histogram vs ones whose magnitude here
|
||||||
// Sums larger than 64 are totally broken as leadingZeroCountBase would go negative.
|
// fits in 62 bits is debatable, and it makes it harder to work through the logic.
|
||||||
throw new IllegalArgumentException("Cannot represent numberOfSignificantValueDigits worth of values " +
|
// Sums larger than 64 are totally broken as leadingZeroCountBase would go negative.
|
||||||
"beyond lowestDiscernibleValue");
|
throw new IllegalArgumentException("Cannot represent numberOfSignificantValueDigits worth of values " +
|
||||||
}
|
"beyond lowestDiscernibleValue");
|
||||||
|
}
|
||||||
final int expectedCapacity = payloadLengthInBytes;
|
|
||||||
|
final int expectedCapacity = payloadLengthInBytes;
|
||||||
ByteBuffer sourceBuffer = ByteBuffer.allocate(expectedCapacity).order(BIG_ENDIAN);
|
|
||||||
int decompressedByteCount = decompressor.inflate(sourceBuffer.array());
|
ByteBuffer sourceBuffer = ByteBuffer.allocate(expectedCapacity).order(BIG_ENDIAN);
|
||||||
decompressor.end(); // 必须手动调用，否则快速调用可能内存溢出(堆外内存)
|
int decompressedByteCount = decompressor.inflate(sourceBuffer.array());
|
||||||
if ((payloadLengthInBytes != Integer.MAX_VALUE) && (decompressedByteCount < payloadLengthInBytes)) {
|
decompressor.end(); // 必须手动调用，否则快速调用可能内存溢出(堆外内存)
|
||||||
throw new IllegalArgumentException("The buffer does not contain the indicated payload amount");
|
if ((payloadLengthInBytes != Integer.MAX_VALUE) && (decompressedByteCount < payloadLengthInBytes)) {
|
||||||
}
|
throw new IllegalArgumentException("The buffer does not contain the indicated payload amount");
|
||||||
assert decompressedByteCount == expectedCapacity;
|
}
|
||||||
|
assert decompressedByteCount == expectedCapacity;
|
||||||
int dstIndex = 0;
|
|
||||||
int endPosition = sourceBuffer.position() + expectedCapacity; //期望的结束读取的索引
|
int dstIndex = 0;
|
||||||
while (sourceBuffer.position() < endPosition) {
|
int endPosition = sourceBuffer.position() + expectedCapacity; //期望的结束读取的索引
|
||||||
long count;
|
while (sourceBuffer.position() < endPosition) {
|
||||||
int zerosCount = 0;
|
long count;
|
||||||
// V2 encoding format uses a long encoded in a ZigZag LEB128 format (up to V2maxWordSizeInBytes):
|
int zerosCount = 0;
|
||||||
count = ZigZagEncoding.getLong(sourceBuffer);
|
// V2 encoding format uses a long encoded in a ZigZag LEB128 format (up to V2maxWordSizeInBytes):
|
||||||
if (count < 0) {
|
count = ZigZagEncoding.getLong(sourceBuffer);
|
||||||
long zc = -count; // 0值的连续个数
|
if (count < 0) {
|
||||||
if (zc > Integer.MAX_VALUE) {
|
long zc = -count; // 0值的连续个数
|
||||||
throw new IllegalArgumentException(
|
if (zc > Integer.MAX_VALUE) {
|
||||||
"An encoded zero count of > Integer.MAX_VALUE was encountered in the source");
|
throw new IllegalArgumentException(
|
||||||
}
|
"An encoded zero count of > Integer.MAX_VALUE was encountered in the source");
|
||||||
zerosCount = (int) zc;
|
}
|
||||||
}
|
zerosCount = (int) zc;
|
||||||
if (zerosCount > 0) {
|
}
|
||||||
dstIndex += zerosCount; // No need to set zeros in array. Just skip them.
|
if (zerosCount > 0) {
|
||||||
} else {
|
dstIndex += zerosCount; // No need to set zeros in array. Just skip them.
|
||||||
// 单个非连续的0也会被输出
|
} else {
|
||||||
if(count > 0){
|
// 单个非连续的0也会被输出
|
||||||
//long value = valueFromIndex(dstIndex, subBucketHalfCountMagnitude, subBucketHalfCount, unitMagnitude);
|
if(count > 0){
|
||||||
//histogram.recordValueWithCount(value, count);
|
//long value = valueFromIndex(dstIndex, subBucketHalfCountMagnitude, subBucketHalfCount, unitMagnitude);
|
||||||
totalCount += count;
|
//histogram.recordValueWithCount(value, count);
|
||||||
}
|
totalCount += count;
|
||||||
dstIndex++;
|
}
|
||||||
}
|
dstIndex++;
|
||||||
}
|
}
|
||||||
return totalCount;
|
}
|
||||||
}else if(getCookieBase(cookie) == V2EncodingCookieBase){
|
return totalCount;
|
||||||
final int payloadLengthInBytes;
|
}else if(getCookieBase(cookie) == V2EncodingCookieBase){
|
||||||
final int normalizingIndexOffset;
|
final int payloadLengthInBytes;
|
||||||
final int numberOfSignificantValueDigits;
|
final int normalizingIndexOffset;
|
||||||
final long lowestTrackableUnitValue;
|
final int numberOfSignificantValueDigits;
|
||||||
long highestTrackableValue;
|
final long lowestTrackableUnitValue;
|
||||||
final double integerToDoubleValueConversionRatio;
|
long highestTrackableValue;
|
||||||
|
final double integerToDoubleValueConversionRatio;
|
||||||
payloadLengthInBytes = byteBuffer.getInt(initPosition + 4);
|
|
||||||
normalizingIndexOffset = byteBuffer.getInt(initPosition + 8);
|
payloadLengthInBytes = byteBuffer.getInt(initPosition + 4);
|
||||||
numberOfSignificantValueDigits = byteBuffer.getInt(initPosition + 12);
|
normalizingIndexOffset = byteBuffer.getInt(initPosition + 8);
|
||||||
lowestTrackableUnitValue = byteBuffer.getLong(initPosition + 16);
|
numberOfSignificantValueDigits = byteBuffer.getInt(initPosition + 12);
|
||||||
highestTrackableValue = byteBuffer.getLong(initPosition + 24);
|
lowestTrackableUnitValue = byteBuffer.getLong(initPosition + 16);
|
||||||
integerToDoubleValueConversionRatio = byteBuffer.getDouble(initPosition + 32);
|
highestTrackableValue = byteBuffer.getLong(initPosition + 24);
|
||||||
|
integerToDoubleValueConversionRatio = byteBuffer.getDouble(initPosition + 32);
|
||||||
highestTrackableValue = Math.max(highestTrackableValue, 2);
|
|
||||||
|
highestTrackableValue = Math.max(highestTrackableValue, 2);
|
||||||
final long largestValueWithSingleUnitResolution = 2 * (long) Math.pow(10, numberOfSignificantValueDigits);
|
|
||||||
final int unitMagnitude = (int) (Math.log(lowestTrackableUnitValue)/Math.log(2));
|
final long largestValueWithSingleUnitResolution = 2 * (long) Math.pow(10, numberOfSignificantValueDigits);
|
||||||
final long unitMagnitudeMask = (1 << unitMagnitude) - 1;
|
final int unitMagnitude = (int) (Math.log(lowestTrackableUnitValue)/Math.log(2));
|
||||||
int subBucketCountMagnitude = (int) Math.ceil(Math.log(largestValueWithSingleUnitResolution)/Math.log(2));
|
final long unitMagnitudeMask = (1 << unitMagnitude) - 1;
|
||||||
final int subBucketHalfCountMagnitude = subBucketCountMagnitude - 1;
|
int subBucketCountMagnitude = (int) Math.ceil(Math.log(largestValueWithSingleUnitResolution)/Math.log(2));
|
||||||
final int subBucketCount = 1 << subBucketCountMagnitude;
|
final int subBucketHalfCountMagnitude = subBucketCountMagnitude - 1;
|
||||||
final int subBucketHalfCount = subBucketCount / 2;
|
final int subBucketCount = 1 << subBucketCountMagnitude;
|
||||||
final long subBucketMask = ((long)subBucketCount - 1) << unitMagnitude;
|
final int subBucketHalfCount = subBucketCount / 2;
|
||||||
if (subBucketCountMagnitude + unitMagnitude > 62) {
|
final long subBucketMask = ((long)subBucketCount - 1) << unitMagnitude;
|
||||||
// subBucketCount entries can't be represented, with unitMagnitude applied, in a positive long.
|
if (subBucketCountMagnitude + unitMagnitude > 62) {
|
||||||
// Technically it still sort of works if their sum is 63: you can represent all but the last number
|
// subBucketCount entries can't be represented, with unitMagnitude applied, in a positive long.
|
||||||
// in the shifted subBucketCount. However, the utility of such a histogram vs ones whose magnitude here
|
// Technically it still sort of works if their sum is 63: you can represent all but the last number
|
||||||
// fits in 62 bits is debatable, and it makes it harder to work through the logic.
|
// in the shifted subBucketCount. However, the utility of such a histogram vs ones whose magnitude here
|
||||||
// Sums larger than 64 are totally broken as leadingZeroCountBase would go negative.
|
// fits in 62 bits is debatable, and it makes it harder to work through the logic.
|
||||||
throw new IllegalArgumentException("Cannot represent numberOfSignificantValueDigits worth of values " +
|
// Sums larger than 64 are totally broken as leadingZeroCountBase would go negative.
|
||||||
"beyond lowestDiscernibleValue");
|
throw new IllegalArgumentException("Cannot represent numberOfSignificantValueDigits worth of values " +
|
||||||
}
|
"beyond lowestDiscernibleValue");
|
||||||
|
}
|
||||||
final int expectedCapacity =payloadLengthInBytes;
|
|
||||||
assert expectedCapacity == payloadLengthInBytes;
|
final int expectedCapacity =payloadLengthInBytes;
|
||||||
if(expectedCapacity > byteBuffer.limit() - 40){
|
assert expectedCapacity == payloadLengthInBytes;
|
||||||
throw new IllegalArgumentException("The buffer does not contain the full Histogram payload");
|
if(expectedCapacity > byteBuffer.limit() - 40){
|
||||||
}
|
throw new IllegalArgumentException("The buffer does not contain the full Histogram payload");
|
||||||
final int position = initPosition + 40;
|
}
|
||||||
final int lengthInBytes = expectedCapacity;
|
final int position = initPosition + 40;
|
||||||
final int wordSizeInBytes = V2maxWordSizeInBytes;
|
final int lengthInBytes = expectedCapacity;
|
||||||
// fillCountsArrayFromSourceBuffer
|
final int wordSizeInBytes = V2maxWordSizeInBytes;
|
||||||
|
// fillCountsArrayFromSourceBuffer
|
||||||
ByteBuffer sourceBuffer = byteBuffer.duplicate();
|
|
||||||
sourceBuffer.position(position);
|
ByteBuffer sourceBuffer = byteBuffer.duplicate();
|
||||||
final long maxAllowableCountInHistigram = Long.MAX_VALUE;
|
sourceBuffer.position(position);
|
||||||
int dstIndex = 0;
|
final long maxAllowableCountInHistigram = Long.MAX_VALUE;
|
||||||
int endPosition = sourceBuffer.position() + lengthInBytes; //期望的结束读取的索引
|
int dstIndex = 0;
|
||||||
while (sourceBuffer.position() < endPosition) {
|
int endPosition = sourceBuffer.position() + lengthInBytes; //期望的结束读取的索引
|
||||||
long count;
|
while (sourceBuffer.position() < endPosition) {
|
||||||
int zerosCount = 0;
|
long count;
|
||||||
// V2 encoding format uses a long encoded in a ZigZag LEB128 format (up to V2maxWordSizeInBytes):
|
int zerosCount = 0;
|
||||||
count = ZigZagEncoding.getLong(sourceBuffer);
|
// V2 encoding format uses a long encoded in a ZigZag LEB128 format (up to V2maxWordSizeInBytes):
|
||||||
if (count < 0) {
|
count = ZigZagEncoding.getLong(sourceBuffer);
|
||||||
long zc = -count; // 0值的连续个数
|
if (count < 0) {
|
||||||
if (zc > Integer.MAX_VALUE) {
|
long zc = -count; // 0值的连续个数
|
||||||
throw new IllegalArgumentException(
|
if (zc > Integer.MAX_VALUE) {
|
||||||
"An encoded zero count of > Integer.MAX_VALUE was encountered in the source");
|
throw new IllegalArgumentException(
|
||||||
}
|
"An encoded zero count of > Integer.MAX_VALUE was encountered in the source");
|
||||||
zerosCount = (int) zc;
|
}
|
||||||
}
|
zerosCount = (int) zc;
|
||||||
if (zerosCount > 0) {
|
}
|
||||||
dstIndex += zerosCount; // No need to set zeros in array. Just skip them.
|
if (zerosCount > 0) {
|
||||||
} else {
|
dstIndex += zerosCount; // No need to set zeros in array. Just skip them.
|
||||||
// 单个非连续的0也会被输出
|
} else {
|
||||||
if(count > 0){
|
// 单个非连续的0也会被输出
|
||||||
//long value = valueFromIndex(dstIndex, subBucketHalfCountMagnitude, subBucketHalfCount, unitMagnitude);
|
if(count > 0){
|
||||||
//histogram.recordValueWithCount(value, count);
|
//long value = valueFromIndex(dstIndex, subBucketHalfCountMagnitude, subBucketHalfCount, unitMagnitude);
|
||||||
totalCount += count;
|
//histogram.recordValueWithCount(value, count);
|
||||||
}
|
totalCount += count;
|
||||||
dstIndex++;
|
}
|
||||||
}
|
dstIndex++;
|
||||||
}
|
}
|
||||||
return totalCount;
|
}
|
||||||
}else{
|
return totalCount;
|
||||||
throw new UnsupportedOperationException("unsupported method");
|
}else{
|
||||||
}
|
throw new UnsupportedOperationException("unsupported method");
|
||||||
} catch (DataFormatException e) {
|
}
|
||||||
throw new RuntimeException(e);
|
} catch (DataFormatException e) {
|
||||||
}
|
throw new RuntimeException(e);
|
||||||
}
|
}
|
||||||
|
}
|
||||||
@Override
|
|
||||||
public void recordValue(long value) throws RuntimeException {
|
@Override
|
||||||
throw new UnsupportedOperationException("unsupported method");
|
public void recordValue(long value) throws RuntimeException {
|
||||||
}
|
throw new UnsupportedOperationException("unsupported method");
|
||||||
|
}
|
||||||
@Override
|
|
||||||
public void recordValueWithCount(long value, long count) throws RuntimeException {
|
@Override
|
||||||
throw new UnsupportedOperationException("unsupported method");
|
public void recordValueWithCount(long value, long count) throws RuntimeException {
|
||||||
}
|
throw new UnsupportedOperationException("unsupported method");
|
||||||
|
}
|
||||||
@Override
|
|
||||||
public long getValueAtPercentile(double percentile) {
|
@Override
|
||||||
throw new UnsupportedOperationException("unsupported method");
|
public long getValueAtPercentile(double percentile) {
|
||||||
}
|
throw new UnsupportedOperationException("unsupported method");
|
||||||
|
}
|
||||||
@Override
|
|
||||||
public List<Percentile> percentileList(int percentileTicksPerHalfDistance) {
|
@Override
|
||||||
throw new UnsupportedOperationException("unsupported method");
|
public List<Percentile> percentileList(int percentileTicksPerHalfDistance) {
|
||||||
}
|
throw new UnsupportedOperationException("unsupported method");
|
||||||
|
}
|
||||||
@Override
|
|
||||||
public Histogramer resetHistogram() {
|
@Override
|
||||||
throw new UnsupportedOperationException("unsupported method");
|
public Map<String, Object> describe() {
|
||||||
}
|
throw new UnsupportedOperationException("unsupported method");
|
||||||
|
}
|
||||||
@Override
|
|
||||||
public Histogramer merge(Histogramer histogram) {
|
@Override
|
||||||
throw new UnsupportedOperationException("unsupported method");
|
public Histogramer resetHistogram() {
|
||||||
}
|
throw new UnsupportedOperationException("unsupported method");
|
||||||
|
}
|
||||||
@Override
|
|
||||||
public Histogramer makeCopy() throws RuntimeException{
|
@Override
|
||||||
int cookie = byteBuffer.getInt(initPosition);
|
public Histogramer merge(Histogramer histogram) {
|
||||||
if(getCookieBase(cookie) == V2CompressedEncodingCookieBase){
|
throw new UnsupportedOperationException("unsupported method");
|
||||||
try {
|
}
|
||||||
return ArrayHistogram.decodeFromCompressedByteBuffer(byteBuffer, 2);
|
|
||||||
} catch (DataFormatException e) {
|
@Override
|
||||||
throw new RuntimeException(e);
|
public Histogramer makeCopy() throws RuntimeException{
|
||||||
}
|
int cookie = byteBuffer.getInt(initPosition);
|
||||||
}else if(getCookieBase(cookie) == V2EncodingCookieBase){
|
if(getCookieBase(cookie) == V2CompressedEncodingCookieBase){
|
||||||
return ArrayHistogram.decodeFromByteBuffer(byteBuffer, 2);
|
try {
|
||||||
}
|
return ArrayHistogram.decodeFromCompressedByteBuffer(byteBuffer, 2);
|
||||||
throw new UnsupportedOperationException("unsupported method");
|
} catch (DataFormatException e) {
|
||||||
}
|
throw new RuntimeException(e);
|
||||||
|
}
|
||||||
@Override
|
}else if(getCookieBase(cookie) == V2EncodingCookieBase){
|
||||||
public byte[] toBytes() {
|
return ArrayHistogram.decodeFromByteBuffer(byteBuffer, 2);
|
||||||
int size = byteBuffer.limit() - initPosition;
|
}
|
||||||
byte[] bytes = new byte[size];
|
throw new UnsupportedOperationException("unsupported method");
|
||||||
assert byteBuffer.order() == ByteOrder.BIG_ENDIAN;
|
}
|
||||||
int oldPosition = byteBuffer.position();
|
|
||||||
byteBuffer.position(initPosition);
|
@Override
|
||||||
byteBuffer.get(bytes, 0, size);
|
public byte[] toBytes() {
|
||||||
byteBuffer.position(oldPosition);
|
int size = byteBuffer.limit() - initPosition;
|
||||||
return bytes;
|
byte[] bytes = new byte[size];
|
||||||
}
|
assert byteBuffer.order() == ByteOrder.BIG_ENDIAN;
|
||||||
}
|
int oldPosition = byteBuffer.position();
|
||||||
|
byteBuffer.position(initPosition);
|
||||||
|
byteBuffer.get(bytes, 0, size);
|
||||||
|
byteBuffer.position(oldPosition);
|
||||||
|
return bytes;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -1,85 +1,90 @@
|
|||||||
package org.HdrHistogram;
|
package org.HdrHistogram;
|
||||||
|
|
||||||
import java.nio.ByteBuffer;
|
import java.nio.ByteBuffer;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
|
import java.util.Map;
|
||||||
public class HistogramSketch {
|
|
||||||
public Histogramer hisImpl = null;
|
public class HistogramSketch {
|
||||||
|
public Histogramer hisImpl = null;
|
||||||
public HistogramSketch(final int numberOfSignificantValueDigits){
|
|
||||||
hisImpl = new ArrayHistogram(numberOfSignificantValueDigits);
|
public HistogramSketch(final int numberOfSignificantValueDigits){
|
||||||
}
|
hisImpl = new ArrayHistogram(numberOfSignificantValueDigits);
|
||||||
|
}
|
||||||
public HistogramSketch(final long lowestDiscernibleValue, final long highestTrackableValue,
|
|
||||||
final int numberOfSignificantValueDigits, final boolean autoResize){
|
public HistogramSketch(final long lowestDiscernibleValue, final long highestTrackableValue,
|
||||||
ArrayHistogram histogram = new ArrayHistogram(lowestDiscernibleValue, highestTrackableValue, numberOfSignificantValueDigits);
|
final int numberOfSignificantValueDigits, final boolean autoResize){
|
||||||
histogram.setAutoResize(autoResize);
|
ArrayHistogram histogram = new ArrayHistogram(lowestDiscernibleValue, highestTrackableValue, numberOfSignificantValueDigits);
|
||||||
hisImpl = histogram;
|
histogram.setAutoResize(autoResize);
|
||||||
}
|
hisImpl = histogram;
|
||||||
|
}
|
||||||
public HistogramSketch(final Histogramer that) {
|
|
||||||
hisImpl = that;
|
public HistogramSketch(final Histogramer that) {
|
||||||
}
|
hisImpl = that;
|
||||||
|
}
|
||||||
/**
|
|
||||||
* Copy constructor used by copy().
|
/**
|
||||||
*/
|
* Copy constructor used by copy().
|
||||||
HistogramSketch(final HistogramSketch that) {
|
*/
|
||||||
hisImpl = that.hisImpl.makeCopy();
|
HistogramSketch(final HistogramSketch that) {
|
||||||
}
|
hisImpl = that.hisImpl.makeCopy();
|
||||||
|
}
|
||||||
/**
|
|
||||||
* 复制hisImpl到堆内存实例hisImpl
|
/**
|
||||||
*/
|
* 复制hisImpl到堆内存实例hisImpl
|
||||||
public HistogramSketch copy() {
|
*/
|
||||||
return new HistogramSketch(this);
|
public HistogramSketch copy() {
|
||||||
}
|
return new HistogramSketch(this);
|
||||||
|
}
|
||||||
public void reset() {
|
|
||||||
hisImpl = hisImpl.resetHistogram();
|
public void reset() {
|
||||||
}
|
hisImpl = hisImpl.resetHistogram();
|
||||||
|
}
|
||||||
public long getTotalCount(){
|
|
||||||
return hisImpl.getTotalCount();
|
public long getTotalCount(){
|
||||||
}
|
return hisImpl.getTotalCount();
|
||||||
|
}
|
||||||
public void recordValue(long value){
|
|
||||||
hisImpl.recordValue(value);
|
public void recordValue(long value){
|
||||||
}
|
hisImpl.recordValue(value);
|
||||||
|
}
|
||||||
public void recordValueWithCount(long value, long count){
|
|
||||||
hisImpl.recordValueWithCount(value, count);
|
public void recordValueWithCount(long value, long count){
|
||||||
}
|
hisImpl.recordValueWithCount(value, count);
|
||||||
|
}
|
||||||
public long getValueAtPercentile(double percentile){
|
|
||||||
return hisImpl.getValueAtPercentile(percentile);
|
public long getValueAtPercentile(double percentile){
|
||||||
}
|
return hisImpl.getValueAtPercentile(percentile);
|
||||||
|
}
|
||||||
public List<Percentile> percentileList(int percentileTicksPerHalfDistance){
|
|
||||||
return hisImpl.percentileList(percentileTicksPerHalfDistance);
|
public List<Percentile> percentileList(int percentileTicksPerHalfDistance){
|
||||||
}
|
return hisImpl.percentileList(percentileTicksPerHalfDistance);
|
||||||
|
}
|
||||||
public static final int getUpdatableSerializationBytes(long lowestDiscernibleValue, long highestTrackableValue, int numberOfSignificantValueDigits){
|
|
||||||
return DirectArrayHistogram.getUpdatableSerializationBytes(lowestDiscernibleValue, highestTrackableValue, numberOfSignificantValueDigits);
|
public Map<String, Object> describe(){
|
||||||
}
|
return hisImpl.describe();
|
||||||
|
}
|
||||||
public byte[] toBytes() {
|
|
||||||
return hisImpl.toBytes();
|
public static final int getUpdatableSerializationBytes(long lowestDiscernibleValue, long highestTrackableValue, int numberOfSignificantValueDigits){
|
||||||
}
|
return DirectArrayHistogram.getUpdatableSerializationBytes(lowestDiscernibleValue, highestTrackableValue, numberOfSignificantValueDigits);
|
||||||
|
}
|
||||||
public static HistogramSketch fromBytes(byte[] bytes) {
|
|
||||||
return new HistogramSketch(ArrayHistogram.fromBytes(bytes));
|
public byte[] toBytes() {
|
||||||
}
|
return hisImpl.toBytes();
|
||||||
|
}
|
||||||
public static HistogramSketch fromByteBuffer(ByteBuffer byteBuffer) {
|
|
||||||
return new HistogramSketch(ArrayHistogram.fromByteBuffer(byteBuffer));
|
public static HistogramSketch fromBytes(byte[] bytes) {
|
||||||
}
|
return new HistogramSketch(ArrayHistogram.fromBytes(bytes));
|
||||||
|
}
|
||||||
public static HistogramSketch wrapBytes(byte[] bytes) {
|
|
||||||
return new HistogramSketch(DirectMapHistogram.wrapBytes(bytes));
|
public static HistogramSketch fromByteBuffer(ByteBuffer byteBuffer) {
|
||||||
}
|
return new HistogramSketch(ArrayHistogram.fromByteBuffer(byteBuffer));
|
||||||
|
}
|
||||||
public static HistogramSketch wrapByteBuffer(ByteBuffer byteBuffer) {
|
|
||||||
return new HistogramSketch(DirectMapHistogram.wrapByteBuffer(byteBuffer));
|
public static HistogramSketch wrapBytes(byte[] bytes) {
|
||||||
}
|
return new HistogramSketch(DirectMapHistogram.wrapBytes(bytes));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public static HistogramSketch wrapByteBuffer(ByteBuffer byteBuffer) {
|
||||||
|
return new HistogramSketch(DirectMapHistogram.wrapByteBuffer(byteBuffer));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -1,34 +1,37 @@
|
|||||||
package org.HdrHistogram;
|
package org.HdrHistogram;
|
||||||
|
|
||||||
import java.nio.ByteBuffer;
|
import java.nio.ByteBuffer;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
|
import java.util.Map;
|
||||||
public interface Histogramer {
|
|
||||||
long getTotalCount();
|
public interface Histogramer {
|
||||||
|
long getTotalCount();
|
||||||
void recordValue(long value) throws RuntimeException;
|
|
||||||
|
void recordValue(long value) throws RuntimeException;
|
||||||
void recordValueWithCount(long value, long count) throws RuntimeException;
|
|
||||||
|
void recordValueWithCount(long value, long count) throws RuntimeException;
|
||||||
long getValueAtPercentile(double percentile);
|
|
||||||
|
long getValueAtPercentile(double percentile);
|
||||||
List<Percentile> percentileList(int percentileTicksPerHalfDistance);
|
|
||||||
|
List<Percentile> percentileList(int percentileTicksPerHalfDistance);
|
||||||
Histogramer resetHistogram();
|
|
||||||
|
Map<String, Object> describe();
|
||||||
Histogramer merge(Histogramer histogram);
|
|
||||||
|
Histogramer resetHistogram();
|
||||||
// 复制到堆内存实例ArrayHistogram
|
|
||||||
Histogramer makeCopy();
|
Histogramer merge(Histogramer histogram);
|
||||||
|
|
||||||
byte[] toBytes();
|
// 复制到堆内存实例ArrayHistogram
|
||||||
|
Histogramer makeCopy();
|
||||||
default byte[] byteBuffer2Bytes(ByteBuffer byteBuffer){
|
|
||||||
//必须调用完后flip()才可以调用此方法
|
byte[] toBytes();
|
||||||
byteBuffer.flip();
|
|
||||||
int len = byteBuffer.limit() - byteBuffer.position();
|
default byte[] byteBuffer2Bytes(ByteBuffer byteBuffer){
|
||||||
byte[] bytes = new byte[len];
|
//必须调用完后flip()才可以调用此方法
|
||||||
byteBuffer.get(bytes);
|
byteBuffer.flip();
|
||||||
return bytes;
|
int len = byteBuffer.limit() - byteBuffer.position();
|
||||||
}
|
byte[] bytes = new byte[len];
|
||||||
}
|
byteBuffer.get(bytes);
|
||||||
|
return bytes;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -1,41 +1,50 @@
|
|||||||
package org.HdrHistogram;
|
package org.HdrHistogram;
|
||||||
|
|
||||||
public class Percentile {
|
public class Percentile {
|
||||||
public long value;
|
public long value;
|
||||||
public long count;
|
public long count;
|
||||||
public double percentile;
|
public double percentile;
|
||||||
|
|
||||||
public Percentile() {
|
public Percentile() {
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
public Percentile(long value, long count, double percentile) {
|
public Percentile(long value, long count, double percentile) {
|
||||||
this.value = value;
|
this.value = value;
|
||||||
this.count = count;
|
this.count = count;
|
||||||
this.percentile = percentile;
|
this.percentile = percentile;
|
||||||
}
|
}
|
||||||
|
|
||||||
public long getValue() {
|
public long getValue() {
|
||||||
return value;
|
return value;
|
||||||
}
|
}
|
||||||
|
|
||||||
public void setValue(long value) {
|
public void setValue(long value) {
|
||||||
this.value = value;
|
this.value = value;
|
||||||
}
|
}
|
||||||
|
|
||||||
public long getCount() {
|
public long getCount() {
|
||||||
return count;
|
return count;
|
||||||
}
|
}
|
||||||
|
|
||||||
public void setCount(long count) {
|
public void setCount(long count) {
|
||||||
this.count = count;
|
this.count = count;
|
||||||
}
|
}
|
||||||
|
|
||||||
public double getPercentile() {
|
public double getPercentile() {
|
||||||
return percentile;
|
return percentile;
|
||||||
}
|
}
|
||||||
|
|
||||||
public void setPercentile(double percentile) {
|
public void setPercentile(double percentile) {
|
||||||
this.percentile = percentile;
|
this.percentile = percentile;
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
@Override
|
||||||
|
public String toString() {
|
||||||
|
return "Percentile{" +
|
||||||
|
"value=" + value +
|
||||||
|
", count=" + count +
|
||||||
|
", percentile=" + percentile +
|
||||||
|
'}';
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -21,7 +21,7 @@ public class HdrHistogramAggregatorFactory extends AggregatorFactory {
|
|||||||
public static final long DEFAULT_HIGHEST = 2;
|
public static final long DEFAULT_HIGHEST = 2;
|
||||||
public static final int DEFAULT_SIGNIFICANT = 1;
|
public static final int DEFAULT_SIGNIFICANT = 1;
|
||||||
public static final boolean DEFAULT_AUTO_RESIZE = true;
|
public static final boolean DEFAULT_AUTO_RESIZE = true;
|
||||||
public static final long BUFFER_AUTO_RESIZE_HIGHEST = 100000000L * 1000000L;
|
public static final long BUFFER_AUTO_RESIZE_HIGHEST = 100000000L * 100L;
|
||||||
public static final Comparator<HistogramSketch> COMPARATOR =
|
public static final Comparator<HistogramSketch> COMPARATOR =
|
||||||
Comparator.nullsFirst(Comparator.comparingLong(HistogramSketch::getTotalCount));
|
Comparator.nullsFirst(Comparator.comparingLong(HistogramSketch::getTotalCount));
|
||||||
|
|
||||||
|
|||||||
@@ -1,134 +1,134 @@
|
|||||||
package org.apache.druid.query.aggregation.sketch.HdrHistogram;
|
package org.apache.druid.query.aggregation.sketch.HdrHistogram;
|
||||||
|
|
||||||
import it.unimi.dsi.fastutil.ints.Int2ObjectMap;
|
import it.unimi.dsi.fastutil.ints.Int2ObjectMap;
|
||||||
import it.unimi.dsi.fastutil.ints.Int2ObjectOpenHashMap;
|
import it.unimi.dsi.fastutil.ints.Int2ObjectOpenHashMap;
|
||||||
import org.HdrHistogram.*;
|
import org.HdrHistogram.*;
|
||||||
import org.apache.druid.java.util.common.logger.Logger;
|
import org.apache.druid.java.util.common.logger.Logger;
|
||||||
import org.apache.druid.query.aggregation.BufferAggregator;
|
import org.apache.druid.query.aggregation.BufferAggregator;
|
||||||
import org.apache.druid.query.monomorphicprocessing.RuntimeShapeInspector;
|
import org.apache.druid.query.monomorphicprocessing.RuntimeShapeInspector;
|
||||||
import org.apache.druid.segment.BaseObjectColumnValueSelector;
|
import org.apache.druid.segment.BaseObjectColumnValueSelector;
|
||||||
|
|
||||||
import javax.annotation.Nullable;
|
import javax.annotation.Nullable;
|
||||||
import java.nio.ByteBuffer;
|
import java.nio.ByteBuffer;
|
||||||
import java.util.IdentityHashMap;
|
import java.util.IdentityHashMap;
|
||||||
|
|
||||||
public class HdrHistogramMergeBufferAggregator implements BufferAggregator {
|
public class HdrHistogramMergeBufferAggregator implements BufferAggregator {
|
||||||
private static final Logger LOG = new Logger(HdrHistogramAggregator.class);
|
private static final Logger LOG = new Logger(HdrHistogramAggregator.class);
|
||||||
private long lastTs = 0L;
|
private long lastTs = 0L;
|
||||||
private final BaseObjectColumnValueSelector<HistogramSketch> selector;
|
private final BaseObjectColumnValueSelector<HistogramSketch> selector;
|
||||||
private final long lowestDiscernibleValue;
|
private final long lowestDiscernibleValue;
|
||||||
private final long highestTrackableValue;
|
private final long highestTrackableValue;
|
||||||
private final int numberOfSignificantValueDigits;
|
private final int numberOfSignificantValueDigits;
|
||||||
private final boolean autoResize;
|
private final boolean autoResize;
|
||||||
private final int size;
|
private final int size;
|
||||||
private final IdentityHashMap<ByteBuffer, Int2ObjectMap<HistogramUnion>> histograms = new IdentityHashMap<>();
|
private final IdentityHashMap<ByteBuffer, Int2ObjectMap<HistogramUnion>> histograms = new IdentityHashMap<>();
|
||||||
|
|
||||||
public HdrHistogramMergeBufferAggregator(
|
public HdrHistogramMergeBufferAggregator(
|
||||||
BaseObjectColumnValueSelector<HistogramSketch> selector,
|
BaseObjectColumnValueSelector<HistogramSketch> selector,
|
||||||
long lowestDiscernibleValue,
|
long lowestDiscernibleValue,
|
||||||
long highestTrackableValue,
|
long highestTrackableValue,
|
||||||
int numberOfSignificantValueDigits,
|
int numberOfSignificantValueDigits,
|
||||||
boolean autoResize,
|
boolean autoResize,
|
||||||
int size
|
int size
|
||||||
) {
|
) {
|
||||||
this.selector = selector;
|
this.selector = selector;
|
||||||
this.lowestDiscernibleValue = lowestDiscernibleValue;
|
this.lowestDiscernibleValue = lowestDiscernibleValue;
|
||||||
this.highestTrackableValue = highestTrackableValue;
|
this.highestTrackableValue = highestTrackableValue;
|
||||||
this.numberOfSignificantValueDigits = numberOfSignificantValueDigits;
|
this.numberOfSignificantValueDigits = numberOfSignificantValueDigits;
|
||||||
this.autoResize = autoResize;
|
this.autoResize = autoResize;
|
||||||
this.size = size;
|
this.size = size;
|
||||||
LOG.error("HdrHistogramMergeBufferAggregator gene:" + Thread.currentThread().getName() + "-" + Thread.currentThread().getId());
|
//LOG.error("HdrHistogramMergeBufferAggregator gene:" + Thread.currentThread().getName() + "-" + Thread.currentThread().getId());
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public synchronized void init(ByteBuffer buf, int position) {
|
public synchronized void init(ByteBuffer buf, int position) {
|
||||||
final int oldPosition = buf.position();
|
final int oldPosition = buf.position();
|
||||||
try {
|
try {
|
||||||
buf.position(position);
|
buf.position(position);
|
||||||
|
|
||||||
long highest = autoResize?HdrHistogramAggregatorFactory.BUFFER_AUTO_RESIZE_HIGHEST: highestTrackableValue;
|
long highest = autoResize?HdrHistogramAggregatorFactory.BUFFER_AUTO_RESIZE_HIGHEST: highestTrackableValue;
|
||||||
final DirectArrayHistogram histogram = new DirectArrayHistogram(lowestDiscernibleValue, highest, numberOfSignificantValueDigits, buf);
|
final DirectArrayHistogram histogram = new DirectArrayHistogram(lowestDiscernibleValue, highest, numberOfSignificantValueDigits, buf);
|
||||||
histogram.reset();
|
histogram.reset();
|
||||||
HistogramUnion union = new HistogramUnion(new HistogramSketch(histogram));
|
HistogramUnion union = new HistogramUnion(new HistogramSketch(histogram));
|
||||||
putUnion(buf, position, union);
|
putUnion(buf, position, union);
|
||||||
}finally {
|
}finally {
|
||||||
buf.position(oldPosition);
|
buf.position(oldPosition);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public synchronized void aggregate(ByteBuffer buf, int position) {
|
public synchronized void aggregate(ByteBuffer buf, int position) {
|
||||||
/*long ts = System.currentTimeMillis();
|
/*long ts = System.currentTimeMillis();
|
||||||
if(ts - lastTs > 2000){
|
if(ts - lastTs > 2000){
|
||||||
//LOG.warn("HdrHistogramMergeBufferAggregator call");
|
//LOG.warn("HdrHistogramMergeBufferAggregator call");
|
||||||
LOG.error("HdrHistogramMergeBufferAggregator call");
|
LOG.error("HdrHistogramMergeBufferAggregator call");
|
||||||
lastTs = ts;
|
lastTs = ts;
|
||||||
}*/
|
}*/
|
||||||
HistogramSketch h = selector.getObject();
|
HistogramSketch h = selector.getObject();
|
||||||
if (h == null) {
|
if (h == null) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
final int oldPosition = buf.position();
|
final int oldPosition = buf.position();
|
||||||
try {
|
try {
|
||||||
buf.position(position);
|
buf.position(position);
|
||||||
|
|
||||||
HistogramUnion union = histograms.get(buf).get(position);
|
HistogramUnion union = histograms.get(buf).get(position);
|
||||||
union.update(h);
|
union.update(h);
|
||||||
}finally{
|
}finally{
|
||||||
buf.position(oldPosition);
|
buf.position(oldPosition);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@Nullable
|
@Nullable
|
||||||
@Override
|
@Override
|
||||||
public synchronized HistogramSketch get(ByteBuffer buf, int position) {
|
public synchronized HistogramSketch get(ByteBuffer buf, int position) {
|
||||||
LOG.error("HdrHistogramMergeBufferAggregator get:" + 0 + "-" + Thread.currentThread().getId() + "-" + this);
|
//LOG.error("HdrHistogramMergeBufferAggregator get:" + 0 + "-" + Thread.currentThread().getId() + "-" + this);
|
||||||
HistogramUnion union = histograms.get(buf).get(position);
|
HistogramUnion union = histograms.get(buf).get(position);
|
||||||
//return histogram.copy();
|
//return histogram.copy();
|
||||||
return union.getResult().copy();
|
return union.getResult().copy();
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public synchronized void relocate(int oldPosition, int newPosition, ByteBuffer oldBuffer, ByteBuffer newBuffer) {
|
public synchronized void relocate(int oldPosition, int newPosition, ByteBuffer oldBuffer, ByteBuffer newBuffer) {
|
||||||
HistogramUnion union = histograms.get(oldBuffer).get(oldPosition);
|
HistogramUnion union = histograms.get(oldBuffer).get(oldPosition);
|
||||||
|
|
||||||
Int2ObjectMap<HistogramUnion> map = histograms.get(oldBuffer);
|
Int2ObjectMap<HistogramUnion> map = histograms.get(oldBuffer);
|
||||||
map.remove(oldPosition);
|
map.remove(oldPosition);
|
||||||
if (map.isEmpty()) {
|
if (map.isEmpty()) {
|
||||||
histograms.remove(oldBuffer);
|
histograms.remove(oldBuffer);
|
||||||
}
|
}
|
||||||
|
|
||||||
try {
|
try {
|
||||||
newBuffer.position(newPosition);
|
newBuffer.position(newPosition);
|
||||||
union.resetByteBuffer(newBuffer);
|
union.resetByteBuffer(newBuffer);
|
||||||
putUnion(newBuffer, newPosition, union);
|
putUnion(newBuffer, newPosition, union);
|
||||||
}finally {
|
}finally {
|
||||||
newBuffer.position(newPosition);
|
newBuffer.position(newPosition);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private void putUnion(final ByteBuffer buffer, final int position, final HistogramUnion union) {
|
private void putUnion(final ByteBuffer buffer, final int position, final HistogramUnion union) {
|
||||||
Int2ObjectMap<HistogramUnion> map = histograms.computeIfAbsent(buffer, buf -> new Int2ObjectOpenHashMap<>());
|
Int2ObjectMap<HistogramUnion> map = histograms.computeIfAbsent(buffer, buf -> new Int2ObjectOpenHashMap<>());
|
||||||
map.put(position, union);
|
map.put(position, union);
|
||||||
}
|
}
|
||||||
@Override
|
@Override
|
||||||
public float getFloat(ByteBuffer buf, int position) {
|
public float getFloat(ByteBuffer buf, int position) {
|
||||||
throw new UnsupportedOperationException("Not implemented");
|
throw new UnsupportedOperationException("Not implemented");
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public long getLong(ByteBuffer buf, int position) {
|
public long getLong(ByteBuffer buf, int position) {
|
||||||
throw new UnsupportedOperationException("Not implemented");
|
throw new UnsupportedOperationException("Not implemented");
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public void close() {
|
public void close() {
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public void inspectRuntimeShape(RuntimeShapeInspector inspector){
|
public void inspectRuntimeShape(RuntimeShapeInspector inspector){
|
||||||
inspector.visit("selector", selector);
|
inspector.visit("selector", selector);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -9,10 +9,7 @@ import com.google.common.annotations.VisibleForTesting;
|
|||||||
import com.google.inject.Binder;
|
import com.google.inject.Binder;
|
||||||
import org.HdrHistogram.HistogramSketch;
|
import org.HdrHistogram.HistogramSketch;
|
||||||
import org.apache.druid.initialization.DruidModule;
|
import org.apache.druid.initialization.DruidModule;
|
||||||
import org.apache.druid.query.aggregation.sketch.HdrHistogram.sql.HdrHistogramObjectSqlAggregator;
|
import org.apache.druid.query.aggregation.sketch.HdrHistogram.sql.*;
|
||||||
import org.apache.druid.query.aggregation.sketch.HdrHistogram.sql.HdrHistogramPercentilesOperatorConversion;
|
|
||||||
import org.apache.druid.query.aggregation.sketch.HdrHistogram.sql.HdrHistogramQuantileSqlAggregator;
|
|
||||||
import org.apache.druid.query.aggregation.sketch.HdrHistogram.sql.HdrHistogramQuantilesOperatorConversion;
|
|
||||||
import org.apache.druid.segment.column.ColumnType;
|
import org.apache.druid.segment.column.ColumnType;
|
||||||
import org.apache.druid.segment.serde.ComplexMetrics;
|
import org.apache.druid.segment.serde.ComplexMetrics;
|
||||||
import org.apache.druid.sql.guice.SqlBindings;
|
import org.apache.druid.sql.guice.SqlBindings;
|
||||||
@@ -27,6 +24,8 @@ public class HdrHistogramModule implements DruidModule {
|
|||||||
public static final byte QUANTILES_HDRHISTOGRAM_TO_QUANTILE_CACHE_TYPE_ID = 0x03;
|
public static final byte QUANTILES_HDRHISTOGRAM_TO_QUANTILE_CACHE_TYPE_ID = 0x03;
|
||||||
public static final byte QUANTILES_HDRHISTOGRAM_TO_QUANTILES_CACHE_TYPE_ID = 0x04;
|
public static final byte QUANTILES_HDRHISTOGRAM_TO_QUANTILES_CACHE_TYPE_ID = 0x04;
|
||||||
public static final byte QUANTILES_HDRHISTOGRAM_TO_PERCENTILES_CACHE_TYPE_ID = 0x05;
|
public static final byte QUANTILES_HDRHISTOGRAM_TO_PERCENTILES_CACHE_TYPE_ID = 0x05;
|
||||||
|
public static final byte QUANTILES_HDRHISTOGRAM_TO_DESCRIBE_CACHE_TYPE_ID = 0x06;
|
||||||
|
public static final byte QUANTILES_HDRHISTOGRAM_TO_PERCENTILES_DESCRIBE_CACHE_TYPE_ID = 0x07;
|
||||||
|
|
||||||
public static final String HDRHISTOGRAM_TYPE_NAME = "HdrHistogramSketch";
|
public static final String HDRHISTOGRAM_TYPE_NAME = "HdrHistogramSketch";
|
||||||
public static final ColumnType TYPE = ColumnType.ofComplex(HDRHISTOGRAM_TYPE_NAME);
|
public static final ColumnType TYPE = ColumnType.ofComplex(HDRHISTOGRAM_TYPE_NAME);
|
||||||
@@ -50,6 +49,8 @@ public class HdrHistogramModule implements DruidModule {
|
|||||||
|
|
||||||
SqlBindings.addOperatorConversion(binder, HdrHistogramQuantilesOperatorConversion.class);
|
SqlBindings.addOperatorConversion(binder, HdrHistogramQuantilesOperatorConversion.class);
|
||||||
SqlBindings.addOperatorConversion(binder, HdrHistogramPercentilesOperatorConversion.class);
|
SqlBindings.addOperatorConversion(binder, HdrHistogramPercentilesOperatorConversion.class);
|
||||||
|
SqlBindings.addOperatorConversion(binder, HdrHistogramDescribeOperatorConversion.class);
|
||||||
|
SqlBindings.addOperatorConversion(binder, HdrHistogramPercentilesDescribeOperatorConversion.class);
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
@@ -61,7 +62,9 @@ public class HdrHistogramModule implements DruidModule {
|
|||||||
new NamedType(HdrHistogramMergeAggregatorFactory.class, "HdrHistogramSketchMerge"),
|
new NamedType(HdrHistogramMergeAggregatorFactory.class, "HdrHistogramSketchMerge"),
|
||||||
new NamedType(HdrHistogramToQuantilePostAggregator.class, "HdrHistogramSketchToQuantile"),
|
new NamedType(HdrHistogramToQuantilePostAggregator.class, "HdrHistogramSketchToQuantile"),
|
||||||
new NamedType(HdrHistogramToQuantilesPostAggregator.class, "HdrHistogramSketchToQuantiles"),
|
new NamedType(HdrHistogramToQuantilesPostAggregator.class, "HdrHistogramSketchToQuantiles"),
|
||||||
new NamedType(HdrHistogramToPercentilesPostAggregator.class, "HdrHistogramSketchToPercentiles")
|
new NamedType(HdrHistogramToPercentilesPostAggregator.class, "HdrHistogramSketchToPercentiles"),
|
||||||
|
new NamedType(HdrHistogramToDescribePostAggregator.class, "HdrHistogramSketchToDescribe"),
|
||||||
|
new NamedType(HdrHistogramToPercentilesDescribePostAggregator.class, "HdrHistogramSketchToPercentilesDescription")
|
||||||
).addSerializer(HistogramSketch.class, new HistogramJsonSerializer())
|
).addSerializer(HistogramSketch.class, new HistogramJsonSerializer())
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,108 @@
|
|||||||
|
package org.apache.druid.query.aggregation.sketch.HdrHistogram;
|
||||||
|
|
||||||
|
import com.fasterxml.jackson.annotation.JsonCreator;
|
||||||
|
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||||
|
import com.google.common.collect.Sets;
|
||||||
|
import org.HdrHistogram.HistogramSketch;
|
||||||
|
import org.apache.druid.java.util.common.IAE;
|
||||||
|
import org.apache.druid.query.aggregation.AggregatorFactory;
|
||||||
|
import org.apache.druid.query.aggregation.PostAggregator;
|
||||||
|
import org.apache.druid.query.cache.CacheKeyBuilder;
|
||||||
|
import org.apache.druid.segment.ColumnInspector;
|
||||||
|
import org.apache.druid.segment.column.ColumnType;
|
||||||
|
|
||||||
|
import javax.annotation.Nullable;
|
||||||
|
import java.util.*;
|
||||||
|
|
||||||
|
public class HdrHistogramToDescribePostAggregator implements PostAggregator {
|
||||||
|
private final String name;
|
||||||
|
private final String fieldName;
|
||||||
|
|
||||||
|
@JsonCreator
|
||||||
|
public HdrHistogramToDescribePostAggregator(
|
||||||
|
@JsonProperty("name") String name,
|
||||||
|
@JsonProperty("fieldName") String fieldName
|
||||||
|
){
|
||||||
|
this.name = name;
|
||||||
|
this.fieldName = fieldName;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public ColumnType getType(ColumnInspector signature){
|
||||||
|
return ColumnType.STRING;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
@JsonProperty
|
||||||
|
public String getName() {
|
||||||
|
return name;
|
||||||
|
}
|
||||||
|
|
||||||
|
@JsonProperty
|
||||||
|
public String getFieldName() {
|
||||||
|
return fieldName;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Nullable
|
||||||
|
@Override
|
||||||
|
public Object compute(Map<String, Object> values) {
|
||||||
|
HistogramSketch histogram = (HistogramSketch) values.get(fieldName);
|
||||||
|
if(histogram == null){
|
||||||
|
return "{}"; //"[]"
|
||||||
|
}
|
||||||
|
return HdrHistogramModule.toJson(histogram.describe());
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public Comparator<double[]> getComparator()
|
||||||
|
{
|
||||||
|
throw new IAE("Comparing arrays of quantiles is not supported");
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public Set<String> getDependentFields()
|
||||||
|
{
|
||||||
|
return Sets.newHashSet(fieldName);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public PostAggregator decorate(Map<String, AggregatorFactory> aggregators) {
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public byte[] getCacheKey() {
|
||||||
|
CacheKeyBuilder builder = new CacheKeyBuilder(HdrHistogramModule.CACHE_TYPE_ID_OFFSET).appendByte(HdrHistogramModule.QUANTILES_HDRHISTOGRAM_TO_DESCRIBE_CACHE_TYPE_ID)
|
||||||
|
.appendString(fieldName);
|
||||||
|
return builder.build();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public boolean equals(Object o) {
|
||||||
|
if (this == o) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
if (o == null || getClass() != o.getClass()) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
HdrHistogramToDescribePostAggregator that = (HdrHistogramToDescribePostAggregator) o;
|
||||||
|
|
||||||
|
return name.equals(that.name) &&
|
||||||
|
fieldName.equals(that.fieldName);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public int hashCode() {
|
||||||
|
return Objects.hash(name, fieldName);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public String toString() {
|
||||||
|
return "HdrHistogramToDescribePostAggregator{" +
|
||||||
|
"name='" + name + '\'' +
|
||||||
|
", fieldName='" + fieldName + '\'' +
|
||||||
|
'}';
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
}
|
||||||
@@ -0,0 +1,125 @@
|
|||||||
|
package org.apache.druid.query.aggregation.sketch.HdrHistogram;
|
||||||
|
|
||||||
|
import com.fasterxml.jackson.annotation.JsonCreator;
|
||||||
|
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||||
|
import com.google.common.collect.Sets;
|
||||||
|
import org.HdrHistogram.HistogramSketch;
|
||||||
|
import org.HdrHistogram.Percentile;
|
||||||
|
import org.apache.druid.java.util.common.IAE;
|
||||||
|
import org.apache.druid.query.aggregation.AggregatorFactory;
|
||||||
|
import org.apache.druid.query.aggregation.PostAggregator;
|
||||||
|
import org.apache.druid.query.cache.CacheKeyBuilder;
|
||||||
|
import org.apache.druid.segment.ColumnInspector;
|
||||||
|
import org.apache.druid.segment.column.ColumnType;
|
||||||
|
|
||||||
|
import javax.annotation.Nullable;
|
||||||
|
import java.util.*;
|
||||||
|
|
||||||
|
public class HdrHistogramToPercentilesDescribePostAggregator implements PostAggregator {
|
||||||
|
private final String name;
|
||||||
|
private final String fieldName;
|
||||||
|
private final int percentileTicksPerHalfDistance;
|
||||||
|
|
||||||
|
@JsonCreator
|
||||||
|
public HdrHistogramToPercentilesDescribePostAggregator(
|
||||||
|
@JsonProperty("name") String name,
|
||||||
|
@JsonProperty("fieldName") String fieldName,
|
||||||
|
@JsonProperty("percentileTicksPerHalfDistance") int percentileTicksPerHalfDistance
|
||||||
|
){
|
||||||
|
this.name = name;
|
||||||
|
this.fieldName = fieldName;
|
||||||
|
this.percentileTicksPerHalfDistance = percentileTicksPerHalfDistance;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public ColumnType getType(ColumnInspector signature){
|
||||||
|
return ColumnType.STRING;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
@JsonProperty
|
||||||
|
public String getName() {
|
||||||
|
return name;
|
||||||
|
}
|
||||||
|
|
||||||
|
@JsonProperty
|
||||||
|
public String getFieldName() {
|
||||||
|
return fieldName;
|
||||||
|
}
|
||||||
|
|
||||||
|
@JsonProperty
|
||||||
|
public int getPercentileTicksPerHalfDistance() {
|
||||||
|
return percentileTicksPerHalfDistance;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Nullable
|
||||||
|
@Override
|
||||||
|
public Object compute(Map<String, Object> values) {
|
||||||
|
HistogramSketch histogram = (HistogramSketch) values.get(fieldName);
|
||||||
|
if(histogram == null){
|
||||||
|
return "{\"percentiles\":[],\"describe\":{}}";
|
||||||
|
}
|
||||||
|
List<Percentile> percentiles = histogram.percentileList(percentileTicksPerHalfDistance);
|
||||||
|
Map<String, Object> describe = histogram.describe();
|
||||||
|
Map<String, Object> rst = new LinkedHashMap<>();
|
||||||
|
rst.put("percentiles", percentiles);
|
||||||
|
rst.put("description", describe);
|
||||||
|
return HdrHistogramModule.toJson(rst);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public Comparator<double[]> getComparator()
|
||||||
|
{
|
||||||
|
throw new IAE("Comparing object is not supported");
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public Set<String> getDependentFields()
|
||||||
|
{
|
||||||
|
return Sets.newHashSet(fieldName);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public PostAggregator decorate(Map<String, AggregatorFactory> aggregators) {
|
||||||
|
return this;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public byte[] getCacheKey() {
|
||||||
|
CacheKeyBuilder builder = new CacheKeyBuilder(HdrHistogramModule.CACHE_TYPE_ID_OFFSET).appendByte(HdrHistogramModule.QUANTILES_HDRHISTOGRAM_TO_PERCENTILES_DESCRIBE_CACHE_TYPE_ID)
|
||||||
|
.appendString(fieldName);
|
||||||
|
builder.appendInt(percentileTicksPerHalfDistance);
|
||||||
|
return builder.build();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public boolean equals(Object o) {
|
||||||
|
if (this == o) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
if (o == null || getClass() != o.getClass()) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
HdrHistogramToPercentilesDescribePostAggregator that = (HdrHistogramToPercentilesDescribePostAggregator) o;
|
||||||
|
|
||||||
|
return percentileTicksPerHalfDistance == that.percentileTicksPerHalfDistance &&
|
||||||
|
name.equals(that.name) &&
|
||||||
|
fieldName.equals(that.fieldName);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public int hashCode() {
|
||||||
|
return Objects.hash(name, fieldName, percentileTicksPerHalfDistance);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public String toString() {
|
||||||
|
return "HdrHistogramToPercentilesDescribePostAggregator{" +
|
||||||
|
"name='" + name + '\'' +
|
||||||
|
", fieldName='" + fieldName + '\'' +
|
||||||
|
", probabilitys=" + percentileTicksPerHalfDistance +
|
||||||
|
'}';
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
}
|
||||||
@@ -0,0 +1,77 @@
|
|||||||
|
package org.apache.druid.query.aggregation.sketch.HdrHistogram.sql;
|
||||||
|
|
||||||
|
import org.apache.calcite.rex.RexCall;
|
||||||
|
import org.apache.calcite.rex.RexNode;
|
||||||
|
import org.apache.calcite.sql.SqlFunction;
|
||||||
|
import org.apache.calcite.sql.SqlOperator;
|
||||||
|
import org.apache.calcite.sql.type.ReturnTypes;
|
||||||
|
import org.apache.calcite.sql.type.SqlTypeFamily;
|
||||||
|
import org.apache.calcite.sql.type.SqlTypeName;
|
||||||
|
import org.apache.druid.java.util.common.StringUtils;
|
||||||
|
import org.apache.druid.query.aggregation.PostAggregator;
|
||||||
|
import org.apache.druid.query.aggregation.post.FieldAccessPostAggregator;
|
||||||
|
import org.apache.druid.query.aggregation.sketch.HdrHistogram.HdrHistogramToDescribePostAggregator;
|
||||||
|
import org.apache.druid.segment.column.RowSignature;
|
||||||
|
import org.apache.druid.sql.calcite.expression.DruidExpression;
|
||||||
|
import org.apache.druid.sql.calcite.expression.OperatorConversions;
|
||||||
|
import org.apache.druid.sql.calcite.expression.PostAggregatorVisitor;
|
||||||
|
import org.apache.druid.sql.calcite.expression.SqlOperatorConversion;
|
||||||
|
import org.apache.druid.sql.calcite.planner.PlannerContext;
|
||||||
|
|
||||||
|
import javax.annotation.Nullable;
|
||||||
|
import java.util.List;
|
||||||
|
|
||||||
|
public class HdrHistogramDescribeOperatorConversion implements SqlOperatorConversion {
|
||||||
|
private static final String FUNCTION_NAME = "HDR_DESCRIBE";
|
||||||
|
private static final SqlFunction SQL_FUNCTION = OperatorConversions
|
||||||
|
.operatorBuilder(StringUtils.toUpperCase(FUNCTION_NAME))
|
||||||
|
.operandTypes(SqlTypeFamily.ANY)
|
||||||
|
.requiredOperands(1)
|
||||||
|
.returnTypeInference(ReturnTypes.explicit(SqlTypeName.VARCHAR))
|
||||||
|
.build();
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public SqlOperator calciteOperator()
|
||||||
|
{
|
||||||
|
return SQL_FUNCTION;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public DruidExpression toDruidExpression(
|
||||||
|
PlannerContext plannerContext,
|
||||||
|
RowSignature rowSignature,
|
||||||
|
RexNode rexNode
|
||||||
|
)
|
||||||
|
{
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Nullable
|
||||||
|
@Override
|
||||||
|
public PostAggregator toPostAggregator(
|
||||||
|
PlannerContext plannerContext,
|
||||||
|
RowSignature rowSignature,
|
||||||
|
RexNode rexNode,
|
||||||
|
PostAggregatorVisitor postAggregatorVisitor
|
||||||
|
)
|
||||||
|
{
|
||||||
|
final List<RexNode> operands = ((RexCall) rexNode).getOperands();
|
||||||
|
final PostAggregator postAgg = OperatorConversions.toPostAggregator(
|
||||||
|
plannerContext,
|
||||||
|
rowSignature,
|
||||||
|
operands.get(0),
|
||||||
|
postAggregatorVisitor,
|
||||||
|
true
|
||||||
|
);
|
||||||
|
|
||||||
|
if (postAgg == null) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
return new HdrHistogramToDescribePostAggregator(
|
||||||
|
postAggregatorVisitor.getOutputNamePrefix() + postAggregatorVisitor.getAndIncrementCounter(),
|
||||||
|
((FieldAccessPostAggregator)postAgg).getFieldName()
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,88 @@
|
|||||||
|
package org.apache.druid.query.aggregation.sketch.HdrHistogram.sql;
|
||||||
|
|
||||||
|
import org.apache.calcite.rex.RexCall;
|
||||||
|
import org.apache.calcite.rex.RexLiteral;
|
||||||
|
import org.apache.calcite.rex.RexNode;
|
||||||
|
import org.apache.calcite.sql.SqlFunction;
|
||||||
|
import org.apache.calcite.sql.SqlKind;
|
||||||
|
import org.apache.calcite.sql.SqlOperator;
|
||||||
|
import org.apache.calcite.sql.type.ReturnTypes;
|
||||||
|
import org.apache.calcite.sql.type.SqlTypeFamily;
|
||||||
|
import org.apache.calcite.sql.type.SqlTypeName;
|
||||||
|
import org.apache.druid.java.util.common.StringUtils;
|
||||||
|
import org.apache.druid.query.aggregation.PostAggregator;
|
||||||
|
import org.apache.druid.query.aggregation.post.FieldAccessPostAggregator;
|
||||||
|
import org.apache.druid.query.aggregation.sketch.HdrHistogram.HdrHistogramToPercentilesDescribePostAggregator;
|
||||||
|
import org.apache.druid.segment.column.RowSignature;
|
||||||
|
import org.apache.druid.sql.calcite.expression.DruidExpression;
|
||||||
|
import org.apache.druid.sql.calcite.expression.OperatorConversions;
|
||||||
|
import org.apache.druid.sql.calcite.expression.PostAggregatorVisitor;
|
||||||
|
import org.apache.druid.sql.calcite.expression.SqlOperatorConversion;
|
||||||
|
import org.apache.druid.sql.calcite.planner.PlannerContext;
|
||||||
|
|
||||||
|
import javax.annotation.Nullable;
|
||||||
|
import java.util.List;
|
||||||
|
|
||||||
|
public class HdrHistogramPercentilesDescribeOperatorConversion implements SqlOperatorConversion {
|
||||||
|
private static final String FUNCTION_NAME = "HDR_GET_PERCENTILES_DESCRIPTION";
|
||||||
|
private static final SqlFunction SQL_FUNCTION = OperatorConversions
|
||||||
|
.operatorBuilder(StringUtils.toUpperCase(FUNCTION_NAME))
|
||||||
|
.operandTypes(SqlTypeFamily.ANY, SqlTypeFamily.NUMERIC)
|
||||||
|
.requiredOperands(1)
|
||||||
|
.returnTypeInference(ReturnTypes.explicit(SqlTypeName.VARCHAR))
|
||||||
|
.build();
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public SqlOperator calciteOperator()
|
||||||
|
{
|
||||||
|
return SQL_FUNCTION;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public DruidExpression toDruidExpression(
|
||||||
|
PlannerContext plannerContext,
|
||||||
|
RowSignature rowSignature,
|
||||||
|
RexNode rexNode
|
||||||
|
)
|
||||||
|
{
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Nullable
|
||||||
|
@Override
|
||||||
|
public PostAggregator toPostAggregator(
|
||||||
|
PlannerContext plannerContext,
|
||||||
|
RowSignature rowSignature,
|
||||||
|
RexNode rexNode,
|
||||||
|
PostAggregatorVisitor postAggregatorVisitor
|
||||||
|
)
|
||||||
|
{
|
||||||
|
final List<RexNode> operands = ((RexCall) rexNode).getOperands();
|
||||||
|
final PostAggregator postAgg = OperatorConversions.toPostAggregator(
|
||||||
|
plannerContext,
|
||||||
|
rowSignature,
|
||||||
|
operands.get(0),
|
||||||
|
postAggregatorVisitor,
|
||||||
|
true
|
||||||
|
);
|
||||||
|
|
||||||
|
if (postAgg == null) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
int percentileTicksPerHalfDistance = 5;
|
||||||
|
if (operands.size() == 2) {
|
||||||
|
if (!operands.get(1).isA(SqlKind.LITERAL)) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
percentileTicksPerHalfDistance = RexLiteral.intValue(operands.get(1));
|
||||||
|
}
|
||||||
|
|
||||||
|
return new HdrHistogramToPercentilesDescribePostAggregator(
|
||||||
|
postAggregatorVisitor.getOutputNamePrefix() + postAggregatorVisitor.getAndIncrementCounter(),
|
||||||
|
((FieldAccessPostAggregator)postAgg).getFieldName(),
|
||||||
|
percentileTicksPerHalfDistance
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,79 @@
|
|||||||
|
package org.apache.druid.query.aggregation.sketch.HdrHistogram;
|
||||||
|
|
||||||
|
import org.HdrHistogram.DirectArrayHistogram;
|
||||||
|
import org.HdrHistogram.HistogramSketch;
|
||||||
|
import org.HdrHistogram.Histogramer;
|
||||||
|
import org.HdrHistogram.Percentile;
|
||||||
|
import org.apache.commons.lang3.StringUtils;
|
||||||
|
import org.junit.Test;
|
||||||
|
|
||||||
|
import java.io.BufferedWriter;
|
||||||
|
import java.nio.ByteBuffer;
|
||||||
|
import java.nio.charset.StandardCharsets;
|
||||||
|
import java.nio.file.FileSystems;
|
||||||
|
import java.nio.file.Files;
|
||||||
|
import java.nio.file.Path;
|
||||||
|
import java.util.Random;
|
||||||
|
import java.util.concurrent.ThreadLocalRandom;
|
||||||
|
|
||||||
|
public class HistogramSketchTest {
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void describeTest() throws Exception{
|
||||||
|
DirectArrayHistogram histogram = new DirectArrayHistogram(1, 1000000, 3,
|
||||||
|
ByteBuffer.allocate(HistogramSketch.getUpdatableSerializationBytes(1, 1000000, 3)));
|
||||||
|
System.out.println(histogram.describe());
|
||||||
|
for (int i = 0; i < 10000; i++) {
|
||||||
|
histogram.recordValue(i);
|
||||||
|
}
|
||||||
|
System.out.println(histogram.describe());
|
||||||
|
for (Percentile percentile : histogram.percentileList(100)) {
|
||||||
|
System.out.println(percentile);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void describeTest1() throws Exception{
|
||||||
|
HistogramSketch histogram = new HistogramSketch(1);
|
||||||
|
System.out.println(histogram.describe());
|
||||||
|
for (int i = 0; i < 10000; i++) {
|
||||||
|
histogram.recordValue(i);
|
||||||
|
}
|
||||||
|
System.out.println(histogram.describe());
|
||||||
|
for (Percentile percentile : histogram.percentileList(100)) {
|
||||||
|
System.out.println(percentile);
|
||||||
|
}
|
||||||
|
System.out.println(StringUtils.repeat('#', 100));
|
||||||
|
histogram = new HistogramSketch(1);
|
||||||
|
for (int i = 0; i < 10000; i++) {
|
||||||
|
histogram.recordValue(ThreadLocalRandom.current().nextLong(100000));
|
||||||
|
}
|
||||||
|
System.out.println(histogram.describe());
|
||||||
|
for (Percentile percentile : histogram.percentileList(100)) {
|
||||||
|
System.out.println(percentile);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void describeTest3() throws Exception{
|
||||||
|
HistogramSketch histogram = new HistogramSketch(3);
|
||||||
|
System.out.println(histogram.describe());
|
||||||
|
for (int i = 0; i < 10000; i++) {
|
||||||
|
histogram.recordValue(i);
|
||||||
|
}
|
||||||
|
System.out.println(histogram.describe());
|
||||||
|
for (Percentile percentile : histogram.percentileList(100)) {
|
||||||
|
System.out.println(percentile);
|
||||||
|
}
|
||||||
|
System.out.println(StringUtils.repeat('#', 100));
|
||||||
|
histogram = new HistogramSketch(3);
|
||||||
|
for (int i = 0; i < 10000; i++) {
|
||||||
|
histogram.recordValue(ThreadLocalRandom.current().nextLong(100000));
|
||||||
|
}
|
||||||
|
System.out.println(histogram.describe());
|
||||||
|
for (Percentile percentile : histogram.percentileList(100)) {
|
||||||
|
System.out.println(percentile);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
@@ -219,6 +219,30 @@ public class HdrHistogramQuantileSqlAggregatorTest extends BaseCalciteQueryTest
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testSqlDESCRIBE() throws Exception {
|
||||||
|
String sql = "select HDR_GET_QUANTILES(HDR_HISTOGRAM(m1, 1, 100, 2), 0, 0.25, 0.5, 0.75, 1) a, HDR_DESCRIBE(HDR_HISTOGRAM(m1, 1, 100, 2)) b, HDR_DESCRIBE(HDR_HISTOGRAM(hist_m1, 1, 100, 2)) c from druid.foo";
|
||||||
|
QueryTestBuilder builder = testBuilder().sql(sql).skipVectorize();
|
||||||
|
builder.run();
|
||||||
|
QueryTestRunner.QueryResults queryResults = builder.results();
|
||||||
|
List<Object[]> results = queryResults.results;
|
||||||
|
for (Object[] result : results) {
|
||||||
|
System.out.println(Arrays.toString(result));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testSqlDESCRIBE2() throws Exception {
|
||||||
|
String sql = "select HDR_GET_QUANTILES(HDR_HISTOGRAM(m1, 1, 100, 2), 0, 0.25, 0.5, 0.75, 1) a, HDR_GET_PERCENTILES_DESCRIPTION(HDR_HISTOGRAM(m1, 1, 100, 2)) b, HDR_GET_PERCENTILES_DESCRIPTION(HDR_HISTOGRAM(hist_m1, 1, 100, 2)) c from druid.foo";
|
||||||
|
QueryTestBuilder builder = testBuilder().sql(sql).skipVectorize();
|
||||||
|
builder.run();
|
||||||
|
QueryTestRunner.QueryResults queryResults = builder.results();
|
||||||
|
List<Object[]> results = queryResults.results;
|
||||||
|
for (Object[] result : results) {
|
||||||
|
System.out.println(Arrays.toString(result));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void testSqlQuery() throws Exception {
|
public void testSqlQuery() throws Exception {
|
||||||
String[] columns = new String[]{"__time", "dim1", "dim2", "dim3", "cnt", "hist_m1", "m1"};
|
String[] columns = new String[]{"__time", "dim1", "dim2", "dim3", "cnt", "hist_m1", "m1"};
|
||||||
|
|||||||
143
druid-udf/pom.xml
Normal file
143
druid-udf/pom.xml
Normal file
@@ -0,0 +1,143 @@
|
|||||||
|
<?xml version="1.0" encoding="UTF-8"?>
|
||||||
|
<project xmlns="http://maven.apache.org/POM/4.0.0"
|
||||||
|
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||||
|
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||||
|
<modelVersion>4.0.0</modelVersion>
|
||||||
|
|
||||||
|
<groupId>org.example</groupId>
|
||||||
|
<artifactId>druid-udf_26.0.0</artifactId>
|
||||||
|
<name>druid-udf</name>
|
||||||
|
<version>1.0-SNAPSHOT</version>
|
||||||
|
|
||||||
|
<properties>
|
||||||
|
<maven.compiler.source>11</maven.compiler.source>
|
||||||
|
<maven.compiler.target>11</maven.compiler.target>
|
||||||
|
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
|
||||||
|
<druid.version>26.0.0</druid.version>
|
||||||
|
</properties>
|
||||||
|
|
||||||
|
<dependencies>
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.apache.druid</groupId>
|
||||||
|
<artifactId>druid-server</artifactId>
|
||||||
|
<version>${druid.version}</version>
|
||||||
|
<scope>provided</scope>
|
||||||
|
</dependency>
|
||||||
|
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.apache.druid</groupId>
|
||||||
|
<artifactId>druid-sql</artifactId>
|
||||||
|
<version>${druid.version}</version>
|
||||||
|
<scope>provided</scope>
|
||||||
|
</dependency>
|
||||||
|
|
||||||
|
<!-- Tests -->
|
||||||
|
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.easymock</groupId>
|
||||||
|
<artifactId>easymock</artifactId>
|
||||||
|
<version>4.3</version>
|
||||||
|
<scope>test</scope>
|
||||||
|
</dependency>
|
||||||
|
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.apache.druid</groupId>
|
||||||
|
<artifactId>druid-processing</artifactId>
|
||||||
|
<version>${druid.version}</version>
|
||||||
|
<type>test-jar</type>
|
||||||
|
<scope>test</scope>
|
||||||
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.apache.druid</groupId>
|
||||||
|
<artifactId>druid-server</artifactId>
|
||||||
|
<version>${druid.version}</version>
|
||||||
|
<scope>test</scope>
|
||||||
|
<type>test-jar</type>
|
||||||
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.apache.druid</groupId>
|
||||||
|
<artifactId>druid-sql</artifactId>
|
||||||
|
<version>${druid.version}</version>
|
||||||
|
<type>test-jar</type>
|
||||||
|
<scope>test</scope>
|
||||||
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>junit</groupId>
|
||||||
|
<artifactId>junit</artifactId>
|
||||||
|
<version>4.12</version>
|
||||||
|
<scope>test</scope>
|
||||||
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>com.alibaba.fastjson2</groupId>
|
||||||
|
<artifactId>fastjson2</artifactId>
|
||||||
|
<version>2.0.34</version>
|
||||||
|
<scope>test</scope>
|
||||||
|
</dependency>
|
||||||
|
</dependencies>
|
||||||
|
|
||||||
|
<build>
|
||||||
|
<plugins>
|
||||||
|
<plugin>
|
||||||
|
<groupId>org.apache.maven.plugins</groupId>
|
||||||
|
<artifactId>maven-compiler-plugin</artifactId>
|
||||||
|
<version>3.1</version>
|
||||||
|
<configuration>
|
||||||
|
<compilerArgument>-Xlint:unchecked</compilerArgument>
|
||||||
|
<source>11</source>
|
||||||
|
<target>11</target>
|
||||||
|
</configuration>
|
||||||
|
</plugin>
|
||||||
|
<plugin>
|
||||||
|
<groupId>org.apache.maven.plugins</groupId>
|
||||||
|
<artifactId>maven-surefire-plugin</artifactId>
|
||||||
|
<version>2.19.1</version>
|
||||||
|
<configuration>
|
||||||
|
<argLine>-Duser.timezone=UTC</argLine>
|
||||||
|
<redirectTestOutputToFile>true</redirectTestOutputToFile>
|
||||||
|
</configuration>
|
||||||
|
</plugin>
|
||||||
|
<plugin>
|
||||||
|
<groupId>org.apache.maven.plugins</groupId>
|
||||||
|
<artifactId>maven-assembly-plugin</artifactId>
|
||||||
|
<version>2.5.5</version>
|
||||||
|
<executions>
|
||||||
|
<execution>
|
||||||
|
<id>distro-assembly</id>
|
||||||
|
<phase>package</phase>
|
||||||
|
<goals>
|
||||||
|
<goal>single</goal>
|
||||||
|
</goals>
|
||||||
|
<configuration>
|
||||||
|
<finalName>${project.artifactId}-${project.version}</finalName>
|
||||||
|
<tarLongFileMode>posix</tarLongFileMode>
|
||||||
|
<descriptors>
|
||||||
|
<descriptor>src/assembly/assembly.xml</descriptor>
|
||||||
|
</descriptors>
|
||||||
|
</configuration>
|
||||||
|
</execution>
|
||||||
|
</executions>
|
||||||
|
</plugin>
|
||||||
|
<plugin>
|
||||||
|
<artifactId>maven-release-plugin</artifactId>
|
||||||
|
<version>2.5.3</version>
|
||||||
|
<dependencies>
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.apache.maven.scm</groupId>
|
||||||
|
<artifactId>maven-scm-provider-gitexe</artifactId>
|
||||||
|
<version>1.9.4</version>
|
||||||
|
</dependency>
|
||||||
|
</dependencies>
|
||||||
|
</plugin>
|
||||||
|
<plugin>
|
||||||
|
<groupId>org.apache.maven.plugins</groupId>
|
||||||
|
<artifactId>maven-jar-plugin</artifactId>
|
||||||
|
<version>3.0.2</version>
|
||||||
|
<configuration>
|
||||||
|
<archive>
|
||||||
|
<addMavenDescriptor>false</addMavenDescriptor>
|
||||||
|
</archive>
|
||||||
|
</configuration>
|
||||||
|
</plugin>
|
||||||
|
</plugins>
|
||||||
|
</build>
|
||||||
|
</project>
|
||||||
54
druid-udf/src/assembly/assembly.xml
Normal file
54
druid-udf/src/assembly/assembly.xml
Normal file
@@ -0,0 +1,54 @@
|
|||||||
|
<?xml version="1.0"?>
|
||||||
|
<!--
|
||||||
|
~ Copyright 2016 Imply Data, Inc.
|
||||||
|
~
|
||||||
|
~ Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
~ you may not use this file except in compliance with the License.
|
||||||
|
~ You may obtain a copy of the License at
|
||||||
|
~
|
||||||
|
~ http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
~
|
||||||
|
~ Unless required by applicable law or agreed to in writing, software
|
||||||
|
~ distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
~ See the License for the specific language governing permissions and
|
||||||
|
~ limitations under the License.
|
||||||
|
-->
|
||||||
|
|
||||||
|
<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.3"
|
||||||
|
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||||
|
xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.3 http://maven.apache.org/xsd/assembly-1.1.3.xsd">
|
||||||
|
<id>bin</id>
|
||||||
|
<formats>
|
||||||
|
<format>tar.gz</format>
|
||||||
|
</formats>
|
||||||
|
|
||||||
|
<baseDirectory>${project.name}</baseDirectory>
|
||||||
|
|
||||||
|
<dependencySets>
|
||||||
|
<dependencySet>
|
||||||
|
<useProjectArtifact>false</useProjectArtifact>
|
||||||
|
<useTransitiveDependencies>true</useTransitiveDependencies>
|
||||||
|
<outputDirectory>.</outputDirectory>
|
||||||
|
<unpack>false</unpack>
|
||||||
|
</dependencySet>
|
||||||
|
</dependencySets>
|
||||||
|
|
||||||
|
<fileSets>
|
||||||
|
<fileSet>
|
||||||
|
<directory>.</directory>
|
||||||
|
<outputDirectory/>
|
||||||
|
<includes>
|
||||||
|
<include>README.md</include>
|
||||||
|
<include>LICENSE</include>
|
||||||
|
</includes>
|
||||||
|
</fileSet>
|
||||||
|
<fileSet>
|
||||||
|
<directory>${project.build.directory}</directory>
|
||||||
|
<outputDirectory>.</outputDirectory>
|
||||||
|
<includes>
|
||||||
|
<include>*.jar</include>
|
||||||
|
</includes>
|
||||||
|
</fileSet>
|
||||||
|
</fileSets>
|
||||||
|
</assembly>
|
||||||
@@ -0,0 +1,23 @@
|
|||||||
|
package org.apache.druid.query.udf;
|
||||||
|
|
||||||
|
import com.google.inject.Binder;
|
||||||
|
import org.apache.druid.guice.ExpressionModule;
|
||||||
|
import org.apache.druid.initialization.DruidModule;
|
||||||
|
import org.apache.druid.query.udf.expressions.DimensionBucketExprMacro;
|
||||||
|
import org.apache.druid.query.udf.sql.DimensionBucketOperatorConversion;
|
||||||
|
import org.apache.druid.sql.guice.SqlBindings;
|
||||||
|
|
||||||
|
public class UdfModule implements DruidModule {
|
||||||
|
@Override
|
||||||
|
public void configure(Binder binder) {
|
||||||
|
SqlBindings.addOperatorConversion(binder, DimensionBucketOperatorConversion.class);
|
||||||
|
ExpressionModule.addExprMacro(binder, DimensionBucketExprMacro.class);
|
||||||
|
}
|
||||||
|
|
||||||
|
/*@Override
|
||||||
|
public List<? extends Module> getJacksonModules() {
|
||||||
|
// Register Jackson module for any classes we need to be able to use in JSON queries or ingestion specs.
|
||||||
|
return Collections.<Module>singletonList(new SimpleModule("UdfModule"));
|
||||||
|
}*/
|
||||||
|
|
||||||
|
}
|
||||||
@@ -0,0 +1,82 @@
|
|||||||
|
package org.apache.druid.query.udf.expressions;
|
||||||
|
|
||||||
|
import org.apache.druid.math.expr.*;
|
||||||
|
import org.apache.druid.math.expr.ExprMacroTable.ExprMacro;
|
||||||
|
|
||||||
|
import javax.annotation.Nullable;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.stream.Collectors;
|
||||||
|
|
||||||
|
public class DimensionBucketExprMacro implements ExprMacro {
|
||||||
|
private static final String NAME = "dimension_bucket";
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public String name() {
|
||||||
|
return NAME;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public Expr apply(List<Expr> args) {
|
||||||
|
validationHelperCheckMinArgumentCount(args, 2);
|
||||||
|
Expr bucketCnt = args.get(0);
|
||||||
|
if(!bucketCnt.isLiteral()|| bucketCnt.eval(InputBindings.nilBindings()).asInt() <= 0) {
|
||||||
|
throw validationFailed("first bucketCount argument must is int literal and > 0");
|
||||||
|
}
|
||||||
|
return new DimensionBucketExpr(args);
|
||||||
|
}
|
||||||
|
|
||||||
|
static class DimensionBucketExpr extends ExprMacroTable.BaseScalarMacroFunctionExpr {
|
||||||
|
private final int bucketCount;
|
||||||
|
|
||||||
|
public DimensionBucketExpr(List<Expr> args) {
|
||||||
|
super(NAME, args);
|
||||||
|
bucketCount = args.get(0).eval(InputBindings.nilBindings()).asInt();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public ExprEval eval(ObjectBinding bindings) {
|
||||||
|
int result = 1;
|
||||||
|
for (int i = 1; i < args.size(); i++) {
|
||||||
|
ExprEval eval = args.get(i).eval(bindings);
|
||||||
|
Object element = eval.value();
|
||||||
|
if(element instanceof Object[]){
|
||||||
|
for (Object ele : (Object[]) element) {
|
||||||
|
result = 31 * result + (ele == null ? 0 : ele.hashCode());
|
||||||
|
}
|
||||||
|
}else{
|
||||||
|
result = 31 * result + (element == null ? 0 : element.hashCode());
|
||||||
|
}
|
||||||
|
|
||||||
|
/*else if (element instanceof Number) {
|
||||||
|
//result = 31 * result + Integer.hashCode(((Number)element).intValue());
|
||||||
|
result = 31 * result + Long.hashCode(((Number)element).longValue());
|
||||||
|
}*/
|
||||||
|
}
|
||||||
|
|
||||||
|
int bucket = Math.abs(result) % bucketCount;
|
||||||
|
return ExprEval.of(IntToHexUtil.uInt16ToHexStringFast(bucket));
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public Expr visit(Shuttle shuttle) {
|
||||||
|
List<Expr> newArgs = args.stream().map(x -> x.visit(shuttle)).collect(Collectors.toList());
|
||||||
|
return shuttle.visit(new DimensionBucketExpr(newArgs));
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public BindingAnalysis analyzeInputs() {
|
||||||
|
return super.analyzeInputs();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Nullable
|
||||||
|
@Override
|
||||||
|
public ExpressionType getOutputType(InputBindingInspector inspector) {
|
||||||
|
return ExpressionType.STRING;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public boolean canVectorize(InputBindingInspector inspector) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,45 @@
|
|||||||
|
package org.apache.druid.query.udf.expressions;
|
||||||
|
|
||||||
|
import java.nio.charset.StandardCharsets;
|
||||||
|
|
||||||
|
public class IntToHexUtil {
|
||||||
|
static final byte[] digits = {
|
||||||
|
'0' , '1' , '2' , '3' , '4' , '5' ,
|
||||||
|
'6' , '7' , '8' , '9' , 'a' , 'b' ,
|
||||||
|
'c' , 'd' , 'e' , 'f' , 'g' , 'h' ,
|
||||||
|
'i' , 'j' , 'k' , 'l' , 'm' , 'n' ,
|
||||||
|
'o' , 'p' , 'q' , 'r' , 's' , 't' ,
|
||||||
|
'u' , 'v' , 'w' , 'x' , 'y' , 'z'
|
||||||
|
};
|
||||||
|
static final String[] uInt16HexsCache;
|
||||||
|
static final int uInt16HexsCacheSize = 8192;
|
||||||
|
|
||||||
|
static{
|
||||||
|
uInt16HexsCache = new String[uInt16HexsCacheSize];
|
||||||
|
for (int i = 0; i < uInt16HexsCacheSize; i++) {
|
||||||
|
uInt16HexsCache[i] = uInt16ToHexString(i);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public static String uInt16ToHexStringFast(int i){
|
||||||
|
if(i < uInt16HexsCacheSize){
|
||||||
|
return uInt16HexsCache[i];
|
||||||
|
}else{
|
||||||
|
return uInt16ToHexString(i);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private static String uInt16ToHexString(int i){
|
||||||
|
byte[] bytes = new byte[4];
|
||||||
|
int mask = 15; // 16 - 1
|
||||||
|
int value = i;
|
||||||
|
bytes[3] = digits[value & mask];
|
||||||
|
value >>>= 4;
|
||||||
|
bytes[2] = digits[value & mask];
|
||||||
|
value >>>= 4;
|
||||||
|
bytes[1] = digits[value & mask];
|
||||||
|
value >>>= 4;
|
||||||
|
bytes[0] = digits[value & mask];
|
||||||
|
return new String(bytes, StandardCharsets.US_ASCII);
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,43 @@
|
|||||||
|
package org.apache.druid.query.udf.sql;
|
||||||
|
|
||||||
|
import org.apache.calcite.rex.RexNode;
|
||||||
|
import org.apache.calcite.sql.SqlFunction;
|
||||||
|
import org.apache.calcite.sql.SqlFunctionCategory;
|
||||||
|
import org.apache.calcite.sql.SqlKind;
|
||||||
|
import org.apache.calcite.sql.SqlOperator;
|
||||||
|
import org.apache.calcite.sql.type.OperandTypes;
|
||||||
|
import org.apache.calcite.sql.type.ReturnTypes;
|
||||||
|
import org.apache.calcite.sql.type.SqlOperandCountRanges;
|
||||||
|
import org.apache.calcite.sql.type.SqlTypeName;
|
||||||
|
import org.apache.druid.segment.column.RowSignature;
|
||||||
|
import org.apache.druid.sql.calcite.expression.DruidExpression;
|
||||||
|
import org.apache.druid.sql.calcite.expression.OperatorConversions;
|
||||||
|
import org.apache.druid.sql.calcite.expression.SqlOperatorConversion;
|
||||||
|
import org.apache.druid.sql.calcite.planner.Calcites;
|
||||||
|
import org.apache.druid.sql.calcite.planner.PlannerContext;
|
||||||
|
|
||||||
|
import javax.annotation.Nullable;
|
||||||
|
|
||||||
|
public class DimensionBucketOperatorConversion implements SqlOperatorConversion {
|
||||||
|
private static final SqlFunction SQL_FUNCTION = new SqlFunction(
|
||||||
|
"DIMENSION_BUCKET",
|
||||||
|
SqlKind.OTHER_FUNCTION,
|
||||||
|
ReturnTypes.explicit(
|
||||||
|
factory -> Calcites.createSqlTypeWithNullability(factory, SqlTypeName.VARCHAR, true)
|
||||||
|
),
|
||||||
|
null,
|
||||||
|
OperandTypes.variadic(SqlOperandCountRanges.from(2)),
|
||||||
|
SqlFunctionCategory.USER_DEFINED_FUNCTION
|
||||||
|
);
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public SqlOperator calciteOperator() {
|
||||||
|
return SQL_FUNCTION;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Nullable
|
||||||
|
@Override
|
||||||
|
public DruidExpression toDruidExpression(PlannerContext plannerContext, RowSignature rowSignature, RexNode rexNode) {
|
||||||
|
return OperatorConversions.convertDirectCall(plannerContext, rowSignature, rexNode, "dimension_bucket");
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1 @@
|
|||||||
|
org.apache.druid.query.udf.UdfModule
|
||||||
@@ -0,0 +1,146 @@
|
|||||||
|
package org.apache.druid.query.udf.expressions;
|
||||||
|
|
||||||
|
import com.google.common.collect.ImmutableMap;
|
||||||
|
import org.apache.druid.math.expr.*;
|
||||||
|
import org.apache.druid.testing.InitializedNullHandlingTest;
|
||||||
|
import org.junit.Test;
|
||||||
|
|
||||||
|
import java.util.Collections;
|
||||||
|
|
||||||
|
public class DimensionBucketExprTest extends InitializedNullHandlingTest {
|
||||||
|
private final ExprMacroTable exprMacroTable = new ExprMacroTable(Collections.singletonList(new DimensionBucketExprMacro()));
|
||||||
|
Expr.ObjectBinding inputBindings = InputBindings.forInputSuppliers(
|
||||||
|
new ImmutableMap.Builder<String, InputBindings.InputSupplier>()
|
||||||
|
.put("string", InputBindings.inputSupplier(ExpressionType.STRING, () -> "abcdef"))
|
||||||
|
.put("long", InputBindings.inputSupplier(ExpressionType.LONG, () -> 1234L))
|
||||||
|
.put("double", InputBindings.inputSupplier(ExpressionType.DOUBLE, () -> 1.234))
|
||||||
|
.put("array1", InputBindings.inputSupplier(ExpressionType.STRING_ARRAY, () -> new Object[]{"1", "2", "3"}))
|
||||||
|
.put("array2", InputBindings.inputSupplier(ExpressionType.STRING_ARRAY, () -> new String[]{"1", "2", "3"}))
|
||||||
|
.put("nullString", InputBindings.inputSupplier(ExpressionType.STRING, () -> null))
|
||||||
|
.put("nullLong", InputBindings.inputSupplier(ExpressionType.LONG, () -> null))
|
||||||
|
.put("nullDouble", InputBindings.inputSupplier(ExpressionType.DOUBLE, () -> null))
|
||||||
|
.build()
|
||||||
|
);
|
||||||
|
|
||||||
|
Expr.ObjectBinding[] inputBindingArray = new Expr.ObjectBinding[]{
|
||||||
|
InputBindings.forInputSuppliers(
|
||||||
|
new ImmutableMap.Builder<String, InputBindings.InputSupplier>()
|
||||||
|
.put("device_id", InputBindings.inputSupplier(ExpressionType.STRING, () -> "1"))
|
||||||
|
.put("rule_id", InputBindings.inputSupplier(ExpressionType.LONG, () -> 81))
|
||||||
|
.put("template_id", InputBindings.inputSupplier(ExpressionType.LONG, () -> 81))
|
||||||
|
.put("chart_id", InputBindings.inputSupplier(ExpressionType.LONG, () -> 81))
|
||||||
|
.put("version", InputBindings.inputSupplier(ExpressionType.LONG, () -> 1))
|
||||||
|
.put("client_ip_object", InputBindings.inputSupplier(ExpressionType.STRING_ARRAY, () -> null))
|
||||||
|
.put("server_ip_object", InputBindings.inputSupplier(ExpressionType.STRING_ARRAY, () -> null))
|
||||||
|
.put("fqdn_category", InputBindings.inputSupplier(ExpressionType.STRING_ARRAY, () -> null))
|
||||||
|
.put("client_ip", InputBindings.inputSupplier(ExpressionType.STRING, () -> null))
|
||||||
|
.put("server_ip", InputBindings.inputSupplier(ExpressionType.STRING, () -> null))
|
||||||
|
.put("server_fqdn", InputBindings.inputSupplier(ExpressionType.STRING, () -> null))
|
||||||
|
.put("server_domain", InputBindings.inputSupplier(ExpressionType.STRING, () -> null))
|
||||||
|
.put("application", InputBindings.inputSupplier(ExpressionType.STRING, () -> null))
|
||||||
|
.build()
|
||||||
|
),
|
||||||
|
InputBindings.forInputSuppliers(
|
||||||
|
new ImmutableMap.Builder<String, InputBindings.InputSupplier>()
|
||||||
|
.put("device_id", InputBindings.inputSupplier(ExpressionType.STRING, () -> "1"))
|
||||||
|
.put("rule_id", InputBindings.inputSupplier(ExpressionType.LONG, () -> 101))
|
||||||
|
.put("template_id", InputBindings.inputSupplier(ExpressionType.LONG, () -> 101))
|
||||||
|
.put("chart_id", InputBindings.inputSupplier(ExpressionType.LONG, () -> 101))
|
||||||
|
.put("version", InputBindings.inputSupplier(ExpressionType.LONG, () -> 1))
|
||||||
|
.put("client_ip_object", InputBindings.inputSupplier(ExpressionType.STRING_ARRAY, () -> new Object[]{"5","7","8"}))
|
||||||
|
.put("server_ip_object", InputBindings.inputSupplier(ExpressionType.STRING_ARRAY, () -> null))
|
||||||
|
.put("fqdn_category", InputBindings.inputSupplier(ExpressionType.STRING_ARRAY, () -> null))
|
||||||
|
.put("client_ip", InputBindings.inputSupplier(ExpressionType.STRING, () -> null))
|
||||||
|
.put("server_ip", InputBindings.inputSupplier(ExpressionType.STRING, () -> null))
|
||||||
|
.put("server_fqdn", InputBindings.inputSupplier(ExpressionType.STRING, () -> null))
|
||||||
|
.put("server_domain", InputBindings.inputSupplier(ExpressionType.STRING, () -> null))
|
||||||
|
.put("application", InputBindings.inputSupplier(ExpressionType.STRING, () -> null))
|
||||||
|
.build()
|
||||||
|
),
|
||||||
|
InputBindings.forInputSuppliers(
|
||||||
|
new ImmutableMap.Builder<String, InputBindings.InputSupplier>()
|
||||||
|
.put("device_id", InputBindings.inputSupplier(ExpressionType.STRING, () -> "1"))
|
||||||
|
.put("rule_id", InputBindings.inputSupplier(ExpressionType.LONG, () -> 271L))
|
||||||
|
.put("template_id", InputBindings.inputSupplier(ExpressionType.LONG, () -> 271L))
|
||||||
|
.put("chart_id", InputBindings.inputSupplier(ExpressionType.LONG, () -> 271L))
|
||||||
|
.put("version", InputBindings.inputSupplier(ExpressionType.LONG, () -> 1L))
|
||||||
|
.put("client_ip_object", InputBindings.inputSupplier(ExpressionType.STRING_ARRAY, () -> null))
|
||||||
|
.put("server_ip_object", InputBindings.inputSupplier(ExpressionType.STRING_ARRAY, () -> null))
|
||||||
|
.put("fqdn_category", InputBindings.inputSupplier(ExpressionType.STRING_ARRAY, () -> null))
|
||||||
|
.put("client_ip", InputBindings.inputSupplier(ExpressionType.STRING, () -> null))
|
||||||
|
.put("server_ip", InputBindings.inputSupplier(ExpressionType.STRING, () -> "5.245.228.51"))
|
||||||
|
.put("server_fqdn", InputBindings.inputSupplier(ExpressionType.STRING, () -> null))
|
||||||
|
.put("server_domain", InputBindings.inputSupplier(ExpressionType.STRING, () -> null))
|
||||||
|
.put("application", InputBindings.inputSupplier(ExpressionType.STRING, () -> null))
|
||||||
|
.build()
|
||||||
|
),
|
||||||
|
// ...
|
||||||
|
InputBindings.forInputSuppliers(
|
||||||
|
new ImmutableMap.Builder<String, InputBindings.InputSupplier>()
|
||||||
|
.put("device_id", InputBindings.inputSupplier(ExpressionType.STRING, () -> "1"))
|
||||||
|
.put("rule_id", InputBindings.inputSupplier(ExpressionType.LONG, () -> 81))
|
||||||
|
.put("template_id", InputBindings.inputSupplier(ExpressionType.LONG, () -> 81))
|
||||||
|
.put("chart_id", InputBindings.inputSupplier(ExpressionType.LONG, () -> 81))
|
||||||
|
.put("version", InputBindings.inputSupplier(ExpressionType.LONG, () -> 1))
|
||||||
|
.put("client_ip_object", InputBindings.inputSupplier(ExpressionType.STRING, () -> null))
|
||||||
|
.put("server_ip_object", InputBindings.inputSupplier(ExpressionType.STRING, () -> null))
|
||||||
|
.put("fqdn_category", InputBindings.inputSupplier(ExpressionType.STRING, () -> null))
|
||||||
|
.put("client_ip", InputBindings.inputSupplier(ExpressionType.STRING, () -> null))
|
||||||
|
.put("server_ip", InputBindings.inputSupplier(ExpressionType.STRING, () -> null))
|
||||||
|
.put("server_fqdn", InputBindings.inputSupplier(ExpressionType.STRING, () -> null))
|
||||||
|
.put("server_domain", InputBindings.inputSupplier(ExpressionType.STRING, () -> null))
|
||||||
|
.put("application", InputBindings.inputSupplier(ExpressionType.STRING, () -> null))
|
||||||
|
.build()
|
||||||
|
),
|
||||||
|
InputBindings.forInputSuppliers(
|
||||||
|
new ImmutableMap.Builder<String, InputBindings.InputSupplier>()
|
||||||
|
.put("device_id", InputBindings.inputSupplier(ExpressionType.STRING, () -> "1"))
|
||||||
|
.put("rule_id", InputBindings.inputSupplier(ExpressionType.LONG, () -> 101))
|
||||||
|
.put("template_id", InputBindings.inputSupplier(ExpressionType.LONG, () -> 101))
|
||||||
|
.put("chart_id", InputBindings.inputSupplier(ExpressionType.LONG, () -> 101))
|
||||||
|
.put("version", InputBindings.inputSupplier(ExpressionType.LONG, () -> 1))
|
||||||
|
.put("client_ip_object", InputBindings.inputSupplier(ExpressionType.STRING, () -> "5,7,8"))
|
||||||
|
.put("server_ip_object", InputBindings.inputSupplier(ExpressionType.STRING, () -> null))
|
||||||
|
.put("fqdn_category", InputBindings.inputSupplier(ExpressionType.STRING, () -> null))
|
||||||
|
.put("client_ip", InputBindings.inputSupplier(ExpressionType.STRING, () -> null))
|
||||||
|
.put("server_ip", InputBindings.inputSupplier(ExpressionType.STRING, () -> null))
|
||||||
|
.put("server_fqdn", InputBindings.inputSupplier(ExpressionType.STRING, () -> null))
|
||||||
|
.put("server_domain", InputBindings.inputSupplier(ExpressionType.STRING, () -> null))
|
||||||
|
.put("application", InputBindings.inputSupplier(ExpressionType.STRING, () -> null))
|
||||||
|
.build()
|
||||||
|
),
|
||||||
|
InputBindings.forInputSuppliers(
|
||||||
|
new ImmutableMap.Builder<String, InputBindings.InputSupplier>()
|
||||||
|
.put("device_id", InputBindings.inputSupplier(ExpressionType.STRING, () -> "1"))
|
||||||
|
.put("rule_id", InputBindings.inputSupplier(ExpressionType.LONG, () -> 271L))
|
||||||
|
.put("template_id", InputBindings.inputSupplier(ExpressionType.LONG, () -> 271L))
|
||||||
|
.put("chart_id", InputBindings.inputSupplier(ExpressionType.LONG, () -> 271L))
|
||||||
|
.put("version", InputBindings.inputSupplier(ExpressionType.LONG, () -> 1L))
|
||||||
|
.put("client_ip_object", InputBindings.inputSupplier(ExpressionType.STRING, () -> null))
|
||||||
|
.put("server_ip_object", InputBindings.inputSupplier(ExpressionType.STRING, () -> null))
|
||||||
|
.put("fqdn_category", InputBindings.inputSupplier(ExpressionType.STRING, () -> null))
|
||||||
|
.put("client_ip", InputBindings.inputSupplier(ExpressionType.STRING, () -> null))
|
||||||
|
.put("server_ip", InputBindings.inputSupplier(ExpressionType.STRING, () -> "5.245.228.51"))
|
||||||
|
.put("server_fqdn", InputBindings.inputSupplier(ExpressionType.STRING, () -> null))
|
||||||
|
.put("server_domain", InputBindings.inputSupplier(ExpressionType.STRING, () -> null))
|
||||||
|
.put("application", InputBindings.inputSupplier(ExpressionType.STRING, () -> null))
|
||||||
|
.build()
|
||||||
|
),
|
||||||
|
};
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void test() {
|
||||||
|
Expr expr = Parser.parse("dimension_bucket(1024, 100, 'aaa', string,long,double,array1, array2, nullString, nullLong)", exprMacroTable);
|
||||||
|
ExprEval eval = expr.eval(inputBindings);
|
||||||
|
System.out.println(eval.value());
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void test2() {
|
||||||
|
for (Expr.ObjectBinding objectBinding : inputBindingArray) {
|
||||||
|
Expr expr = Parser.parse("dimension_bucket(1024, device_id, rule_id, template_id, chart_id, version, client_ip_object, server_ip_object, fqdn_category, client_ip, server_ip, server_fqdn, server_domain, application)", exprMacroTable);
|
||||||
|
ExprEval eval = expr.eval(objectBinding);
|
||||||
|
System.out.println(objectBinding.get("rule_id") + ", bucket_id:" + eval.value());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user