gfwleak/galaxy-platform-algorithm-druid-extensions
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

GAL-617 Druid hdrhistogram扩展添加HDR_DESCRIBE、HDR_GET_PERCENTILES_DESCRIPTION函数

Browse Source
This commit is contained in:
lifengchao
2024-07-11 09:56:14 +08:00
parent 38c22db84d
commit 9a6c44112e
15 changed files with 1935 additions and 1350 deletions
Download Patch File Download Diff File Expand all files Collapse all files

33
druid-hdrhistogram/src/main/java/org/HdrHistogram/ArrayHistogram.java
View File

@@ -8,9 +8,7 @@ package org.HdrHistogram; /**
import java.io.IOException;
import java.io.ObjectInputStream;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.*;
import java.util.zip.DataFormatException;
/**
@@ -305,6 +303,35 @@ public class ArrayHistogram extends AbstractHistogram implements Histogramer{
return percentiles;
}
@Override
public Map<String, Object> describe() {
long min = getMinValue();
long max = getMaxValue(); // max = this.maxValue;
long count = getTotalCount();
double mean = getMean();
long sum = (long) (mean * count);
mean = Math.round(mean * 100.0) / 100.0;
long p25 = getValueAtPercentile(25);
long p50 = getValueAtPercentile(50);
long p75 = getValueAtPercentile(75);
long p90 = getValueAtPercentile(90);
long p95 = getValueAtPercentile(95);
long p99 = getValueAtPercentile(99);
Map<String, Object> rst = new LinkedHashMap<>();
rst.put("count", count);
rst.put("mean", mean);
rst.put("sum", sum);
rst.put("min", min);
rst.put("p25", p25);
rst.put("p50", p50);
rst.put("p75", p75);
rst.put("p90", p90);
rst.put("p95", p95);
rst.put("p99", p99);
rst.put("max", max);
return rst;
}
@Override
public Histogramer resetHistogram() {
if(isAutoResize()){

31
druid-hdrhistogram/src/main/java/org/HdrHistogram/DirectArrayHistogram.java
View File

@@ -2,7 +2,9 @@ package org.HdrHistogram;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
public class DirectArrayHistogram extends AbstractHistogram implements Histogramer{
long totalCount;
@@ -172,6 +174,35 @@ public class DirectArrayHistogram extends AbstractHistogram implements Histogram
return percentiles;
}
@Override
public Map<String, Object> describe() {
long min = getMinValue();
long max = getMaxValue(); // max = this.maxValue;
long count = getTotalCount();
double mean = getMean();
long sum = (long) (mean * count);
mean = Math.round(mean * 100.0) / 100.0;
long p25 = getValueAtPercentile(25);
long p50 = getValueAtPercentile(50);
long p75 = getValueAtPercentile(75);
long p90 = getValueAtPercentile(90);
long p95 = getValueAtPercentile(95);
long p99 = getValueAtPercentile(99);
Map<String, Object> rst = new LinkedHashMap<>();
rst.put("count", count);
rst.put("mean", mean);
rst.put("sum", sum);
rst.put("min", min);
rst.put("p25", p25);
rst.put("p50", p50);
rst.put("p75", p75);
rst.put("p90", p90);
rst.put("p95", p95);
rst.put("p99", p99);
rst.put("max", max);
return rst;
}
@Override
public Histogramer resetHistogram() {
throw new UnsupportedOperationException("unsupported method");

6
druid-hdrhistogram/src/main/java/org/HdrHistogram/DirectMapHistogram.java
View File

@@ -3,6 +3,7 @@ package org.HdrHistogram;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
import java.util.List;
import java.util.Map;
import java.util.zip.DataFormatException;
import java.util.zip.Inflater;
@@ -446,6 +447,11 @@ public class DirectMapHistogram implements Histogramer{
throw new UnsupportedOperationException("unsupported method");
}
@Override
public Map<String, Object> describe() {
throw new UnsupportedOperationException("unsupported method");
}
@Override
public Histogramer resetHistogram() {
throw new UnsupportedOperationException("unsupported method");

5
druid-hdrhistogram/src/main/java/org/HdrHistogram/HistogramSketch.java
View File

@@ -2,6 +2,7 @@ package org.HdrHistogram;
import java.nio.ByteBuffer;
import java.util.List;
import java.util.Map;
public class HistogramSketch {
public Histogramer hisImpl = null;
@@ -59,6 +60,10 @@ public class HistogramSketch {
return hisImpl.percentileList(percentileTicksPerHalfDistance);
}
public Map<String, Object> describe(){
return hisImpl.describe();
}
public static final int getUpdatableSerializationBytes(long lowestDiscernibleValue, long highestTrackableValue, int numberOfSignificantValueDigits){
return DirectArrayHistogram.getUpdatableSerializationBytes(lowestDiscernibleValue, highestTrackableValue, numberOfSignificantValueDigits);
}

3
druid-hdrhistogram/src/main/java/org/HdrHistogram/Histogramer.java
View File

@@ -2,6 +2,7 @@ package org.HdrHistogram;
import java.nio.ByteBuffer;
import java.util.List;
import java.util.Map;
public interface Histogramer {
long getTotalCount();
@@ -14,6 +15,8 @@ public interface Histogramer {
List<Percentile> percentileList(int percentileTicksPerHalfDistance);
Map<String, Object> describe();
Histogramer resetHistogram();
Histogramer merge(Histogramer histogram);

9
druid-hdrhistogram/src/main/java/org/HdrHistogram/Percentile.java
View File

@@ -38,4 +38,13 @@ public class Percentile {
public void setPercentile(double percentile) {
this.percentile = percentile;
}
@Override
public String toString() {
return "Percentile{" +
"value=" + value +
", count=" + count +
", percentile=" + percentile +
'}';
}
}

2
druid-hdrhistogram/src/main/java/org/apache/druid/query/aggregation/sketch/HdrHistogram/HdrHistogramAggregatorFactory.java
View File

@@ -21,7 +21,7 @@ public class HdrHistogramAggregatorFactory extends AggregatorFactory {
public static final long DEFAULT_HIGHEST = 2;
public static final int DEFAULT_SIGNIFICANT = 1;
public static final boolean DEFAULT_AUTO_RESIZE = true;
public static final long BUFFER_AUTO_RESIZE_HIGHEST = 100000000L * 1000000L;
public static final long BUFFER_AUTO_RESIZE_HIGHEST = 100000000L * 100L;
public static final Comparator<HistogramSketch> COMPARATOR =
Comparator.nullsFirst(Comparator.comparingLong(HistogramSketch::getTotalCount));

4
druid-hdrhistogram/src/main/java/org/apache/druid/query/aggregation/sketch/HdrHistogram/HdrHistogramMergeBufferAggregator.java
View File

@@ -37,7 +37,7 @@ public class HdrHistogramMergeBufferAggregator implements BufferAggregator {
this.numberOfSignificantValueDigits = numberOfSignificantValueDigits;
this.autoResize = autoResize;
this.size = size;
LOG.error("HdrHistogramMergeBufferAggregator gene:" + Thread.currentThread().getName() + "-" + Thread.currentThread().getId());
//LOG.error("HdrHistogramMergeBufferAggregator gene:" + Thread.currentThread().getName() + "-" + Thread.currentThread().getId());
}
@Override
@@ -83,7 +83,7 @@ public class HdrHistogramMergeBufferAggregator implements BufferAggregator {
@Nullable
@Override
public synchronized HistogramSketch get(ByteBuffer buf, int position) {
LOG.error("HdrHistogramMergeBufferAggregator get:" + 0 + "-" + Thread.currentThread().getId() + "-" + this);
//LOG.error("HdrHistogramMergeBufferAggregator get:" + 0 + "-" + Thread.currentThread().getId() + "-" + this);
HistogramUnion union = histograms.get(buf).get(position);
//return histogram.copy();
return union.getResult().copy();

13
druid-hdrhistogram/src/main/java/org/apache/druid/query/aggregation/sketch/HdrHistogram/HdrHistogramModule.java
View File

@@ -9,10 +9,7 @@ import com.google.common.annotations.VisibleForTesting;
import com.google.inject.Binder;
import org.HdrHistogram.HistogramSketch;
import org.apache.druid.initialization.DruidModule;
import org.apache.druid.query.aggregation.sketch.HdrHistogram.sql.HdrHistogramObjectSqlAggregator;
import org.apache.druid.query.aggregation.sketch.HdrHistogram.sql.HdrHistogramPercentilesOperatorConversion;
import org.apache.druid.query.aggregation.sketch.HdrHistogram.sql.HdrHistogramQuantileSqlAggregator;
import org.apache.druid.query.aggregation.sketch.HdrHistogram.sql.HdrHistogramQuantilesOperatorConversion;
import org.apache.druid.query.aggregation.sketch.HdrHistogram.sql.*;
import org.apache.druid.segment.column.ColumnType;
import org.apache.druid.segment.serde.ComplexMetrics;
import org.apache.druid.sql.guice.SqlBindings;
@@ -27,6 +24,8 @@ public class HdrHistogramModule implements DruidModule {
public static final byte QUANTILES_HDRHISTOGRAM_TO_QUANTILE_CACHE_TYPE_ID = 0x03;
public static final byte QUANTILES_HDRHISTOGRAM_TO_QUANTILES_CACHE_TYPE_ID = 0x04;
public static final byte QUANTILES_HDRHISTOGRAM_TO_PERCENTILES_CACHE_TYPE_ID = 0x05;
public static final byte QUANTILES_HDRHISTOGRAM_TO_DESCRIBE_CACHE_TYPE_ID = 0x06;
public static final byte QUANTILES_HDRHISTOGRAM_TO_PERCENTILES_DESCRIBE_CACHE_TYPE_ID = 0x07;
public static final String HDRHISTOGRAM_TYPE_NAME = "HdrHistogramSketch";
public static final ColumnType TYPE = ColumnType.ofComplex(HDRHISTOGRAM_TYPE_NAME);
@@ -50,6 +49,8 @@ public class HdrHistogramModule implements DruidModule {
SqlBindings.addOperatorConversion(binder, HdrHistogramQuantilesOperatorConversion.class);
SqlBindings.addOperatorConversion(binder, HdrHistogramPercentilesOperatorConversion.class);
SqlBindings.addOperatorConversion(binder, HdrHistogramDescribeOperatorConversion.class);
SqlBindings.addOperatorConversion(binder, HdrHistogramPercentilesDescribeOperatorConversion.class);
}
@Override
@@ -61,7 +62,9 @@ public class HdrHistogramModule implements DruidModule {
new NamedType(HdrHistogramMergeAggregatorFactory.class, "HdrHistogramSketchMerge"),
new NamedType(HdrHistogramToQuantilePostAggregator.class, "HdrHistogramSketchToQuantile"),
new NamedType(HdrHistogramToQuantilesPostAggregator.class, "HdrHistogramSketchToQuantiles"),
new NamedType(HdrHistogramToPercentilesPostAggregator.class, "HdrHistogramSketchToPercentiles")
new NamedType(HdrHistogramToPercentilesPostAggregator.class, "HdrHistogramSketchToPercentiles"),
new NamedType(HdrHistogramToDescribePostAggregator.class, "HdrHistogramSketchToDescribe"),
new NamedType(HdrHistogramToPercentilesDescribePostAggregator.class, "HdrHistogramSketchToPercentilesDescription")
).addSerializer(HistogramSketch.class, new HistogramJsonSerializer())
);
}

108
druid-hdrhistogram/src/main/java/org/apache/druid/query/aggregation/sketch/HdrHistogram/HdrHistogramToDescribePostAggregator.java Normal file
View File

@@ -0,0 +1,108 @@
package org.apache.druid.query.aggregation.sketch.HdrHistogram;
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.common.collect.Sets;
import org.HdrHistogram.HistogramSketch;
import org.apache.druid.java.util.common.IAE;
import org.apache.druid.query.aggregation.AggregatorFactory;
import org.apache.druid.query.aggregation.PostAggregator;
import org.apache.druid.query.cache.CacheKeyBuilder;
import org.apache.druid.segment.ColumnInspector;
import org.apache.druid.segment.column.ColumnType;
import javax.annotation.Nullable;
import java.util.*;
public class HdrHistogramToDescribePostAggregator implements PostAggregator {
private final String name;
private final String fieldName;
@JsonCreator
public HdrHistogramToDescribePostAggregator(
@JsonProperty("name") String name,
@JsonProperty("fieldName") String fieldName
){
this.name = name;
this.fieldName = fieldName;
}
@Override
public ColumnType getType(ColumnInspector signature){
return ColumnType.STRING;
}
@Override
@JsonProperty
public String getName() {
return name;
}
@JsonProperty
public String getFieldName() {
return fieldName;
}
@Nullable
@Override
public Object compute(Map<String, Object> values) {
HistogramSketch histogram = (HistogramSketch) values.get(fieldName);
if(histogram == null){
return "{}"; //"[]"
}
return HdrHistogramModule.toJson(histogram.describe());
}
@Override
public Comparator<double[]> getComparator()
{
throw new IAE("Comparing arrays of quantiles is not supported");
}
@Override
public Set<String> getDependentFields()
{
return Sets.newHashSet(fieldName);
}
@Override
public PostAggregator decorate(Map<String, AggregatorFactory> aggregators) {
return this;
}
@Override
public byte[] getCacheKey() {
CacheKeyBuilder builder = new CacheKeyBuilder(HdrHistogramModule.CACHE_TYPE_ID_OFFSET).appendByte(HdrHistogramModule.QUANTILES_HDRHISTOGRAM_TO_DESCRIBE_CACHE_TYPE_ID)
.appendString(fieldName);
return builder.build();
}
@Override
public boolean equals(Object o) {
if (this == o) {
return true;
}
if (o == null || getClass() != o.getClass()) {
return false;
}
HdrHistogramToDescribePostAggregator that = (HdrHistogramToDescribePostAggregator) o;
return name.equals(that.name) &&
fieldName.equals(that.fieldName);
}
@Override
public int hashCode() {
return Objects.hash(name, fieldName);
}
@Override
public String toString() {
return "HdrHistogramToDescribePostAggregator{" +
"name='" + name + '\'' +
", fieldName='" + fieldName + '\'' +
'}';
}
}

125
druid-hdrhistogram/src/main/java/org/apache/druid/query/aggregation/sketch/HdrHistogram/HdrHistogramToPercentilesDescribePostAggregator.java Normal file
View File

@@ -0,0 +1,125 @@
package org.apache.druid.query.aggregation.sketch.HdrHistogram;
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.common.collect.Sets;
import org.HdrHistogram.HistogramSketch;
import org.HdrHistogram.Percentile;
import org.apache.druid.java.util.common.IAE;
import org.apache.druid.query.aggregation.AggregatorFactory;
import org.apache.druid.query.aggregation.PostAggregator;
import org.apache.druid.query.cache.CacheKeyBuilder;
import org.apache.druid.segment.ColumnInspector;
import org.apache.druid.segment.column.ColumnType;
import javax.annotation.Nullable;
import java.util.*;
public class HdrHistogramToPercentilesDescribePostAggregator implements PostAggregator {
private final String name;
private final String fieldName;
private final int percentileTicksPerHalfDistance;
@JsonCreator
public HdrHistogramToPercentilesDescribePostAggregator(
@JsonProperty("name") String name,
@JsonProperty("fieldName") String fieldName,
@JsonProperty("percentileTicksPerHalfDistance") int percentileTicksPerHalfDistance
){
this.name = name;
this.fieldName = fieldName;
this.percentileTicksPerHalfDistance = percentileTicksPerHalfDistance;
}
@Override
public ColumnType getType(ColumnInspector signature){
return ColumnType.STRING;
}
@Override
@JsonProperty
public String getName() {
return name;
}
@JsonProperty
public String getFieldName() {
return fieldName;
}
@JsonProperty
public int getPercentileTicksPerHalfDistance() {
return percentileTicksPerHalfDistance;
}
@Nullable
@Override
public Object compute(Map<String, Object> values) {
HistogramSketch histogram = (HistogramSketch) values.get(fieldName);
if(histogram == null){
return "{\"percentiles\":[],\"describe\":{}}";
}
List<Percentile> percentiles = histogram.percentileList(percentileTicksPerHalfDistance);
Map<String, Object> describe = histogram.describe();
Map<String, Object> rst = new LinkedHashMap<>();
rst.put("percentiles", percentiles);
rst.put("description", describe);
return HdrHistogramModule.toJson(rst);
}
@Override
public Comparator<double[]> getComparator()
{
throw new IAE("Comparing object is not supported");
}
@Override
public Set<String> getDependentFields()
{
return Sets.newHashSet(fieldName);
}
@Override
public PostAggregator decorate(Map<String, AggregatorFactory> aggregators) {
return this;
}
@Override
public byte[] getCacheKey() {
CacheKeyBuilder builder = new CacheKeyBuilder(HdrHistogramModule.CACHE_TYPE_ID_OFFSET).appendByte(HdrHistogramModule.QUANTILES_HDRHISTOGRAM_TO_PERCENTILES_DESCRIBE_CACHE_TYPE_ID)
.appendString(fieldName);
builder.appendInt(percentileTicksPerHalfDistance);
return builder.build();
}
@Override
public boolean equals(Object o) {
if (this == o) {
return true;
}
if (o == null || getClass() != o.getClass()) {
return false;
}
HdrHistogramToPercentilesDescribePostAggregator that = (HdrHistogramToPercentilesDescribePostAggregator) o;
return percentileTicksPerHalfDistance == that.percentileTicksPerHalfDistance &&
name.equals(that.name) &&
fieldName.equals(that.fieldName);
}
@Override
public int hashCode() {
return Objects.hash(name, fieldName, percentileTicksPerHalfDistance);
}
@Override
public String toString() {
return "HdrHistogramToPercentilesDescribePostAggregator{" +
"name='" + name + '\'' +
", fieldName='" + fieldName + '\'' +
", probabilitys=" + percentileTicksPerHalfDistance +
'}';
}
}

77
druid-hdrhistogram/src/main/java/org/apache/druid/query/aggregation/sketch/HdrHistogram/sql/HdrHistogramDescribeOperatorConversion.java Normal file
View File

@@ -0,0 +1,77 @@
package org.apache.druid.query.aggregation.sketch.HdrHistogram.sql;
import org.apache.calcite.rex.RexCall;
import org.apache.calcite.rex.RexNode;
import org.apache.calcite.sql.SqlFunction;
import org.apache.calcite.sql.SqlOperator;
import org.apache.calcite.sql.type.ReturnTypes;
import org.apache.calcite.sql.type.SqlTypeFamily;
import org.apache.calcite.sql.type.SqlTypeName;
import org.apache.druid.java.util.common.StringUtils;
import org.apache.druid.query.aggregation.PostAggregator;
import org.apache.druid.query.aggregation.post.FieldAccessPostAggregator;
import org.apache.druid.query.aggregation.sketch.HdrHistogram.HdrHistogramToDescribePostAggregator;
import org.apache.druid.segment.column.RowSignature;
import org.apache.druid.sql.calcite.expression.DruidExpression;
import org.apache.druid.sql.calcite.expression.OperatorConversions;
import org.apache.druid.sql.calcite.expression.PostAggregatorVisitor;
import org.apache.druid.sql.calcite.expression.SqlOperatorConversion;
import org.apache.druid.sql.calcite.planner.PlannerContext;
import javax.annotation.Nullable;
import java.util.List;
public class HdrHistogramDescribeOperatorConversion implements SqlOperatorConversion {
private static final String FUNCTION_NAME = "HDR_DESCRIBE";
private static final SqlFunction SQL_FUNCTION = OperatorConversions
.operatorBuilder(StringUtils.toUpperCase(FUNCTION_NAME))
.operandTypes(SqlTypeFamily.ANY)
.requiredOperands(1)
.returnTypeInference(ReturnTypes.explicit(SqlTypeName.VARCHAR))
.build();
@Override
public SqlOperator calciteOperator()
{
return SQL_FUNCTION;
}
@Override
public DruidExpression toDruidExpression(
PlannerContext plannerContext,
RowSignature rowSignature,
RexNode rexNode
)
{
return null;
}
@Nullable
@Override
public PostAggregator toPostAggregator(
PlannerContext plannerContext,
RowSignature rowSignature,
RexNode rexNode,
PostAggregatorVisitor postAggregatorVisitor
)
{
final List<RexNode> operands = ((RexCall) rexNode).getOperands();
final PostAggregator postAgg = OperatorConversions.toPostAggregator(
plannerContext,
rowSignature,
operands.get(0),
postAggregatorVisitor,
true
);
if (postAgg == null) {
return null;
}
return new HdrHistogramToDescribePostAggregator(
postAggregatorVisitor.getOutputNamePrefix() + postAggregatorVisitor.getAndIncrementCounter(),
((FieldAccessPostAggregator)postAgg).getFieldName()
);
}
}

88
druid-hdrhistogram/src/main/java/org/apache/druid/query/aggregation/sketch/HdrHistogram/sql/HdrHistogramPercentilesDescribeOperatorConversion.java Normal file
View File

@@ -0,0 +1,88 @@
package org.apache.druid.query.aggregation.sketch.HdrHistogram.sql;
import org.apache.calcite.rex.RexCall;
import org.apache.calcite.rex.RexLiteral;
import org.apache.calcite.rex.RexNode;
import org.apache.calcite.sql.SqlFunction;
import org.apache.calcite.sql.SqlKind;
import org.apache.calcite.sql.SqlOperator;
import org.apache.calcite.sql.type.ReturnTypes;
import org.apache.calcite.sql.type.SqlTypeFamily;
import org.apache.calcite.sql.type.SqlTypeName;
import org.apache.druid.java.util.common.StringUtils;
import org.apache.druid.query.aggregation.PostAggregator;
import org.apache.druid.query.aggregation.post.FieldAccessPostAggregator;
import org.apache.druid.query.aggregation.sketch.HdrHistogram.HdrHistogramToPercentilesDescribePostAggregator;
import org.apache.druid.segment.column.RowSignature;
import org.apache.druid.sql.calcite.expression.DruidExpression;
import org.apache.druid.sql.calcite.expression.OperatorConversions;
import org.apache.druid.sql.calcite.expression.PostAggregatorVisitor;
import org.apache.druid.sql.calcite.expression.SqlOperatorConversion;
import org.apache.druid.sql.calcite.planner.PlannerContext;
import javax.annotation.Nullable;
import java.util.List;
public class HdrHistogramPercentilesDescribeOperatorConversion implements SqlOperatorConversion {
private static final String FUNCTION_NAME = "HDR_GET_PERCENTILES_DESCRIPTION";
private static final SqlFunction SQL_FUNCTION = OperatorConversions
.operatorBuilder(StringUtils.toUpperCase(FUNCTION_NAME))
.operandTypes(SqlTypeFamily.ANY, SqlTypeFamily.NUMERIC)
.requiredOperands(1)
.returnTypeInference(ReturnTypes.explicit(SqlTypeName.VARCHAR))
.build();
@Override
public SqlOperator calciteOperator()
{
return SQL_FUNCTION;
}
@Override
public DruidExpression toDruidExpression(
PlannerContext plannerContext,
RowSignature rowSignature,
RexNode rexNode
)
{
return null;
}
@Nullable
@Override
public PostAggregator toPostAggregator(
PlannerContext plannerContext,
RowSignature rowSignature,
RexNode rexNode,
PostAggregatorVisitor postAggregatorVisitor
)
{
final List<RexNode> operands = ((RexCall) rexNode).getOperands();
final PostAggregator postAgg = OperatorConversions.toPostAggregator(
plannerContext,
rowSignature,
operands.get(0),
postAggregatorVisitor,
true
);
if (postAgg == null) {
return null;
}
int percentileTicksPerHalfDistance = 5;
if (operands.size() == 2) {
if (!operands.get(1).isA(SqlKind.LITERAL)) {
return null;
}
percentileTicksPerHalfDistance = RexLiteral.intValue(operands.get(1));
}
return new HdrHistogramToPercentilesDescribePostAggregator(
postAggregatorVisitor.getOutputNamePrefix() + postAggregatorVisitor.getAndIncrementCounter(),
((FieldAccessPostAggregator)postAgg).getFieldName(),
percentileTicksPerHalfDistance
);
}
}

79
druid-hdrhistogram/src/test/java/org/apache/druid/query/aggregation/sketch/HdrHistogram/HistogramSketchTest.java Normal file
View File

@@ -0,0 +1,79 @@
package org.apache.druid.query.aggregation.sketch.HdrHistogram;
import org.HdrHistogram.DirectArrayHistogram;
import org.HdrHistogram.HistogramSketch;
import org.HdrHistogram.Histogramer;
import org.HdrHistogram.Percentile;
import org.apache.commons.lang3.StringUtils;
import org.junit.Test;
import java.io.BufferedWriter;
import java.nio.ByteBuffer;
import java.nio.charset.StandardCharsets;
import java.nio.file.FileSystems;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.Random;
import java.util.concurrent.ThreadLocalRandom;
public class HistogramSketchTest {
@Test
public void describeTest() throws Exception{
DirectArrayHistogram histogram = new DirectArrayHistogram(1, 1000000, 3,
ByteBuffer.allocate(HistogramSketch.getUpdatableSerializationBytes(1, 1000000, 3)));
System.out.println(histogram.describe());
for (int i = 0; i < 10000; i++) {
histogram.recordValue(i);
}
System.out.println(histogram.describe());
for (Percentile percentile : histogram.percentileList(100)) {
System.out.println(percentile);
}
}
@Test
public void describeTest1() throws Exception{
HistogramSketch histogram = new HistogramSketch(1);
System.out.println(histogram.describe());
for (int i = 0; i < 10000; i++) {
histogram.recordValue(i);
}
System.out.println(histogram.describe());
for (Percentile percentile : histogram.percentileList(100)) {
System.out.println(percentile);
}
System.out.println(StringUtils.repeat('#', 100));
histogram = new HistogramSketch(1);
for (int i = 0; i < 10000; i++) {
histogram.recordValue(ThreadLocalRandom.current().nextLong(100000));
}
System.out.println(histogram.describe());
for (Percentile percentile : histogram.percentileList(100)) {
System.out.println(percentile);
}
}
@Test
public void describeTest3() throws Exception{
HistogramSketch histogram = new HistogramSketch(3);
System.out.println(histogram.describe());
for (int i = 0; i < 10000; i++) {
histogram.recordValue(i);
}
System.out.println(histogram.describe());
for (Percentile percentile : histogram.percentileList(100)) {
System.out.println(percentile);
}
System.out.println(StringUtils.repeat('#', 100));
histogram = new HistogramSketch(3);
for (int i = 0; i < 10000; i++) {
histogram.recordValue(ThreadLocalRandom.current().nextLong(100000));
}
System.out.println(histogram.describe());
for (Percentile percentile : histogram.percentileList(100)) {
System.out.println(percentile);
}
}
}

24
druid-hdrhistogram/src/test/java/org/apache/druid/query/aggregation/sketch/HdrHistogram/sql/HdrHistogramQuantileSqlAggregatorTest.java
View File

@@ -219,6 +219,30 @@ public class HdrHistogramQuantileSqlAggregatorTest extends BaseCalciteQueryTest
}
}
@Test
public void testSqlDESCRIBE() throws Exception {
String sql = "select HDR_GET_QUANTILES(HDR_HISTOGRAM(m1, 1, 100, 2), 0, 0.25, 0.5, 0.75, 1) a, HDR_DESCRIBE(HDR_HISTOGRAM(m1, 1, 100, 2)) b, HDR_DESCRIBE(HDR_HISTOGRAM(hist_m1, 1, 100, 2)) c from druid.foo";
QueryTestBuilder builder = testBuilder().sql(sql).skipVectorize();
builder.run();
QueryTestRunner.QueryResults queryResults = builder.results();
List<Object[]> results = queryResults.results;
for (Object[] result : results) {
System.out.println(Arrays.toString(result));
}
}
@Test
public void testSqlDESCRIBE2() throws Exception {
String sql = "select HDR_GET_QUANTILES(HDR_HISTOGRAM(m1, 1, 100, 2), 0, 0.25, 0.5, 0.75, 1) a, HDR_GET_PERCENTILES_DESCRIPTION(HDR_HISTOGRAM(m1, 1, 100, 2)) b, HDR_GET_PERCENTILES_DESCRIPTION(HDR_HISTOGRAM(hist_m1, 1, 100, 2)) c from druid.foo";
QueryTestBuilder builder = testBuilder().sql(sql).skipVectorize();
builder.run();
QueryTestRunner.QueryResults queryResults = builder.results();
List<Object[]> results = queryResults.results;
for (Object[] result : results) {
System.out.println(Arrays.toString(result));
}
}
@Test
public void testSqlQuery() throws Exception {
String[] columns = new String[]{"__time", "dim1", "dim2", "dim3", "cnt", "hist_m1", "m1"};