druid-hdrhistogram druid版本升级到26.0.0
This commit is contained in:
@@ -5,7 +5,7 @@
|
|||||||
<modelVersion>4.0.0</modelVersion>
|
<modelVersion>4.0.0</modelVersion>
|
||||||
|
|
||||||
<groupId>org.apache.druid.extensions</groupId>
|
<groupId>org.apache.druid.extensions</groupId>
|
||||||
<artifactId>druid-hdrhistogram_0.18.1</artifactId>
|
<artifactId>druid-hdrhistogram_26.0.0</artifactId>
|
||||||
<name>druid-hdrhistogram</name>
|
<name>druid-hdrhistogram</name>
|
||||||
<version>1.0-SNAPSHOT</version>
|
<version>1.0-SNAPSHOT</version>
|
||||||
|
|
||||||
@@ -14,7 +14,7 @@
|
|||||||
<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
|
<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
|
||||||
<maven.compiler.source>1.8</maven.compiler.source>
|
<maven.compiler.source>1.8</maven.compiler.source>
|
||||||
<maven.compiler.target>1.8</maven.compiler.target>
|
<maven.compiler.target>1.8</maven.compiler.target>
|
||||||
<druid.version>0.18.1</druid.version>
|
<druid.version>26.0.0</druid.version>
|
||||||
</properties>
|
</properties>
|
||||||
|
|
||||||
<dependencies>
|
<dependencies>
|
||||||
@@ -45,6 +45,13 @@
|
|||||||
</dependency>
|
</dependency>
|
||||||
|
|
||||||
<!-- Tests -->
|
<!-- Tests -->
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.easymock</groupId>
|
||||||
|
<artifactId>easymock</artifactId>
|
||||||
|
<version>4.3</version>
|
||||||
|
<scope>test</scope>
|
||||||
|
</dependency>
|
||||||
|
|
||||||
<dependency>
|
<dependency>
|
||||||
<groupId>org.apache.druid</groupId>
|
<groupId>org.apache.druid</groupId>
|
||||||
<artifactId>druid-processing</artifactId>
|
<artifactId>druid-processing</artifactId>
|
||||||
@@ -54,9 +61,17 @@
|
|||||||
</dependency>
|
</dependency>
|
||||||
<dependency>
|
<dependency>
|
||||||
<groupId>org.apache.druid</groupId>
|
<groupId>org.apache.druid</groupId>
|
||||||
<artifactId>druid-benchmarks</artifactId>
|
<artifactId>druid-server</artifactId>
|
||||||
<version>${druid.version}</version>
|
<version>${druid.version}</version>
|
||||||
<scope>test</scope>
|
<scope>test</scope>
|
||||||
|
<type>test-jar</type>
|
||||||
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.apache.druid</groupId>
|
||||||
|
<artifactId>druid-sql</artifactId>
|
||||||
|
<version>${druid.version}</version>
|
||||||
|
<type>test-jar</type>
|
||||||
|
<scope>test</scope>
|
||||||
</dependency>
|
</dependency>
|
||||||
<dependency>
|
<dependency>
|
||||||
<groupId>junit</groupId>
|
<groupId>junit</groupId>
|
||||||
|
|||||||
@@ -1,8 +1,6 @@
|
|||||||
package org.apache.druid.query.aggregation.sketch.HdrHistogram;
|
package org.apache.druid.query.aggregation.sketch.HdrHistogram;
|
||||||
|
|
||||||
import com.fasterxml.jackson.annotation.JsonProperty;
|
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||||
import org.HdrHistogram.DirectHistogram;
|
|
||||||
import org.HdrHistogram.Histogram;
|
|
||||||
import org.HdrHistogram.HistogramSketch;
|
import org.HdrHistogram.HistogramSketch;
|
||||||
import org.HdrHistogram.HistogramUnion;
|
import org.HdrHistogram.HistogramUnion;
|
||||||
import org.apache.druid.java.util.common.IAE;
|
import org.apache.druid.java.util.common.IAE;
|
||||||
@@ -10,6 +8,7 @@ import org.apache.druid.query.aggregation.*;
|
|||||||
import org.apache.druid.query.cache.CacheKeyBuilder;
|
import org.apache.druid.query.cache.CacheKeyBuilder;
|
||||||
import org.apache.druid.segment.ColumnSelectorFactory;
|
import org.apache.druid.segment.ColumnSelectorFactory;
|
||||||
import org.apache.druid.segment.ColumnValueSelector;
|
import org.apache.druid.segment.ColumnValueSelector;
|
||||||
|
import org.apache.druid.segment.column.ColumnType;
|
||||||
|
|
||||||
import javax.annotation.Nullable;
|
import javax.annotation.Nullable;
|
||||||
import java.util.Collections;
|
import java.util.Collections;
|
||||||
@@ -208,15 +207,27 @@ public class HdrHistogramAggregatorFactory extends AggregatorFactory {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public AggregatorFactory withName(String newName) {
|
||||||
|
return new HdrHistogramAggregatorFactory(newName, fieldName, lowestDiscernibleValue, highestTrackableValue, numberOfSignificantValueDigits, autoResize);
|
||||||
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public Object deserialize(Object object) {
|
public Object deserialize(Object object) {
|
||||||
return HistogramUtils.deserializeHistogram(object);
|
return HistogramUtils.deserializeHistogram(object);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public ColumnType getResultType() {
|
||||||
|
//return ColumnType.LONG;
|
||||||
|
return getIntermediateType();
|
||||||
|
}
|
||||||
|
|
||||||
@Nullable
|
@Nullable
|
||||||
@Override
|
@Override
|
||||||
public Object finalizeComputation(@Nullable Object object) {
|
public Object finalizeComputation(@Nullable Object object) {
|
||||||
return object == null ? null : ((HistogramSketch) object).getTotalCount();
|
//return object == null ? null : ((HistogramSketch) object).getTotalCount();
|
||||||
|
return object;
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
@@ -250,9 +261,16 @@ public class HdrHistogramAggregatorFactory extends AggregatorFactory {
|
|||||||
return autoResize;
|
return autoResize;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/*
|
||||||
|
没这个方法了, 新版本需要实现getIntermediateType方法
|
||||||
@Override
|
@Override
|
||||||
public String getTypeName() {
|
public String getTypeName() {
|
||||||
return HdrHistogramModule.HDRHISTOGRAM_TYPE_NAME;
|
return HdrHistogramModule.HDRHISTOGRAM_TYPE_NAME;
|
||||||
|
}*/
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public ColumnType getIntermediateType() {
|
||||||
|
return HdrHistogramModule.TYPE;
|
||||||
}
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
|
|||||||
@@ -1,9 +1,9 @@
|
|||||||
package org.apache.druid.query.aggregation.sketch.HdrHistogram;
|
package org.apache.druid.query.aggregation.sketch.HdrHistogram;
|
||||||
|
|
||||||
import com.fasterxml.jackson.annotation.JsonProperty;
|
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||||
import org.HdrHistogram.Histogram;
|
|
||||||
import org.HdrHistogram.HistogramSketch;
|
import org.HdrHistogram.HistogramSketch;
|
||||||
import org.apache.druid.query.aggregation.Aggregator;
|
import org.apache.druid.query.aggregation.Aggregator;
|
||||||
|
import org.apache.druid.query.aggregation.AggregatorFactory;
|
||||||
import org.apache.druid.query.aggregation.BufferAggregator;
|
import org.apache.druid.query.aggregation.BufferAggregator;
|
||||||
import org.apache.druid.query.cache.CacheKeyBuilder;
|
import org.apache.druid.query.cache.CacheKeyBuilder;
|
||||||
import org.apache.druid.segment.ColumnSelectorFactory;
|
import org.apache.druid.segment.ColumnSelectorFactory;
|
||||||
@@ -48,6 +48,11 @@ public class HdrHistogramMergeAggregatorFactory extends HdrHistogramAggregatorFa
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public AggregatorFactory withName(String newName) {
|
||||||
|
return new HdrHistogramMergeAggregatorFactory(newName, fieldName, lowestDiscernibleValue, highestTrackableValue, numberOfSignificantValueDigits, autoResize);
|
||||||
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public byte[] getCacheKey() {
|
public byte[] getCacheKey() {
|
||||||
return new CacheKeyBuilder(HdrHistogramModule.CACHE_TYPE_ID_OFFSET).appendByte(HdrHistogramModule.QUANTILES_HDRHISTOGRAM_MERGE_CACHE_TYPE_ID)
|
return new CacheKeyBuilder(HdrHistogramModule.CACHE_TYPE_ID_OFFSET).appendByte(HdrHistogramModule.QUANTILES_HDRHISTOGRAM_MERGE_CACHE_TYPE_ID)
|
||||||
|
|||||||
@@ -7,13 +7,13 @@ import com.fasterxml.jackson.databind.jsontype.NamedType;
|
|||||||
import com.fasterxml.jackson.databind.module.SimpleModule;
|
import com.fasterxml.jackson.databind.module.SimpleModule;
|
||||||
import com.google.common.annotations.VisibleForTesting;
|
import com.google.common.annotations.VisibleForTesting;
|
||||||
import com.google.inject.Binder;
|
import com.google.inject.Binder;
|
||||||
import org.HdrHistogram.Histogram;
|
|
||||||
import org.HdrHistogram.HistogramSketch;
|
import org.HdrHistogram.HistogramSketch;
|
||||||
import org.apache.druid.initialization.DruidModule;
|
import org.apache.druid.initialization.DruidModule;
|
||||||
import org.apache.druid.query.aggregation.sketch.HdrHistogram.sql.HdrHistogramObjectSqlAggregator;
|
import org.apache.druid.query.aggregation.sketch.HdrHistogram.sql.HdrHistogramObjectSqlAggregator;
|
||||||
import org.apache.druid.query.aggregation.sketch.HdrHistogram.sql.HdrHistogramPercentilesOperatorConversion;
|
import org.apache.druid.query.aggregation.sketch.HdrHistogram.sql.HdrHistogramPercentilesOperatorConversion;
|
||||||
import org.apache.druid.query.aggregation.sketch.HdrHistogram.sql.HdrHistogramQuantileSqlAggregator;
|
import org.apache.druid.query.aggregation.sketch.HdrHistogram.sql.HdrHistogramQuantileSqlAggregator;
|
||||||
import org.apache.druid.query.aggregation.sketch.HdrHistogram.sql.HdrHistogramQuantilesOperatorConversion;
|
import org.apache.druid.query.aggregation.sketch.HdrHistogram.sql.HdrHistogramQuantilesOperatorConversion;
|
||||||
|
import org.apache.druid.segment.column.ColumnType;
|
||||||
import org.apache.druid.segment.serde.ComplexMetrics;
|
import org.apache.druid.segment.serde.ComplexMetrics;
|
||||||
import org.apache.druid.sql.guice.SqlBindings;
|
import org.apache.druid.sql.guice.SqlBindings;
|
||||||
|
|
||||||
@@ -29,6 +29,7 @@ public class HdrHistogramModule implements DruidModule {
|
|||||||
public static final byte QUANTILES_HDRHISTOGRAM_TO_PERCENTILES_CACHE_TYPE_ID = 0x05;
|
public static final byte QUANTILES_HDRHISTOGRAM_TO_PERCENTILES_CACHE_TYPE_ID = 0x05;
|
||||||
|
|
||||||
public static final String HDRHISTOGRAM_TYPE_NAME = "HdrHistogramSketch";
|
public static final String HDRHISTOGRAM_TYPE_NAME = "HdrHistogramSketch";
|
||||||
|
public static final ColumnType TYPE = ColumnType.ofComplex(HDRHISTOGRAM_TYPE_NAME);
|
||||||
|
|
||||||
public static final ObjectMapper objectMapper = new ObjectMapper();
|
public static final ObjectMapper objectMapper = new ObjectMapper();
|
||||||
|
|
||||||
|
|||||||
@@ -9,6 +9,8 @@ import org.apache.druid.java.util.common.IAE;
|
|||||||
import org.apache.druid.query.aggregation.AggregatorFactory;
|
import org.apache.druid.query.aggregation.AggregatorFactory;
|
||||||
import org.apache.druid.query.aggregation.PostAggregator;
|
import org.apache.druid.query.aggregation.PostAggregator;
|
||||||
import org.apache.druid.query.cache.CacheKeyBuilder;
|
import org.apache.druid.query.cache.CacheKeyBuilder;
|
||||||
|
import org.apache.druid.segment.ColumnInspector;
|
||||||
|
import org.apache.druid.segment.column.ColumnType;
|
||||||
|
|
||||||
import javax.annotation.Nullable;
|
import javax.annotation.Nullable;
|
||||||
import java.util.*;
|
import java.util.*;
|
||||||
@@ -29,6 +31,11 @@ public class HdrHistogramToPercentilesPostAggregator implements PostAggregator {
|
|||||||
this.percentileTicksPerHalfDistance = percentileTicksPerHalfDistance;
|
this.percentileTicksPerHalfDistance = percentileTicksPerHalfDistance;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public ColumnType getType(ColumnInspector signature){
|
||||||
|
return ColumnType.STRING;
|
||||||
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
@JsonProperty
|
@JsonProperty
|
||||||
public String getName() {
|
public String getName() {
|
||||||
|
|||||||
@@ -9,6 +9,8 @@ import org.apache.druid.java.util.common.IAE;
|
|||||||
import org.apache.druid.query.aggregation.AggregatorFactory;
|
import org.apache.druid.query.aggregation.AggregatorFactory;
|
||||||
import org.apache.druid.query.aggregation.PostAggregator;
|
import org.apache.druid.query.aggregation.PostAggregator;
|
||||||
import org.apache.druid.query.cache.CacheKeyBuilder;
|
import org.apache.druid.query.cache.CacheKeyBuilder;
|
||||||
|
import org.apache.druid.segment.ColumnInspector;
|
||||||
|
import org.apache.druid.segment.column.ColumnType;
|
||||||
|
|
||||||
import javax.annotation.Nullable;
|
import javax.annotation.Nullable;
|
||||||
import java.util.Comparator;
|
import java.util.Comparator;
|
||||||
@@ -36,6 +38,11 @@ public class HdrHistogramToQuantilePostAggregator implements PostAggregator {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public ColumnType getType(ColumnInspector signature){
|
||||||
|
return ColumnType.LONG;
|
||||||
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public Set<String> getDependentFields() {
|
public Set<String> getDependentFields() {
|
||||||
return Sets.newHashSet(fieldName);
|
return Sets.newHashSet(fieldName);
|
||||||
|
|||||||
@@ -9,6 +9,8 @@ import org.apache.druid.java.util.common.IAE;
|
|||||||
import org.apache.druid.query.aggregation.AggregatorFactory;
|
import org.apache.druid.query.aggregation.AggregatorFactory;
|
||||||
import org.apache.druid.query.aggregation.PostAggregator;
|
import org.apache.druid.query.aggregation.PostAggregator;
|
||||||
import org.apache.druid.query.cache.CacheKeyBuilder;
|
import org.apache.druid.query.cache.CacheKeyBuilder;
|
||||||
|
import org.apache.druid.segment.ColumnInspector;
|
||||||
|
import org.apache.druid.segment.column.ColumnType;
|
||||||
|
|
||||||
import javax.annotation.Nullable;
|
import javax.annotation.Nullable;
|
||||||
import java.util.*;
|
import java.util.*;
|
||||||
@@ -29,6 +31,11 @@ public class HdrHistogramToQuantilesPostAggregator implements PostAggregator {
|
|||||||
this.probabilitys = probabilitys;
|
this.probabilitys = probabilitys;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public ColumnType getType(ColumnInspector signature){
|
||||||
|
return ColumnType.LONG_ARRAY;
|
||||||
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
@JsonProperty
|
@JsonProperty
|
||||||
public String getName() {
|
public String getName() {
|
||||||
|
|||||||
@@ -18,6 +18,7 @@ import org.apache.druid.query.aggregation.AggregatorFactory;
|
|||||||
import org.apache.druid.query.aggregation.sketch.HdrHistogram.HdrHistogramAggregatorFactory;
|
import org.apache.druid.query.aggregation.sketch.HdrHistogram.HdrHistogramAggregatorFactory;
|
||||||
import org.apache.druid.query.aggregation.sketch.HdrHistogram.HdrHistogramMergeAggregatorFactory;
|
import org.apache.druid.query.aggregation.sketch.HdrHistogram.HdrHistogramMergeAggregatorFactory;
|
||||||
import org.apache.druid.segment.VirtualColumn;
|
import org.apache.druid.segment.VirtualColumn;
|
||||||
|
import org.apache.druid.segment.column.ColumnType;
|
||||||
import org.apache.druid.segment.column.RowSignature;
|
import org.apache.druid.segment.column.RowSignature;
|
||||||
import org.apache.druid.segment.column.ValueType;
|
import org.apache.druid.segment.column.ValueType;
|
||||||
import org.apache.druid.sql.calcite.aggregation.Aggregation;
|
import org.apache.druid.sql.calcite.aggregation.Aggregation;
|
||||||
@@ -118,11 +119,11 @@ public class HdrHistogramObjectSqlAggregator implements SqlAggregator {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// No existing match found. Create a new one.
|
// No existing match found. Create a new one.
|
||||||
final List<VirtualColumn> virtualColumns = new ArrayList<>();
|
// 新版本删除了final List<VirtualColumn> virtualColumns = new ArrayList<>();
|
||||||
|
|
||||||
if (input.isDirectColumnAccess()) {
|
if (input.isDirectColumnAccess()) {
|
||||||
// 参数是Histogram对象
|
// 参数是Histogram对象
|
||||||
if (rowSignature.getColumnType(input.getDirectColumn()).orElse(null) == ValueType.COMPLEX) {
|
if (rowSignature.getColumnType(input.getDirectColumn()).map(type -> type.is(ValueType.COMPLEX)).orElse(false)) {
|
||||||
aggregatorFactory = new HdrHistogramMergeAggregatorFactory(
|
aggregatorFactory = new HdrHistogramMergeAggregatorFactory(
|
||||||
histogramName,
|
histogramName,
|
||||||
input.getDirectColumn(),
|
input.getDirectColumn(),
|
||||||
@@ -142,12 +143,11 @@ public class HdrHistogramObjectSqlAggregator implements SqlAggregator {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
final VirtualColumn virtualColumn =
|
final String virtualColumnName =
|
||||||
virtualColumnRegistry.getOrCreateVirtualColumnForExpression(plannerContext, input, SqlTypeName.BIGINT);
|
virtualColumnRegistry.getOrCreateVirtualColumnForExpression(input, ColumnType.LONG);
|
||||||
virtualColumns.add(virtualColumn);
|
|
||||||
aggregatorFactory = new HdrHistogramAggregatorFactory(
|
aggregatorFactory = new HdrHistogramAggregatorFactory(
|
||||||
histogramName,
|
histogramName,
|
||||||
virtualColumn.getOutputName(),
|
virtualColumnName,
|
||||||
lowestDiscernibleValue,
|
lowestDiscernibleValue,
|
||||||
highestTrackableValue,
|
highestTrackableValue,
|
||||||
numberOfSignificantValueDigits,
|
numberOfSignificantValueDigits,
|
||||||
@@ -156,7 +156,6 @@ public class HdrHistogramObjectSqlAggregator implements SqlAggregator {
|
|||||||
}
|
}
|
||||||
|
|
||||||
return Aggregation.create(
|
return Aggregation.create(
|
||||||
virtualColumns,
|
|
||||||
ImmutableList.of(aggregatorFactory),
|
ImmutableList.of(aggregatorFactory),
|
||||||
null
|
null
|
||||||
);
|
);
|
||||||
|
|||||||
@@ -14,16 +14,16 @@ import org.apache.druid.query.aggregation.sketch.HdrHistogram.HdrHistogramToPerc
|
|||||||
import org.apache.druid.query.aggregation.PostAggregator;
|
import org.apache.druid.query.aggregation.PostAggregator;
|
||||||
import org.apache.druid.query.aggregation.post.FieldAccessPostAggregator;
|
import org.apache.druid.query.aggregation.post.FieldAccessPostAggregator;
|
||||||
import org.apache.druid.segment.column.RowSignature;
|
import org.apache.druid.segment.column.RowSignature;
|
||||||
import org.apache.druid.sql.calcite.expression.DirectOperatorConversion;
|
|
||||||
import org.apache.druid.sql.calcite.expression.DruidExpression;
|
import org.apache.druid.sql.calcite.expression.DruidExpression;
|
||||||
import org.apache.druid.sql.calcite.expression.OperatorConversions;
|
import org.apache.druid.sql.calcite.expression.OperatorConversions;
|
||||||
import org.apache.druid.sql.calcite.expression.PostAggregatorVisitor;
|
import org.apache.druid.sql.calcite.expression.PostAggregatorVisitor;
|
||||||
|
import org.apache.druid.sql.calcite.expression.SqlOperatorConversion;
|
||||||
import org.apache.druid.sql.calcite.planner.PlannerContext;
|
import org.apache.druid.sql.calcite.planner.PlannerContext;
|
||||||
|
|
||||||
import javax.annotation.Nullable;
|
import javax.annotation.Nullable;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
|
|
||||||
public class HdrHistogramPercentilesOperatorConversion extends DirectOperatorConversion {
|
public class HdrHistogramPercentilesOperatorConversion implements SqlOperatorConversion {
|
||||||
private static final String FUNCTION_NAME = "HDR_GET_PERCENTILES";
|
private static final String FUNCTION_NAME = "HDR_GET_PERCENTILES";
|
||||||
private static final SqlFunction SQL_FUNCTION = OperatorConversions
|
private static final SqlFunction SQL_FUNCTION = OperatorConversions
|
||||||
.operatorBuilder(StringUtils.toUpperCase(FUNCTION_NAME))
|
.operatorBuilder(StringUtils.toUpperCase(FUNCTION_NAME))
|
||||||
@@ -32,10 +32,6 @@ public class HdrHistogramPercentilesOperatorConversion extends DirectOperatorCon
|
|||||||
.returnTypeInference(ReturnTypes.explicit(SqlTypeName.VARCHAR))
|
.returnTypeInference(ReturnTypes.explicit(SqlTypeName.VARCHAR))
|
||||||
.build();
|
.build();
|
||||||
|
|
||||||
public HdrHistogramPercentilesOperatorConversion() {
|
|
||||||
super(SQL_FUNCTION, FUNCTION_NAME);
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public SqlOperator calciteOperator()
|
public SqlOperator calciteOperator()
|
||||||
{
|
{
|
||||||
@@ -66,7 +62,8 @@ public class HdrHistogramPercentilesOperatorConversion extends DirectOperatorCon
|
|||||||
plannerContext,
|
plannerContext,
|
||||||
rowSignature,
|
rowSignature,
|
||||||
operands.get(0),
|
operands.get(0),
|
||||||
postAggregatorVisitor
|
postAggregatorVisitor,
|
||||||
|
true
|
||||||
);
|
);
|
||||||
|
|
||||||
if (postAgg == null) {
|
if (postAgg == null) {
|
||||||
|
|||||||
@@ -16,6 +16,7 @@ import org.apache.druid.query.aggregation.sketch.HdrHistogram.HdrHistogramAggreg
|
|||||||
import org.apache.druid.query.aggregation.sketch.HdrHistogram.HdrHistogramMergeAggregatorFactory;
|
import org.apache.druid.query.aggregation.sketch.HdrHistogram.HdrHistogramMergeAggregatorFactory;
|
||||||
import org.apache.druid.query.aggregation.sketch.HdrHistogram.HdrHistogramToQuantilePostAggregator;
|
import org.apache.druid.query.aggregation.sketch.HdrHistogram.HdrHistogramToQuantilePostAggregator;
|
||||||
import org.apache.druid.segment.VirtualColumn;
|
import org.apache.druid.segment.VirtualColumn;
|
||||||
|
import org.apache.druid.segment.column.ColumnType;
|
||||||
import org.apache.druid.segment.column.RowSignature;
|
import org.apache.druid.segment.column.RowSignature;
|
||||||
import org.apache.druid.segment.column.ValueType;
|
import org.apache.druid.segment.column.ValueType;
|
||||||
import org.apache.druid.segment.virtual.ExpressionVirtualColumn;
|
import org.apache.druid.segment.virtual.ExpressionVirtualColumn;
|
||||||
@@ -141,22 +142,16 @@ public class HdrHistogramQuantileSqlAggregator implements SqlAggregator {
|
|||||||
|
|
||||||
// Check input for equivalence.
|
// Check input for equivalence.
|
||||||
final boolean inputMatches;
|
final boolean inputMatches;
|
||||||
final VirtualColumn virtualInput = existing.getVirtualColumns()
|
final DruidExpression virtualInput =
|
||||||
.stream()
|
virtualColumnRegistry.findVirtualColumnExpressions(theFactory.requiredFields())
|
||||||
.filter(
|
.stream()
|
||||||
virtualColumn ->
|
.findFirst()
|
||||||
virtualColumn.getOutputName()
|
.orElse(null);
|
||||||
.equals(theFactory.getFieldName())
|
|
||||||
)
|
|
||||||
.findFirst()
|
|
||||||
.orElse(null);
|
|
||||||
|
|
||||||
if (virtualInput == null) {
|
if (virtualInput == null) {
|
||||||
inputMatches = input.isDirectColumnAccess()
|
inputMatches = input.isDirectColumnAccess() && input.getDirectColumn().equals(theFactory.getFieldName());
|
||||||
&& input.getDirectColumn().equals(theFactory.getFieldName());
|
|
||||||
} else {
|
} else {
|
||||||
inputMatches = ((ExpressionVirtualColumn) virtualInput).getExpression()
|
inputMatches = virtualInput.equals(input);
|
||||||
.equals(input.getExpression());
|
|
||||||
}
|
}
|
||||||
|
|
||||||
final boolean matches = inputMatches
|
final boolean matches = inputMatches
|
||||||
@@ -177,11 +172,11 @@ public class HdrHistogramQuantileSqlAggregator implements SqlAggregator {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// No existing match found. Create a new one.
|
// No existing match found. Create a new one.
|
||||||
final List<VirtualColumn> virtualColumns = new ArrayList<>();
|
//final List<VirtualColumn> virtualColumns = new ArrayList<>();
|
||||||
|
|
||||||
if (input.isDirectColumnAccess()) {
|
if (input.isDirectColumnAccess()) {
|
||||||
// 参数是Histogram对象
|
// 参数是Histogram对象
|
||||||
if (rowSignature.getColumnType(input.getDirectColumn()).orElse(null) == ValueType.COMPLEX) {
|
if (rowSignature.getColumnType(input.getDirectColumn()).map(type -> type.is(ValueType.COMPLEX)).orElse(false)) {
|
||||||
aggregatorFactory = new HdrHistogramMergeAggregatorFactory(
|
aggregatorFactory = new HdrHistogramMergeAggregatorFactory(
|
||||||
histogramName,
|
histogramName,
|
||||||
input.getDirectColumn(),
|
input.getDirectColumn(),
|
||||||
@@ -201,12 +196,11 @@ public class HdrHistogramQuantileSqlAggregator implements SqlAggregator {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
final VirtualColumn virtualColumn =
|
final String virtualColumnName =
|
||||||
virtualColumnRegistry.getOrCreateVirtualColumnForExpression(plannerContext, input, SqlTypeName.BIGINT);
|
virtualColumnRegistry.getOrCreateVirtualColumnForExpression(input, ColumnType.LONG);
|
||||||
virtualColumns.add(virtualColumn);
|
|
||||||
aggregatorFactory = new HdrHistogramAggregatorFactory(
|
aggregatorFactory = new HdrHistogramAggregatorFactory(
|
||||||
histogramName,
|
histogramName,
|
||||||
virtualColumn.getOutputName(),
|
virtualColumnName,
|
||||||
lowestDiscernibleValue,
|
lowestDiscernibleValue,
|
||||||
highestTrackableValue,
|
highestTrackableValue,
|
||||||
numberOfSignificantValueDigits,
|
numberOfSignificantValueDigits,
|
||||||
@@ -234,7 +228,6 @@ public class HdrHistogramQuantileSqlAggregator implements SqlAggregator {
|
|||||||
}
|
}
|
||||||
|
|
||||||
return Aggregation.create(
|
return Aggregation.create(
|
||||||
virtualColumns,
|
|
||||||
ImmutableList.of(aggregatorFactory),
|
ImmutableList.of(aggregatorFactory),
|
||||||
new HdrHistogramToQuantilePostAggregator(name, histogramName, probability)
|
new HdrHistogramToQuantilePostAggregator(name, histogramName, probability)
|
||||||
);
|
);
|
||||||
|
|||||||
@@ -62,50 +62,30 @@ public class HdrHistogramQuantilesOperatorConversion implements SqlOperatorConve
|
|||||||
{
|
{
|
||||||
final List<RexNode> operands = ((RexCall) rexNode).getOperands();
|
final List<RexNode> operands = ((RexCall) rexNode).getOperands();
|
||||||
final float[] args = new float[operands.size() - 1];
|
final float[] args = new float[operands.size() - 1];
|
||||||
PostAggregator postAgg = null;
|
|
||||||
|
|
||||||
int operandCounter = 0;
|
// 新版本直接就从第一个参数取
|
||||||
for (RexNode operand : operands) {
|
final PostAggregator inputSketchPostAgg = OperatorConversions.toPostAggregator(
|
||||||
final PostAggregator convertedPostAgg = OperatorConversions.toPostAggregator(
|
plannerContext,
|
||||||
plannerContext,
|
rowSignature,
|
||||||
rowSignature,
|
operands.get(0),
|
||||||
operand,
|
postAggregatorVisitor,
|
||||||
postAggregatorVisitor
|
true
|
||||||
);
|
);
|
||||||
if (convertedPostAgg == null) {
|
|
||||||
if (operandCounter > 0) {
|
if (inputSketchPostAgg == null) {
|
||||||
try {
|
return null;
|
||||||
if (!operand.isA(SqlKind.LITERAL)) {
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
float arg = ((Number) RexLiteral.value(operand)).floatValue();
|
|
||||||
args[operandCounter - 1] = arg;
|
|
||||||
}
|
|
||||||
catch (ClassCastException cce) {
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
if (operandCounter == 0) {
|
|
||||||
postAgg = convertedPostAgg;
|
|
||||||
} else {
|
|
||||||
if (!operand.isA(SqlKind.LITERAL)) {
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
operandCounter++;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
if (postAgg == null) {
|
// 直接解析
|
||||||
return null;
|
for (int i = 1; i < operands.size(); i++) {
|
||||||
|
RexNode operand = operands.get(i);
|
||||||
|
float arg = ((Number) RexLiteral.value(operand)).floatValue();
|
||||||
|
args[i - 1] = arg;
|
||||||
}
|
}
|
||||||
|
|
||||||
return new HdrHistogramToQuantilesPostAggregator(
|
return new HdrHistogramToQuantilesPostAggregator(
|
||||||
postAggregatorVisitor.getOutputNamePrefix() + postAggregatorVisitor.getAndIncrementCounter(),
|
postAggregatorVisitor.getOutputNamePrefix() + postAggregatorVisitor.getAndIncrementCounter(),
|
||||||
((FieldAccessPostAggregator)postAgg).getFieldName(),
|
((FieldAccessPostAggregator)inputSketchPostAgg).getFieldName(),
|
||||||
args
|
args
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -2,17 +2,13 @@ package org.apache.druid.query.aggregation.sketch.HdrHistogram;
|
|||||||
|
|
||||||
import com.google.common.collect.ImmutableMap;
|
import com.google.common.collect.ImmutableMap;
|
||||||
import org.HdrHistogram.*;
|
import org.HdrHistogram.*;
|
||||||
import org.apache.datasketches.theta.Sketches;
|
|
||||||
import org.apache.datasketches.theta.UpdateSketch;
|
|
||||||
import org.apache.druid.data.input.MapBasedRow;
|
import org.apache.druid.data.input.MapBasedRow;
|
||||||
import org.apache.druid.query.aggregation.AggregatorFactory;
|
import org.apache.druid.query.aggregation.AggregatorFactory;
|
||||||
import org.apache.druid.query.aggregation.BufferAggregator;
|
import org.apache.druid.query.aggregation.BufferAggregator;
|
||||||
import org.apache.druid.query.aggregation.TestLongColumnSelector;
|
import org.apache.druid.query.aggregation.TestLongColumnSelector;
|
||||||
import org.apache.druid.query.aggregation.TestObjectColumnSelector;
|
import org.apache.druid.query.aggregation.TestObjectColumnSelector;
|
||||||
import org.apache.druid.query.aggregation.datasketches.theta.SketchHolder;
|
|
||||||
import org.apache.druid.query.aggregation.datasketches.theta.SketchMergeAggregatorFactory;
|
|
||||||
import org.apache.druid.query.groupby.epinephelinae.GrouperTestUtil;
|
import org.apache.druid.query.groupby.epinephelinae.GrouperTestUtil;
|
||||||
import org.apache.druid.query.groupby.epinephelinae.TestColumnSelectorFactory;
|
import org.apache.druid.query.groupby.epinephelinae.GroupByTestColumnSelectorFactory;
|
||||||
import org.apache.druid.segment.ColumnSelectorFactory;
|
import org.apache.druid.segment.ColumnSelectorFactory;
|
||||||
import org.junit.Assert;
|
import org.junit.Assert;
|
||||||
import org.junit.Test;
|
import org.junit.Test;
|
||||||
@@ -230,7 +226,7 @@ public class HdrHistogramBufferAggregatorTest {
|
|||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void testMergeAggregatorRelocate() {
|
public void testMergeAggregatorRelocate() {
|
||||||
final TestColumnSelectorFactory columnSelectorFactory = GrouperTestUtil.newColumnSelectorFactory();
|
final GroupByTestColumnSelectorFactory columnSelectorFactory = GrouperTestUtil.newColumnSelectorFactory();
|
||||||
HistogramSketch histogram = new HistogramSketch(3);
|
HistogramSketch histogram = new HistogramSketch(3);
|
||||||
for (int i = 0; i < 100000; i++) {
|
for (int i = 0; i < 100000; i++) {
|
||||||
histogram.recordValue(i);
|
histogram.recordValue(i);
|
||||||
@@ -252,7 +248,7 @@ public class HdrHistogramBufferAggregatorTest {
|
|||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void testAggregatorRelocate() {
|
public void testAggregatorRelocate() {
|
||||||
final TestColumnSelectorFactory columnSelectorFactory = GrouperTestUtil.newColumnSelectorFactory();
|
final GroupByTestColumnSelectorFactory columnSelectorFactory = GrouperTestUtil.newColumnSelectorFactory();
|
||||||
HistogramSketch histogram = new HistogramSketch(3);
|
HistogramSketch histogram = new HistogramSketch(3);
|
||||||
for (int i = 0; i < 100000; i++) {
|
for (int i = 0; i < 100000; i++) {
|
||||||
histogram.recordValue(i);
|
histogram.recordValue(i);
|
||||||
|
|||||||
@@ -1,12 +1,15 @@
|
|||||||
package org.apache.druid.query.aggregation.sketch.HdrHistogram.sql;
|
package org.apache.druid.query.aggregation.sketch.HdrHistogram.sql;
|
||||||
|
|
||||||
|
import com.alibaba.fastjson2.JSON;
|
||||||
import com.fasterxml.jackson.databind.Module;
|
import com.fasterxml.jackson.databind.Module;
|
||||||
import com.google.common.collect.ImmutableList;
|
import com.google.common.collect.ImmutableList;
|
||||||
import com.google.common.collect.ImmutableMap;
|
import com.google.common.collect.ImmutableMap;
|
||||||
import com.google.common.collect.ImmutableSet;
|
import com.google.common.collect.ImmutableSet;
|
||||||
import com.google.common.collect.Iterables;
|
import com.google.common.collect.Iterables;
|
||||||
|
import com.google.inject.Injector;
|
||||||
import org.apache.calcite.schema.SchemaPlus;
|
import org.apache.calcite.schema.SchemaPlus;
|
||||||
import org.apache.druid.data.input.InputRow;
|
import org.apache.druid.data.input.InputRow;
|
||||||
|
import org.apache.druid.guice.DruidInjectorBuilder;
|
||||||
import org.apache.druid.java.util.common.granularity.Granularities;
|
import org.apache.druid.java.util.common.granularity.Granularities;
|
||||||
import org.apache.druid.java.util.common.io.Closer;
|
import org.apache.druid.java.util.common.io.Closer;
|
||||||
import org.apache.druid.query.Druids;
|
import org.apache.druid.query.Druids;
|
||||||
@@ -27,66 +30,49 @@ import org.apache.druid.query.spec.MultipleIntervalSegmentSpec;
|
|||||||
import org.apache.druid.segment.IndexBuilder;
|
import org.apache.druid.segment.IndexBuilder;
|
||||||
import org.apache.druid.segment.QueryableIndex;
|
import org.apache.druid.segment.QueryableIndex;
|
||||||
import org.apache.druid.segment.TestHelper;
|
import org.apache.druid.segment.TestHelper;
|
||||||
|
import org.apache.druid.segment.column.ColumnType;
|
||||||
import org.apache.druid.segment.column.ValueType;
|
import org.apache.druid.segment.column.ValueType;
|
||||||
import org.apache.druid.segment.incremental.IncrementalIndexSchema;
|
import org.apache.druid.segment.incremental.IncrementalIndexSchema;
|
||||||
|
import org.apache.druid.segment.join.JoinableFactoryWrapper;
|
||||||
import org.apache.druid.segment.virtual.ExpressionVirtualColumn;
|
import org.apache.druid.segment.virtual.ExpressionVirtualColumn;
|
||||||
import org.apache.druid.segment.writeout.OffHeapMemorySegmentWriteOutMediumFactory;
|
import org.apache.druid.segment.writeout.OffHeapMemorySegmentWriteOutMediumFactory;
|
||||||
import org.apache.druid.server.QueryStackTests;
|
import org.apache.druid.server.QueryStackTests;
|
||||||
import org.apache.druid.server.security.AuthTestUtils;
|
import org.apache.druid.server.security.AuthTestUtils;
|
||||||
import org.apache.druid.server.security.AuthenticationResult;
|
import org.apache.druid.server.security.AuthenticationResult;
|
||||||
import org.apache.druid.sql.SqlLifecycle;
|
import org.apache.druid.sql.calcite.BaseCalciteQueryTest;
|
||||||
import org.apache.druid.sql.SqlLifecycleFactory;
|
import org.apache.druid.sql.calcite.QueryTestBuilder;
|
||||||
|
import org.apache.druid.sql.calcite.QueryTestRunner;
|
||||||
import org.apache.druid.sql.calcite.filtration.Filtration;
|
import org.apache.druid.sql.calcite.filtration.Filtration;
|
||||||
import org.apache.druid.sql.calcite.planner.DruidOperatorTable;
|
import org.apache.druid.sql.calcite.planner.DruidOperatorTable;
|
||||||
import org.apache.druid.sql.calcite.planner.PlannerConfig;
|
import org.apache.druid.sql.calcite.planner.PlannerConfig;
|
||||||
import org.apache.druid.sql.calcite.planner.PlannerContext;
|
import org.apache.druid.sql.calcite.planner.PlannerContext;
|
||||||
import org.apache.druid.sql.calcite.planner.PlannerFactory;
|
import org.apache.druid.sql.calcite.planner.PlannerFactory;
|
||||||
import org.apache.druid.sql.calcite.util.CalciteTestBase;
|
import org.apache.druid.sql.calcite.util.*;
|
||||||
import org.apache.druid.sql.calcite.util.CalciteTests;
|
|
||||||
import org.apache.druid.sql.calcite.util.QueryLogHook;
|
|
||||||
import org.apache.druid.sql.calcite.util.SpecificSegmentsQuerySegmentWalker;
|
|
||||||
import org.apache.druid.timeline.DataSegment;
|
import org.apache.druid.timeline.DataSegment;
|
||||||
import org.apache.druid.timeline.partition.LinearShardSpec;
|
import org.apache.druid.timeline.partition.LinearShardSpec;
|
||||||
import org.junit.*;
|
import org.junit.*;
|
||||||
import org.junit.rules.TemporaryFolder;
|
import org.junit.rules.TemporaryFolder;
|
||||||
|
|
||||||
|
import java.io.File;
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
import java.util.Arrays;
|
import java.util.*;
|
||||||
import java.util.List;
|
|
||||||
import java.util.Map;
|
|
||||||
|
|
||||||
public class HdrHistogramQuantileSqlAggregatorTest extends CalciteTestBase {
|
public class HdrHistogramQuantileSqlAggregatorTest extends BaseCalciteQueryTest {
|
||||||
private static final String DATA_SOURCE = "foo";
|
@Override
|
||||||
|
public void gatherProperties(Properties properties)
|
||||||
private static QueryRunnerFactoryConglomerate conglomerate;
|
{
|
||||||
private static Closer resourceCloser;
|
super.gatherProperties(properties);
|
||||||
private static AuthenticationResult authenticationResult = CalciteTests.REGULAR_USER_AUTH_RESULT;
|
|
||||||
private static final Map<String, Object> QUERY_CONTEXT_DEFAULT = ImmutableMap.of(
|
|
||||||
PlannerContext.CTX_SQL_QUERY_ID, "dummy"
|
|
||||||
);
|
|
||||||
|
|
||||||
@Rule
|
|
||||||
public TemporaryFolder temporaryFolder = new TemporaryFolder();
|
|
||||||
|
|
||||||
@Rule
|
|
||||||
public QueryLogHook queryLogHook = QueryLogHook.create();
|
|
||||||
|
|
||||||
private SpecificSegmentsQuerySegmentWalker walker;
|
|
||||||
private SqlLifecycleFactory sqlLifecycleFactory;
|
|
||||||
|
|
||||||
@BeforeClass
|
|
||||||
public static void setUpClass() {
|
|
||||||
resourceCloser = Closer.create();
|
|
||||||
conglomerate = QueryStackTests.createQueryRunnerFactoryConglomerate(resourceCloser);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
@AfterClass
|
@Override
|
||||||
public static void tearDownClass() throws IOException {
|
public void configureGuice(DruidInjectorBuilder builder)
|
||||||
resourceCloser.close();
|
{
|
||||||
|
super.configureGuice(builder);
|
||||||
|
builder.addModule(new HdrHistogramModule());
|
||||||
}
|
}
|
||||||
|
|
||||||
public static final List<InputRow> ROWS1 = ImmutableList.of(
|
public static final List<InputRow> ROWS1 = ImmutableList.of(
|
||||||
CalciteTests.createRow(
|
TestDataBuilder.createRow(
|
||||||
ImmutableMap.<String, Object>builder()
|
ImmutableMap.<String, Object>builder()
|
||||||
.put("t", "2000-01-01")
|
.put("t", "2000-01-01")
|
||||||
.put("m1", "1")
|
.put("m1", "1")
|
||||||
@@ -96,7 +82,7 @@ public class HdrHistogramQuantileSqlAggregatorTest extends CalciteTestBase {
|
|||||||
.put("dim3", ImmutableList.of("a", "b"))
|
.put("dim3", ImmutableList.of("a", "b"))
|
||||||
.build()
|
.build()
|
||||||
),
|
),
|
||||||
CalciteTests.createRow(
|
TestDataBuilder.createRow(
|
||||||
ImmutableMap.<String, Object>builder()
|
ImmutableMap.<String, Object>builder()
|
||||||
.put("t", "2000-01-02")
|
.put("t", "2000-01-02")
|
||||||
.put("m1", "2.0")
|
.put("m1", "2.0")
|
||||||
@@ -106,7 +92,7 @@ public class HdrHistogramQuantileSqlAggregatorTest extends CalciteTestBase {
|
|||||||
.put("dim3", ImmutableList.of("b", "c"))
|
.put("dim3", ImmutableList.of("b", "c"))
|
||||||
.build()
|
.build()
|
||||||
),
|
),
|
||||||
CalciteTests.createRow(
|
TestDataBuilder.createRow(
|
||||||
ImmutableMap.<String, Object>builder()
|
ImmutableMap.<String, Object>builder()
|
||||||
.put("t", "2000-01-03")
|
.put("t", "2000-01-03")
|
||||||
.put("m1", "3.0")
|
.put("m1", "3.0")
|
||||||
@@ -116,7 +102,7 @@ public class HdrHistogramQuantileSqlAggregatorTest extends CalciteTestBase {
|
|||||||
.put("dim3", ImmutableList.of("d"))
|
.put("dim3", ImmutableList.of("d"))
|
||||||
.build()
|
.build()
|
||||||
),
|
),
|
||||||
CalciteTests.createRow(
|
TestDataBuilder.createRow(
|
||||||
ImmutableMap.<String, Object>builder()
|
ImmutableMap.<String, Object>builder()
|
||||||
.put("t", "2001-01-01")
|
.put("t", "2001-01-01")
|
||||||
.put("m1", "4.0")
|
.put("m1", "4.0")
|
||||||
@@ -126,7 +112,7 @@ public class HdrHistogramQuantileSqlAggregatorTest extends CalciteTestBase {
|
|||||||
.put("dim3", ImmutableList.of(""))
|
.put("dim3", ImmutableList.of(""))
|
||||||
.build()
|
.build()
|
||||||
),
|
),
|
||||||
CalciteTests.createRow(
|
TestDataBuilder.createRow(
|
||||||
ImmutableMap.<String, Object>builder()
|
ImmutableMap.<String, Object>builder()
|
||||||
.put("t", "2001-01-02")
|
.put("t", "2001-01-02")
|
||||||
.put("m1", "5.0")
|
.put("m1", "5.0")
|
||||||
@@ -136,7 +122,7 @@ public class HdrHistogramQuantileSqlAggregatorTest extends CalciteTestBase {
|
|||||||
.put("dim3", ImmutableList.of())
|
.put("dim3", ImmutableList.of())
|
||||||
.build()
|
.build()
|
||||||
),
|
),
|
||||||
CalciteTests.createRow(
|
TestDataBuilder.createRow(
|
||||||
ImmutableMap.<String, Object>builder()
|
ImmutableMap.<String, Object>builder()
|
||||||
.put("t", "2001-01-03")
|
.put("t", "2001-01-03")
|
||||||
.put("m1", "6.0")
|
.put("m1", "6.0")
|
||||||
@@ -146,15 +132,20 @@ public class HdrHistogramQuantileSqlAggregatorTest extends CalciteTestBase {
|
|||||||
)
|
)
|
||||||
);
|
);
|
||||||
|
|
||||||
@Before
|
@SuppressWarnings("resource")
|
||||||
public void setUp() throws Exception {
|
@Override
|
||||||
|
public SpecificSegmentsQuerySegmentWalker createQuerySegmentWalker(
|
||||||
|
final QueryRunnerFactoryConglomerate conglomerate,
|
||||||
|
final JoinableFactoryWrapper joinableFactory,
|
||||||
|
final Injector injector
|
||||||
|
) throws IOException{
|
||||||
HdrHistogramModule.registerSerde();
|
HdrHistogramModule.registerSerde();
|
||||||
for (Module mod : new HdrHistogramModule().getJacksonModules()) {
|
for (Module mod : new HdrHistogramModule().getJacksonModules()) {
|
||||||
CalciteTests.getJsonMapper().registerModule(mod);
|
CalciteTests.getJsonMapper().registerModule(mod);
|
||||||
TestHelper.JSON_MAPPER.registerModule(mod);
|
TestHelper.JSON_MAPPER.registerModule(mod);
|
||||||
}
|
}
|
||||||
|
final QueryableIndex index = TestHelper.getTestIndexIO().loadIndex(new File("D:/doc/datas/testIndex-6201298"));
|
||||||
final QueryableIndex index = IndexBuilder.create()
|
/*final QueryableIndex index = IndexBuilder.create()
|
||||||
.tmpDir(temporaryFolder.newFolder())
|
.tmpDir(temporaryFolder.newFolder())
|
||||||
.segmentWriteOutMediumFactory(OffHeapMemorySegmentWriteOutMediumFactory.instance())
|
.segmentWriteOutMediumFactory(OffHeapMemorySegmentWriteOutMediumFactory.instance())
|
||||||
.schema(
|
.schema(
|
||||||
@@ -176,11 +167,11 @@ public class HdrHistogramQuantileSqlAggregatorTest extends CalciteTestBase {
|
|||||||
)
|
)
|
||||||
//.rows(CalciteTests.ROWS1)
|
//.rows(CalciteTests.ROWS1)
|
||||||
.rows(ROWS1)
|
.rows(ROWS1)
|
||||||
.buildMMappedIndex();
|
.buildMMappedIndex();*/
|
||||||
|
|
||||||
walker = new SpecificSegmentsQuerySegmentWalker(conglomerate).add(
|
return new SpecificSegmentsQuerySegmentWalker(conglomerate).add(
|
||||||
DataSegment.builder()
|
DataSegment.builder()
|
||||||
.dataSource(DATA_SOURCE)
|
.dataSource(CalciteTests.DATASOURCE1)
|
||||||
.interval(index.getDataInterval())
|
.interval(index.getDataInterval())
|
||||||
.version("1")
|
.version("1")
|
||||||
.shardSpec(new LinearShardSpec(0))
|
.shardSpec(new LinearShardSpec(0))
|
||||||
@@ -188,69 +179,77 @@ public class HdrHistogramQuantileSqlAggregatorTest extends CalciteTestBase {
|
|||||||
.build(),
|
.build(),
|
||||||
index
|
index
|
||||||
);
|
);
|
||||||
|
|
||||||
final PlannerConfig plannerConfig = new PlannerConfig();
|
|
||||||
final DruidOperatorTable operatorTable = new DruidOperatorTable(
|
|
||||||
ImmutableSet.of(
|
|
||||||
new HdrHistogramQuantileSqlAggregator(),
|
|
||||||
new HdrHistogramObjectSqlAggregator()
|
|
||||||
),
|
|
||||||
ImmutableSet.of(
|
|
||||||
new HdrHistogramQuantilesOperatorConversion(),
|
|
||||||
new HdrHistogramPercentilesOperatorConversion()
|
|
||||||
)
|
|
||||||
);
|
|
||||||
SchemaPlus rootSchema =
|
|
||||||
CalciteTests.createMockRootSchema(conglomerate, walker, plannerConfig, AuthTestUtils.TEST_AUTHORIZER_MAPPER);
|
|
||||||
|
|
||||||
sqlLifecycleFactory = CalciteTests.createSqlLifecycleFactory(
|
|
||||||
new PlannerFactory(
|
|
||||||
rootSchema,
|
|
||||||
CalciteTests.createMockQueryLifecycleFactory(walker, conglomerate),
|
|
||||||
operatorTable,
|
|
||||||
CalciteTests.createExprMacroTable(),
|
|
||||||
plannerConfig,
|
|
||||||
AuthTestUtils.TEST_AUTHORIZER_MAPPER,
|
|
||||||
CalciteTests.getJsonMapper(),
|
|
||||||
CalciteTests.DRUID_SCHEMA_NAME
|
|
||||||
)
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
@After
|
|
||||||
public void tearDown() throws Exception {
|
|
||||||
walker.close();
|
|
||||||
walker = null;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void testSqlQuery() throws Exception {
|
public void testSqlQuery() throws Exception {
|
||||||
SqlLifecycle sqlLifecycle = sqlLifecycleFactory.factorize();
|
String[] columns = new String[]{"__time", "dim1", "dim2", "dim3", "cnt", "hist_m1", "m1"};
|
||||||
String sql = "select * from druid.foo";
|
String sql = "select " + String.join(",", columns) + " from druid.foo";
|
||||||
final List<Object[]> results =
|
QueryTestBuilder builder = testBuilder().sql(sql);
|
||||||
sqlLifecycle.runSimple(sql, QUERY_CONTEXT_DEFAULT, DEFAULT_PARAMETERS, authenticationResult).toList();
|
builder.run();
|
||||||
|
QueryTestRunner.QueryResults queryResults = builder.results();
|
||||||
|
List<Object[]> results = queryResults.results;
|
||||||
for (Object[] result : results) {
|
for (Object[] result : results) {
|
||||||
System.out.println(Arrays.toString(result));
|
Map row = new LinkedHashMap();
|
||||||
|
for (int i = 0; i < result.length; i++) {
|
||||||
|
row.put(columns[i], result[i]);
|
||||||
|
}
|
||||||
|
System.out.println(JSON.toJSONString(row));
|
||||||
|
// System.out.println(Arrays.toString(result));
|
||||||
|
}
|
||||||
|
|
||||||
|
for (int i = 0; i < columns.length; i++) {
|
||||||
|
Object[] values = new Object[results.size()];
|
||||||
|
for (int j = 0; j < results.size(); j++) {
|
||||||
|
values[j] = results.get(j)[i];
|
||||||
|
}
|
||||||
|
System.out.println(columns[i] + ":" + Arrays.toString(values));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void testGroup() throws Exception {
|
public void testGroup() throws Exception {
|
||||||
SqlLifecycle sqlLifecycle = sqlLifecycleFactory.factorize();
|
|
||||||
String sql = "select cnt, APPROX_QUANTILE_HDR(hist_m1, 0.5, 1, 100, 2) from druid.foo group by cnt";
|
String sql = "select cnt, APPROX_QUANTILE_HDR(hist_m1, 0.5, 1, 100, 2) from druid.foo group by cnt";
|
||||||
final List<Object[]> results =
|
QueryTestBuilder builder = testBuilder().sql(sql).skipVectorize();
|
||||||
sqlLifecycle.runSimple(sql, QUERY_CONTEXT_DEFAULT, DEFAULT_PARAMETERS, authenticationResult).toList();
|
builder.run();
|
||||||
for (Object[] result : results) {
|
QueryTestRunner.QueryResults queryResults = builder.results();
|
||||||
System.out.println(Arrays.toString(result));
|
List<Object[]> results = queryResults.results;
|
||||||
}
|
for (Object[] result : results) {
|
||||||
|
System.out.println(Arrays.toString(result));
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void testGroup2() throws Exception {
|
public void testGroup2() throws Exception {
|
||||||
SqlLifecycle sqlLifecycle = sqlLifecycleFactory.factorize();
|
|
||||||
String sql = "select HDR_HISTOGRAM(hist_m1) from druid.foo";
|
String sql = "select HDR_HISTOGRAM(hist_m1) from druid.foo";
|
||||||
final List<Object[]> results =
|
QueryTestBuilder builder = testBuilder().sql(sql).skipVectorize();
|
||||||
sqlLifecycle.runSimple(sql, QUERY_CONTEXT_DEFAULT, DEFAULT_PARAMETERS, authenticationResult).toList();
|
builder.run();
|
||||||
|
QueryTestRunner.QueryResults queryResults = builder.results();
|
||||||
|
List<Object[]> results = queryResults.results;
|
||||||
|
for (Object[] result : results) {
|
||||||
|
System.out.println(Arrays.toString(result));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testGroup3() throws Exception {
|
||||||
|
String sql = "select APPROX_QUANTILE_HDR(h, 0.5) from(select HDR_HISTOGRAM(hist_m1) h from druid.foo) t";
|
||||||
|
QueryTestBuilder builder = testBuilder().sql(sql).skipVectorize();
|
||||||
|
builder.run();
|
||||||
|
QueryTestRunner.QueryResults queryResults = builder.results();
|
||||||
|
List<Object[]> results = queryResults.results;
|
||||||
|
for (Object[] result : results) {
|
||||||
|
System.out.println(Arrays.toString(result));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testGroup4() throws Exception {
|
||||||
|
String sql = "select hdr_get_quantiles(h, 0.1, 0.2, 0.3, 0.5, 0.9, 0.99, 1) from(select HDR_HISTOGRAM(hist_m1) h from druid.foo) t";
|
||||||
|
QueryTestBuilder builder = testBuilder().sql(sql).skipVectorize();
|
||||||
|
builder.run();
|
||||||
|
QueryTestRunner.QueryResults queryResults = builder.results();
|
||||||
|
List<Object[]> results = queryResults.results;
|
||||||
for (Object[] result : results) {
|
for (Object[] result : results) {
|
||||||
System.out.println(Arrays.toString(result));
|
System.out.println(Arrays.toString(result));
|
||||||
}
|
}
|
||||||
@@ -258,10 +257,11 @@ public class HdrHistogramQuantileSqlAggregatorTest extends CalciteTestBase {
|
|||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void testSqlQueryGeneHdr() throws Exception {
|
public void testSqlQueryGeneHdr() throws Exception {
|
||||||
SqlLifecycle sqlLifecycle = sqlLifecycleFactory.factorize();
|
|
||||||
String sql = "select HDR_HISTOGRAM(hist_m1, 1, 100, 2), HDR_HISTOGRAM(cnt, 1, 100, 2) from druid.foo";
|
String sql = "select HDR_HISTOGRAM(hist_m1, 1, 100, 2), HDR_HISTOGRAM(cnt, 1, 100, 2) from druid.foo";
|
||||||
final List<Object[]> results =
|
QueryTestBuilder builder = testBuilder().sql(sql).skipVectorize();
|
||||||
sqlLifecycle.runSimple(sql, QUERY_CONTEXT_DEFAULT, DEFAULT_PARAMETERS, authenticationResult).toList();
|
builder.run();
|
||||||
|
QueryTestRunner.QueryResults queryResults = builder.results();
|
||||||
|
List<Object[]> results = queryResults.results;
|
||||||
for (Object[] result : results) {
|
for (Object[] result : results) {
|
||||||
System.out.println(Arrays.toString(result));
|
System.out.println(Arrays.toString(result));
|
||||||
}
|
}
|
||||||
@@ -269,11 +269,12 @@ public class HdrHistogramQuantileSqlAggregatorTest extends CalciteTestBase {
|
|||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void testSqlQueryGeneHdr2() throws Exception {
|
public void testSqlQueryGeneHdr2() throws Exception {
|
||||||
SqlLifecycle sqlLifecycle = sqlLifecycleFactory.factorize();
|
|
||||||
// HDR_HISTOGRAM(hist_m1, 1, 100, 2),
|
// HDR_HISTOGRAM(hist_m1, 1, 100, 2),
|
||||||
String sql = "select HDR_GET_QUANTILES(HDR_HISTOGRAM(m1, 1, 100, 2), 0.1, 0.2, 0.3, 0.5, 0.9, 1) from druid.foo";
|
String sql = "select HDR_GET_QUANTILES(HDR_HISTOGRAM(m1, 1, 100, 2), 0.1, 0.2, 0.3, 0.5, 0.9, 1) from druid.foo";
|
||||||
final List<Object[]> results =
|
QueryTestBuilder builder = testBuilder().sql(sql).skipVectorize();
|
||||||
sqlLifecycle.runSimple(sql, QUERY_CONTEXT_DEFAULT, DEFAULT_PARAMETERS, authenticationResult).toList();
|
builder.run();
|
||||||
|
QueryTestRunner.QueryResults queryResults = builder.results();
|
||||||
|
List<Object[]> results = queryResults.results;
|
||||||
for (Object[] result : results) {
|
for (Object[] result : results) {
|
||||||
System.out.println(Arrays.toString(result));
|
System.out.println(Arrays.toString(result));
|
||||||
}
|
}
|
||||||
@@ -281,44 +282,47 @@ public class HdrHistogramQuantileSqlAggregatorTest extends CalciteTestBase {
|
|||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void testSqlQueryGeneHdrArgs() throws Exception {
|
public void testSqlQueryGeneHdrArgs() throws Exception {
|
||||||
SqlLifecycle sqlLifecycle = sqlLifecycleFactory.factorize();
|
|
||||||
String sql = "select HDR_GET_QUANTILEs(HDR_HISTOGRAM(m1), 0.1, 0.2, 0.3, 0.5, 0.9, 1), "
|
String sql = "select HDR_GET_QUANTILEs(HDR_HISTOGRAM(m1), 0.1, 0.2, 0.3, 0.5, 0.9, 1), "
|
||||||
+ "HDR_GET_QUANTILEs(HDR_HISTOGRAM(m1, 2), 0.1, 0.2, 0.3, 0.5, 0.9, 1) ,\n"
|
+ "HDR_GET_QUANTILEs(HDR_HISTOGRAM(m1, 2), 0.1, 0.2, 0.3, 0.5, 0.9, 1) ,\n"
|
||||||
+ "HDR_GET_QUANTILEs(HDR_HISTOGRAM(m1, 1, 110, 2), 0.1, 0.2, 0.3, 0.5, 0.9, 1) ,\n"
|
+ "HDR_GET_QUANTILEs(HDR_HISTOGRAM(m1, 1, 110, 2), 0.1, 0.2, 0.3, 0.5, 0.9, 1) ,\n"
|
||||||
+ "HDR_GET_QUANTILEs(HDR_HISTOGRAM(m1, 1, 110, 2, false), 0.1, 0.2, 0.3, 0.5, 0.9, 1) \n"
|
+ "HDR_GET_QUANTILEs(HDR_HISTOGRAM(m1, 1, 110, 2, false), 0.1, 0.2, 0.3, 0.5, 0.9, 1) \n"
|
||||||
+ "from druid.foo";
|
+ "from druid.foo";
|
||||||
final List<Object[]> results =
|
QueryTestBuilder builder = testBuilder().sql(sql).skipVectorize();
|
||||||
sqlLifecycle.runSimple(sql, QUERY_CONTEXT_DEFAULT, DEFAULT_PARAMETERS, authenticationResult).toList();
|
builder.run();
|
||||||
|
QueryTestRunner.QueryResults queryResults = builder.results();
|
||||||
|
List<Object[]> results = queryResults.results;
|
||||||
for (Object[] result : results) {
|
for (Object[] result : results) {
|
||||||
System.out.println(Arrays.toString(result));
|
System.out.println(Arrays.toString(result));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void testSqlQueryGeneHdrArgs2() throws Exception {
|
public void testSqlQueryGeneHdrArgs2() throws Exception {
|
||||||
SqlLifecycle sqlLifecycle = sqlLifecycleFactory.factorize();
|
|
||||||
String sql = "select APPROX_QUANTILE_HDR(m1, 0.1), "
|
String sql = "select APPROX_QUANTILE_HDR(m1, 0.1), "
|
||||||
+ "APPROX_QUANTILE_HDR(m1, 0.1, 2) ,\n"
|
+ "APPROX_QUANTILE_HDR(m1, 0.1, 2) ,\n"
|
||||||
+ "APPROX_QUANTILE_HDR(m1, 0.1, 1, 110, 2) ,\n"
|
+ "APPROX_QUANTILE_HDR(m1, 0.1, 1, 110, 2) ,\n"
|
||||||
+ "APPROX_QUANTILE_HDR(m1, 0.1, 1, 110, 2, false)\n"
|
+ "APPROX_QUANTILE_HDR(m1, 0.1, 1, 110, 2, false)\n"
|
||||||
+ "from druid.foo";
|
+ "from druid.foo";
|
||||||
final List<Object[]> results =
|
QueryTestBuilder builder = testBuilder().sql(sql).skipVectorize();
|
||||||
sqlLifecycle.runSimple(sql, QUERY_CONTEXT_DEFAULT, DEFAULT_PARAMETERS, authenticationResult).toList();
|
builder.run();
|
||||||
for (Object[] result : results) {
|
QueryTestRunner.QueryResults queryResults = builder.results();
|
||||||
System.out.println(Arrays.toString(result));
|
List<Object[]> results = queryResults.results;
|
||||||
}
|
for (Object[] result : results) {
|
||||||
|
System.out.println(Arrays.toString(result));
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void testSqlQueryGeneHdr3() throws Exception {
|
public void testSqlQueryGeneHdr3() throws Exception {
|
||||||
SqlLifecycle sqlLifecycle = sqlLifecycleFactory.factorize();
|
|
||||||
// 函数不区分大小写
|
// 函数不区分大小写
|
||||||
// HDR_HISTOGRAM(hist_m1, 1, 100, 2),
|
// HDR_HISTOGRAM(hist_m1, 1, 100, 2),
|
||||||
//String sql = "select HDR_GET_PERCENTILES(HDR_HISTOGRAM(m1, 1, 100, 2)) from druid.foo";
|
//String sql = "select HDR_GET_PERCENTILES(HDR_HISTOGRAM(m1, 1, 100, 2)) from druid.foo";
|
||||||
//String sql = "select hdr_get_percentiles(hdr_histogram(m1, 1, 100, 2)) from druid.foo";
|
//String sql = "select hdr_get_percentiles(hdr_histogram(m1, 1, 100, 2)) from druid.foo";
|
||||||
String sql = "select hdr_get_percentiles(hdr_histogram(hist_m1, 1, 100, 2)) from druid.foo";
|
String sql = "select hdr_get_percentiles(hdr_histogram(hist_m1, 1, 100, 2)) from druid.foo";
|
||||||
final List<Object[]> results =
|
QueryTestBuilder builder = testBuilder().sql(sql).skipVectorize();
|
||||||
sqlLifecycle.runSimple(sql, QUERY_CONTEXT_DEFAULT, DEFAULT_PARAMETERS, authenticationResult).toList();
|
builder.run();
|
||||||
|
QueryTestRunner.QueryResults queryResults = builder.results();
|
||||||
|
List<Object[]> results = queryResults.results;
|
||||||
for (Object[] result : results) {
|
for (Object[] result : results) {
|
||||||
System.out.println(Arrays.toString(result));
|
System.out.println(Arrays.toString(result));
|
||||||
}
|
}
|
||||||
@@ -326,7 +330,6 @@ public class HdrHistogramQuantileSqlAggregatorTest extends CalciteTestBase {
|
|||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void testSqlQueryQuantiles() throws Exception {
|
public void testSqlQueryQuantiles() throws Exception {
|
||||||
SqlLifecycle sqlLifecycle = sqlLifecycleFactory.factorize();
|
|
||||||
String sql = "SELECT\n"
|
String sql = "SELECT\n"
|
||||||
+ "APPROX_QUANTILE_HDR(m1, 0.01, 1, 100, 2),\n"
|
+ "APPROX_QUANTILE_HDR(m1, 0.01, 1, 100, 2),\n"
|
||||||
+ "APPROX_QUANTILE_HDR(m1, 0.5, 1, 100, 2),\n"
|
+ "APPROX_QUANTILE_HDR(m1, 0.5, 1, 100, 2),\n"
|
||||||
@@ -338,9 +341,10 @@ public class HdrHistogramQuantileSqlAggregatorTest extends CalciteTestBase {
|
|||||||
+ "APPROX_QUANTILE_HDR(m1, 0.999, 1, 100, 2) FILTER(WHERE dim1 = 'abc'),\n"
|
+ "APPROX_QUANTILE_HDR(m1, 0.999, 1, 100, 2) FILTER(WHERE dim1 = 'abc'),\n"
|
||||||
+ "APPROX_QUANTILE_HDR(cnt, 0.5, 1, 100, 2)\n"
|
+ "APPROX_QUANTILE_HDR(cnt, 0.5, 1, 100, 2)\n"
|
||||||
+ "FROM foo";
|
+ "FROM foo";
|
||||||
final List<Object[]> results =
|
QueryTestBuilder builder = testBuilder().sql(sql).skipVectorize();
|
||||||
sqlLifecycle.runSimple(sql, QUERY_CONTEXT_DEFAULT, DEFAULT_PARAMETERS, authenticationResult).toList();
|
builder.run();
|
||||||
System.out.println(sql);
|
QueryTestRunner.QueryResults queryResults = builder.results();
|
||||||
|
List<Object[]> results = queryResults.results;
|
||||||
for (Object[] result : results) {
|
for (Object[] result : results) {
|
||||||
System.out.println(Arrays.toString(result));
|
System.out.println(Arrays.toString(result));
|
||||||
}
|
}
|
||||||
@@ -348,7 +352,6 @@ public class HdrHistogramQuantileSqlAggregatorTest extends CalciteTestBase {
|
|||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void testSqlQueryQuantilesOnComplexColumn() throws Exception {
|
public void testSqlQueryQuantilesOnComplexColumn() throws Exception {
|
||||||
SqlLifecycle sqlLifecycle = sqlLifecycleFactory.factorize();
|
|
||||||
String sql = "SELECT\n"
|
String sql = "SELECT\n"
|
||||||
+ "APPROX_QUANTILE_HDR(hist_m1, 0.01, 1, 100, 2),\n"
|
+ "APPROX_QUANTILE_HDR(hist_m1, 0.01, 1, 100, 2),\n"
|
||||||
+ "APPROX_QUANTILE_HDR(hist_m1, 0.5, 1, 100, 2),\n"
|
+ "APPROX_QUANTILE_HDR(hist_m1, 0.5, 1, 100, 2),\n"
|
||||||
@@ -358,9 +361,10 @@ public class HdrHistogramQuantileSqlAggregatorTest extends CalciteTestBase {
|
|||||||
+ "APPROX_QUANTILE_HDR(hist_m1, 0.999, 1, 100, 2) FILTER(WHERE dim1 <> 'abc'),\n"
|
+ "APPROX_QUANTILE_HDR(hist_m1, 0.999, 1, 100, 2) FILTER(WHERE dim1 <> 'abc'),\n"
|
||||||
+ "APPROX_QUANTILE_HDR(hist_m1, 0.999, 1, 100, 2) FILTER(WHERE dim1 = 'abc')\n"
|
+ "APPROX_QUANTILE_HDR(hist_m1, 0.999, 1, 100, 2) FILTER(WHERE dim1 = 'abc')\n"
|
||||||
+ "FROM foo";
|
+ "FROM foo";
|
||||||
final List<Object[]> results =
|
QueryTestBuilder builder = testBuilder().sql(sql).skipVectorize();
|
||||||
sqlLifecycle.runSimple(sql, QUERY_CONTEXT_DEFAULT, DEFAULT_PARAMETERS, authenticationResult).toList();
|
builder.run();
|
||||||
System.out.println(sql);
|
QueryTestRunner.QueryResults queryResults = builder.results();
|
||||||
|
List<Object[]> results = queryResults.results;
|
||||||
for (Object[] result : results) {
|
for (Object[] result : results) {
|
||||||
System.out.println(Arrays.toString(result));
|
System.out.println(Arrays.toString(result));
|
||||||
}
|
}
|
||||||
@@ -373,7 +377,6 @@ public class HdrHistogramQuantileSqlAggregatorTest extends CalciteTestBase {
|
|||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void testQuantileOnFloatAndLongs() throws Exception {
|
public void testQuantileOnFloatAndLongs() throws Exception {
|
||||||
SqlLifecycle sqlLifecycle = sqlLifecycleFactory.factorize();
|
|
||||||
String sql = "SELECT\n"
|
String sql = "SELECT\n"
|
||||||
+ "APPROX_QUANTILE_HDR(m1, 0.01, 1, 100, 2),\n"
|
+ "APPROX_QUANTILE_HDR(m1, 0.01, 1, 100, 2),\n"
|
||||||
+ "APPROX_QUANTILE_HDR(m1, 0.5, 1, 100, 2),\n"
|
+ "APPROX_QUANTILE_HDR(m1, 0.5, 1, 100, 2),\n"
|
||||||
@@ -385,60 +388,55 @@ public class HdrHistogramQuantileSqlAggregatorTest extends CalciteTestBase {
|
|||||||
+ "APPROX_QUANTILE_HDR(m1, 0.999, 1, 100, 2) FILTER(WHERE dim1 = 'abc'),\n"
|
+ "APPROX_QUANTILE_HDR(m1, 0.999, 1, 100, 2) FILTER(WHERE dim1 = 'abc'),\n"
|
||||||
+ "APPROX_QUANTILE_HDR(cnt, 0.5, 1, 100, 2)\n"
|
+ "APPROX_QUANTILE_HDR(cnt, 0.5, 1, 100, 2)\n"
|
||||||
+ "FROM foo";
|
+ "FROM foo";
|
||||||
final List<Object[]> results =
|
QueryTestBuilder builder = testBuilder().sql(sql).skipVectorize();
|
||||||
sqlLifecycle.runSimple(sql, QUERY_CONTEXT_DEFAULT, DEFAULT_PARAMETERS, authenticationResult).toList();
|
builder = builder.expectedQueries(Collections.singletonList(Druids.newTimeseriesQueryBuilder()
|
||||||
System.out.println(sql);
|
.dataSource(CalciteTests.DATASOURCE1)
|
||||||
|
.intervals(new MultipleIntervalSegmentSpec(ImmutableList.of(Filtration.eternity())))
|
||||||
|
.granularity(Granularities.ALL)
|
||||||
|
.virtualColumns(
|
||||||
|
new ExpressionVirtualColumn(
|
||||||
|
"v0",
|
||||||
|
"(\"m1\" * 2)",
|
||||||
|
ColumnType.LONG,
|
||||||
|
TestExprMacroTable.INSTANCE
|
||||||
|
)
|
||||||
|
)
|
||||||
|
.aggregators(ImmutableList.of(
|
||||||
|
new HdrHistogramAggregatorFactory("a0:agg", "m1", 1L, 100L, 2, true),
|
||||||
|
new HdrHistogramAggregatorFactory("a4:agg", "v0", 1L, 100L, 2, true),
|
||||||
|
new FilteredAggregatorFactory(
|
||||||
|
new HdrHistogramAggregatorFactory("a5:agg", "m1", 1L, 100L, 2, true),
|
||||||
|
new SelectorDimFilter("dim1", "abc", null)
|
||||||
|
),
|
||||||
|
new FilteredAggregatorFactory(
|
||||||
|
new HdrHistogramAggregatorFactory("a6:agg", "m1", 1L, 100L, 2, true),
|
||||||
|
new NotDimFilter(new SelectorDimFilter("dim1", "abc", null))
|
||||||
|
),
|
||||||
|
new HdrHistogramAggregatorFactory("a8:agg", "cnt", 1L, 100L, 2, true)
|
||||||
|
))
|
||||||
|
.postAggregators(
|
||||||
|
new HdrHistogramToQuantilePostAggregator("a0", "a0:agg", 0.01f),
|
||||||
|
new HdrHistogramToQuantilePostAggregator("a1", "a0:agg", 0.50f),
|
||||||
|
new HdrHistogramToQuantilePostAggregator("a2", "a0:agg", 0.98f),
|
||||||
|
new HdrHistogramToQuantilePostAggregator("a3", "a0:agg", 0.99f),
|
||||||
|
new HdrHistogramToQuantilePostAggregator("a4", "a4:agg", 0.97f),
|
||||||
|
new HdrHistogramToQuantilePostAggregator("a5", "a5:agg", 0.99f),
|
||||||
|
new HdrHistogramToQuantilePostAggregator("a6", "a6:agg", 0.999f),
|
||||||
|
new HdrHistogramToQuantilePostAggregator("a7", "a5:agg", 0.999f),
|
||||||
|
new HdrHistogramToQuantilePostAggregator("a8", "a8:agg", 0.50f)
|
||||||
|
)
|
||||||
|
.context(QUERY_CONTEXT_DEFAULT)
|
||||||
|
.build()));
|
||||||
|
builder.run();
|
||||||
|
QueryTestRunner.QueryResults queryResults = builder.results();
|
||||||
|
List<Object[]> results = queryResults.results;
|
||||||
for (Object[] result : results) {
|
for (Object[] result : results) {
|
||||||
System.out.println(Arrays.toString(result));
|
System.out.println(Arrays.toString(result));
|
||||||
}
|
}
|
||||||
|
|
||||||
// Verify query
|
|
||||||
Assert.assertEquals(
|
|
||||||
Druids.newTimeseriesQueryBuilder()
|
|
||||||
.dataSource(CalciteTests.DATASOURCE1)
|
|
||||||
.intervals(new MultipleIntervalSegmentSpec(ImmutableList.of(Filtration.eternity())))
|
|
||||||
.granularity(Granularities.ALL)
|
|
||||||
.virtualColumns(
|
|
||||||
new ExpressionVirtualColumn(
|
|
||||||
"v0",
|
|
||||||
"(\"m1\" * 2)",
|
|
||||||
ValueType.LONG,
|
|
||||||
TestExprMacroTable.INSTANCE
|
|
||||||
)
|
|
||||||
)
|
|
||||||
.aggregators(ImmutableList.of(
|
|
||||||
new HdrHistogramAggregatorFactory("a0:agg", "m1", 1L, 100L, 2, true),
|
|
||||||
new HdrHistogramAggregatorFactory("a4:agg", "v0", 1L, 100L, 2, true),
|
|
||||||
new FilteredAggregatorFactory(
|
|
||||||
new HdrHistogramAggregatorFactory("a5:agg", "m1", 1L, 100L, 2, true),
|
|
||||||
new SelectorDimFilter("dim1", "abc", null)
|
|
||||||
),
|
|
||||||
new FilteredAggregatorFactory(
|
|
||||||
new HdrHistogramAggregatorFactory("a6:agg", "m1", 1L, 100L, 2, true),
|
|
||||||
new NotDimFilter(new SelectorDimFilter("dim1", "abc", null))
|
|
||||||
),
|
|
||||||
new HdrHistogramAggregatorFactory("a8:agg", "cnt", 1L, 100L, 2, true)
|
|
||||||
))
|
|
||||||
.postAggregators(
|
|
||||||
new HdrHistogramToQuantilePostAggregator("a0", "a0:agg", 0.01f),
|
|
||||||
new HdrHistogramToQuantilePostAggregator("a1", "a0:agg", 0.50f),
|
|
||||||
new HdrHistogramToQuantilePostAggregator("a2", "a0:agg", 0.98f),
|
|
||||||
new HdrHistogramToQuantilePostAggregator("a3", "a0:agg", 0.99f),
|
|
||||||
new HdrHistogramToQuantilePostAggregator("a4", "a4:agg", 0.97f),
|
|
||||||
new HdrHistogramToQuantilePostAggregator("a5", "a5:agg", 0.99f),
|
|
||||||
new HdrHistogramToQuantilePostAggregator("a6", "a6:agg", 0.999f),
|
|
||||||
new HdrHistogramToQuantilePostAggregator("a7", "a5:agg", 0.999f),
|
|
||||||
new HdrHistogramToQuantilePostAggregator("a8", "a8:agg", 0.50f)
|
|
||||||
)
|
|
||||||
.context(ImmutableMap.of("skipEmptyBuckets", true, PlannerContext.CTX_SQL_QUERY_ID, "dummy"))
|
|
||||||
.build(),
|
|
||||||
Iterables.getOnlyElement(queryLogHook.getRecordedQueries())
|
|
||||||
);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void testQuantileOnComplexColumn() throws Exception{
|
public void testQuantileOnComplexColumn() throws Exception{
|
||||||
SqlLifecycle sqlLifecycle = sqlLifecycleFactory.factorize();
|
|
||||||
String sql = "SELECT\n"
|
String sql = "SELECT\n"
|
||||||
+ "APPROX_QUANTILE_HDR(hist_m1, 0.01, 1, 100, 2),\n"
|
+ "APPROX_QUANTILE_HDR(hist_m1, 0.01, 1, 100, 2),\n"
|
||||||
+ "APPROX_QUANTILE_HDR(hist_m1, 0.5, 1, 100, 2),\n"
|
+ "APPROX_QUANTILE_HDR(hist_m1, 0.5, 1, 100, 2),\n"
|
||||||
@@ -448,43 +446,42 @@ public class HdrHistogramQuantileSqlAggregatorTest extends CalciteTestBase {
|
|||||||
+ "APPROX_QUANTILE_HDR(hist_m1, 0.999, 1, 100, 2) FILTER(WHERE dim1 <> 'abc'),\n"
|
+ "APPROX_QUANTILE_HDR(hist_m1, 0.999, 1, 100, 2) FILTER(WHERE dim1 <> 'abc'),\n"
|
||||||
+ "APPROX_QUANTILE_HDR(hist_m1, 0.999, 1, 100, 2) FILTER(WHERE dim1 = 'abc')\n"
|
+ "APPROX_QUANTILE_HDR(hist_m1, 0.999, 1, 100, 2) FILTER(WHERE dim1 = 'abc')\n"
|
||||||
+ "FROM foo";
|
+ "FROM foo";
|
||||||
final List<Object[]> results =
|
QueryTestBuilder builder = testBuilder().sql(sql).skipVectorize();
|
||||||
sqlLifecycle.runSimple(sql, QUERY_CONTEXT_DEFAULT, DEFAULT_PARAMETERS, authenticationResult).toList();
|
builder = builder.expectedQueries(Collections.singletonList(Druids.newTimeseriesQueryBuilder()
|
||||||
System.out.println(sql);
|
.dataSource(CalciteTests.DATASOURCE1)
|
||||||
|
.intervals(new MultipleIntervalSegmentSpec(ImmutableList.of(Filtration.eternity())))
|
||||||
|
.granularity(Granularities.ALL)
|
||||||
|
.aggregators(ImmutableList.of(
|
||||||
|
new HdrHistogramMergeAggregatorFactory("a0:agg", "hist_m1", 1L, 100L, 2, true),
|
||||||
|
new FilteredAggregatorFactory(
|
||||||
|
new HdrHistogramMergeAggregatorFactory("a4:agg", "hist_m1", 1L, 100L, 2, true),
|
||||||
|
new SelectorDimFilter("dim1", "abc", null)
|
||||||
|
),
|
||||||
|
new FilteredAggregatorFactory(
|
||||||
|
new HdrHistogramMergeAggregatorFactory("a5:agg", "hist_m1", 1L, 100L, 2, true),
|
||||||
|
new NotDimFilter(new SelectorDimFilter("dim1", "abc", null))
|
||||||
|
)
|
||||||
|
))
|
||||||
|
.postAggregators(
|
||||||
|
new HdrHistogramToQuantilePostAggregator("a0", "a0:agg", 0.01f),
|
||||||
|
new HdrHistogramToQuantilePostAggregator("a1", "a0:agg", 0.50f),
|
||||||
|
new HdrHistogramToQuantilePostAggregator("a2", "a0:agg", 0.98f),
|
||||||
|
new HdrHistogramToQuantilePostAggregator("a3", "a0:agg", 0.99f),
|
||||||
|
new HdrHistogramToQuantilePostAggregator("a4", "a4:agg", 0.99f),
|
||||||
|
new HdrHistogramToQuantilePostAggregator("a5", "a5:agg", 0.999f),
|
||||||
|
new HdrHistogramToQuantilePostAggregator("a6", "a4:agg", 0.999f)
|
||||||
|
)
|
||||||
|
.context(QUERY_CONTEXT_DEFAULT)
|
||||||
|
.build()));
|
||||||
|
builder.run();
|
||||||
|
QueryTestRunner.QueryResults queryResults = builder.results();
|
||||||
|
List<Object[]> results = queryResults.results;
|
||||||
for (Object[] result : results) {
|
for (Object[] result : results) {
|
||||||
System.out.println(Arrays.toString(result));
|
System.out.println(Arrays.toString(result));
|
||||||
}
|
}
|
||||||
|
|
||||||
// Verify query
|
|
||||||
Assert.assertEquals(
|
|
||||||
Druids.newTimeseriesQueryBuilder()
|
|
||||||
.dataSource(CalciteTests.DATASOURCE1)
|
|
||||||
.intervals(new MultipleIntervalSegmentSpec(ImmutableList.of(Filtration.eternity())))
|
|
||||||
.granularity(Granularities.ALL)
|
|
||||||
.aggregators(ImmutableList.of(
|
|
||||||
new HdrHistogramMergeAggregatorFactory("a0:agg", "hist_m1", 1L, 100L, 2, true),
|
|
||||||
new FilteredAggregatorFactory(
|
|
||||||
new HdrHistogramMergeAggregatorFactory("a4:agg", "hist_m1", 1L, 100L, 2, true),
|
|
||||||
new SelectorDimFilter("dim1", "abc", null)
|
|
||||||
),
|
|
||||||
new FilteredAggregatorFactory(
|
|
||||||
new HdrHistogramMergeAggregatorFactory("a5:agg", "hist_m1", 1L, 100L, 2, true),
|
|
||||||
new NotDimFilter(new SelectorDimFilter("dim1", "abc", null))
|
|
||||||
)
|
|
||||||
))
|
|
||||||
.postAggregators(
|
|
||||||
new HdrHistogramToQuantilePostAggregator("a0", "a0:agg", 0.01f),
|
|
||||||
new HdrHistogramToQuantilePostAggregator("a1", "a0:agg", 0.50f),
|
|
||||||
new HdrHistogramToQuantilePostAggregator("a2", "a0:agg", 0.98f),
|
|
||||||
new HdrHistogramToQuantilePostAggregator("a3", "a0:agg", 0.99f),
|
|
||||||
new HdrHistogramToQuantilePostAggregator("a4", "a4:agg", 0.99f),
|
|
||||||
new HdrHistogramToQuantilePostAggregator("a5", "a5:agg", 0.999f),
|
|
||||||
new HdrHistogramToQuantilePostAggregator("a6", "a4:agg", 0.999f)
|
|
||||||
)
|
|
||||||
.context(ImmutableMap.of("skipEmptyBuckets", true, PlannerContext.CTX_SQL_QUERY_ID, "dummy"))
|
|
||||||
.build(),
|
|
||||||
Iterables.getOnlyElement(queryLogHook.getRecordedQueries())
|
|
||||||
);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
private static PostAggregator makeFieldAccessPostAgg(String name) {
|
private static PostAggregator makeFieldAccessPostAgg(String name) {
|
||||||
|
|||||||
@@ -170,7 +170,8 @@ public class HllAggregatorFactory extends AggregatorFactory {
|
|||||||
|
|
||||||
@Override
|
@Override
|
||||||
public ColumnType getResultType() {
|
public ColumnType getResultType() {
|
||||||
return round ? ColumnType.LONG : ColumnType.DOUBLE;
|
//return round ? ColumnType.LONG : ColumnType.DOUBLE;
|
||||||
|
return getIntermediateType();
|
||||||
}
|
}
|
||||||
|
|
||||||
@Nullable
|
@Nullable
|
||||||
|
|||||||
Reference in New Issue
Block a user