gfwleak/galaxy-platform-algorithm-druid-extensions
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'main' into 'develop'

Browse Source 
Main

See merge request galaxy/platform/algorithm/druid-extensions!3
This commit is contained in:
李奉超
2024-07-10 01:29:58 +00:00
parent 69cd9e3223 25ab1b3f9d
commit 38c22db84d
23 changed files with 2075 additions and 1724 deletions
Download Patch File Download Diff File Expand all files Collapse all files

21
druid-hdrhistogram/pom.xml
View File

@@ -5,7 +5,7 @@
<modelVersion>4.0.0</modelVersion> <modelVersion>4.0.0</modelVersion>
<groupId>org.apache.druid.extensions</groupId> <groupId>org.apache.druid.extensions</groupId>
<artifactId>druid-hdrhistogram_0.18.1</artifactId> <artifactId>druid-hdrhistogram_26.0.0</artifactId>
<name>druid-hdrhistogram</name> <name>druid-hdrhistogram</name>
<version>1.0-SNAPSHOT</version> <version>1.0-SNAPSHOT</version>
@@ -14,7 +14,7 @@
<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding> <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
<maven.compiler.source>1.8</maven.compiler.source> <maven.compiler.source>1.8</maven.compiler.source>
<maven.compiler.target>1.8</maven.compiler.target> <maven.compiler.target>1.8</maven.compiler.target>
<druid.version>0.18.1</druid.version> <druid.version>26.0.0</druid.version>
</properties> </properties>
<dependencies> <dependencies>
@@ -45,6 +45,13 @@
</dependency> </dependency>
<!-- Tests --> <!-- Tests -->
<dependency>
<groupId>org.easymock</groupId>
<artifactId>easymock</artifactId>
<version>4.3</version>
<scope>test</scope>
</dependency>
<dependency> <dependency>
<groupId>org.apache.druid</groupId> <groupId>org.apache.druid</groupId>
<artifactId>druid-processing</artifactId> <artifactId>druid-processing</artifactId>
@@ -54,9 +61,17 @@
</dependency> </dependency>
<dependency> <dependency>
<groupId>org.apache.druid</groupId> <groupId>org.apache.druid</groupId>
<artifactId>druid-benchmarks</artifactId> <artifactId>druid-server</artifactId>
<version>${druid.version}</version> <version>${druid.version}</version>
<scope>test</scope> <scope>test</scope>
<type>test-jar</type>
</dependency>
<dependency>
<groupId>org.apache.druid</groupId>
<artifactId>druid-sql</artifactId>
<version>${druid.version}</version>
<type>test-jar</type>
<scope>test</scope>
</dependency> </dependency>
<dependency> <dependency>
<groupId>junit</groupId> <groupId>junit</groupId>

669
druid-hdrhistogram/src/main/java/org/apache/druid/query/aggregation/sketch/HdrHistogram/HdrHistogramAggregatorFactory.java
View File

@@ -1,321 +1,348 @@
package org.apache.druid.query.aggregation.sketch.HdrHistogram; package org.apache.druid.query.aggregation.sketch.HdrHistogram;
import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonProperty;
import org.HdrHistogram.DirectHistogram; import org.HdrHistogram.HistogramSketch;
import org.HdrHistogram.Histogram; import org.HdrHistogram.HistogramUnion;
import org.HdrHistogram.HistogramSketch; import org.apache.druid.java.util.common.IAE;
import org.HdrHistogram.HistogramUnion; import org.apache.druid.query.aggregation.*;
import org.apache.druid.java.util.common.IAE; import org.apache.druid.query.cache.CacheKeyBuilder;
import org.apache.druid.query.aggregation.*; import org.apache.druid.segment.ColumnSelectorFactory;
import org.apache.druid.query.cache.CacheKeyBuilder; import org.apache.druid.segment.ColumnValueSelector;
import org.apache.druid.segment.ColumnSelectorFactory; import org.apache.druid.segment.column.ColumnType;
import org.apache.druid.segment.ColumnValueSelector;
import javax.annotation.Nullable;
import javax.annotation.Nullable; import java.util.Collections;
import java.util.Collections; import java.util.Comparator;
import java.util.Comparator; import java.util.List;
import java.util.List; import java.util.Objects;
import java.util.Objects;
public class HdrHistogramAggregatorFactory extends AggregatorFactory {
public class HdrHistogramAggregatorFactory extends AggregatorFactory { public static final long DEFAULT_LOWEST = 1;
public static final long DEFAULT_LOWEST = 1; public static final long DEFAULT_HIGHEST = 2;
public static final long DEFAULT_HIGHEST = 2; public static final int DEFAULT_SIGNIFICANT = 1;
public static final int DEFAULT_SIGNIFICANT = 3; public static final boolean DEFAULT_AUTO_RESIZE = true;
public static final boolean DEFAULT_AUTO_RESIZE = true; public static final long BUFFER_AUTO_RESIZE_HIGHEST = 100000000L * 1000000L;
public static final long BUFFER_AUTO_RESIZE_HIGHEST = 100000000L * 1000000L; public static final Comparator<HistogramSketch> COMPARATOR =
public static final Comparator<HistogramSketch> COMPARATOR = Comparator.nullsFirst(Comparator.comparingLong(HistogramSketch::getTotalCount));
Comparator.nullsFirst(Comparator.comparingLong(HistogramSketch::getTotalCount));
protected final String name;
protected final String name; protected final String fieldName;
protected final String fieldName; protected final long lowestDiscernibleValue;
protected final long lowestDiscernibleValue; protected final long highestTrackableValue;
protected final long highestTrackableValue; protected final int numberOfSignificantValueDigits;
protected final int numberOfSignificantValueDigits; protected final boolean autoResize; //默认是false
protected final boolean autoResize; //默认是false protected final int updatableSerializationBytes;
public HdrHistogramAggregatorFactory( public HdrHistogramAggregatorFactory(
@JsonProperty("name") String name, @JsonProperty("name") String name,
@JsonProperty("fieldName") String fieldName, @JsonProperty("fieldName") String fieldName,
@JsonProperty("lowestDiscernibleValue") @Nullable Long lowestDiscernibleValue, @JsonProperty("lowestDiscernibleValue") @Nullable Long lowestDiscernibleValue,
@JsonProperty("highestTrackableValue") @Nullable Long highestTrackableValue, @JsonProperty("highestTrackableValue") @Nullable Long highestTrackableValue,
@JsonProperty("numberOfSignificantValueDigits") @Nullable Integer numberOfSignificantValueDigits, @JsonProperty("numberOfSignificantValueDigits") @Nullable Integer numberOfSignificantValueDigits,
@JsonProperty("autoResize") @Nullable Boolean autoResize @JsonProperty("autoResize") @Nullable Boolean autoResize
) { ) {
if (name == null) { if (name == null) {
throw new IAE("Must have a valid, non-null aggregator name"); throw new IAE("Must have a valid, non-null aggregator name");
} }
if (fieldName == null) { if (fieldName == null) {
throw new IAE("Parameter fieldName must be specified"); throw new IAE("Parameter fieldName must be specified");
} }
if(lowestDiscernibleValue == null){ if(lowestDiscernibleValue == null){
lowestDiscernibleValue = DEFAULT_LOWEST; lowestDiscernibleValue = DEFAULT_LOWEST;
} }
// Verify argument validity // Verify argument validity
if (lowestDiscernibleValue < 1) { if (lowestDiscernibleValue < 1) {
throw new IAE("lowestDiscernibleValue must be >= 1"); throw new IAE("lowestDiscernibleValue must be >= 1");
} }
if (lowestDiscernibleValue > Long.MAX_VALUE / 2) { if (lowestDiscernibleValue > Long.MAX_VALUE / 2) {
// prevent subsequent multiplication by 2 for highestTrackableValue check from overflowing // prevent subsequent multiplication by 2 for highestTrackableValue check from overflowing
throw new IAE("lowestDiscernibleValue must be <= Long.MAX_VALUE / 2"); throw new IAE("lowestDiscernibleValue must be <= Long.MAX_VALUE / 2");
} }
if(highestTrackableValue == null){ if(highestTrackableValue == null){
highestTrackableValue = DEFAULT_HIGHEST; highestTrackableValue = DEFAULT_HIGHEST;
} }
if (highestTrackableValue < 2L * lowestDiscernibleValue) { if (highestTrackableValue < 2L * lowestDiscernibleValue) {
throw new IAE("highestTrackableValue must be >= 2 * lowestDiscernibleValue"); throw new IAE("highestTrackableValue must be >= 2 * lowestDiscernibleValue");
} }
if(numberOfSignificantValueDigits == null){ if(numberOfSignificantValueDigits == null){
numberOfSignificantValueDigits = DEFAULT_SIGNIFICANT; numberOfSignificantValueDigits = DEFAULT_SIGNIFICANT;
} }
if ((numberOfSignificantValueDigits < 0) || (numberOfSignificantValueDigits > 5)) { if ((numberOfSignificantValueDigits < 0) || (numberOfSignificantValueDigits > 5)) {
throw new IAE("numberOfSignificantValueDigits must be between 0 and 5"); throw new IAE("numberOfSignificantValueDigits must be between 0 and 5");
} }
if(autoResize == null){ if(autoResize == null){
autoResize = DEFAULT_AUTO_RESIZE; autoResize = DEFAULT_AUTO_RESIZE;
} }
this.name = name; this.name = name;
this.fieldName = fieldName; this.fieldName = fieldName;
this.lowestDiscernibleValue = lowestDiscernibleValue; this.lowestDiscernibleValue = lowestDiscernibleValue;
this.highestTrackableValue = highestTrackableValue; this.highestTrackableValue = highestTrackableValue;
this.numberOfSignificantValueDigits = numberOfSignificantValueDigits; this.numberOfSignificantValueDigits = numberOfSignificantValueDigits;
this.autoResize = autoResize; this.autoResize = autoResize;
} this.updatableSerializationBytes = getUpdatableSerializationBytes();
}
@Override
public Aggregator factorize(ColumnSelectorFactory metricFactory) { @Override
return new HdrHistogramAggregator( public Aggregator factorize(ColumnSelectorFactory metricFactory) {
metricFactory.makeColumnValueSelector(fieldName), return new HdrHistogramAggregator(
lowestDiscernibleValue, metricFactory.makeColumnValueSelector(fieldName),
highestTrackableValue, lowestDiscernibleValue,
numberOfSignificantValueDigits, highestTrackableValue,
autoResize numberOfSignificantValueDigits,
); autoResize
} );
}
@Override
public BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory) { @Override
return new HdrHistogramBufferAggregator( public BufferAggregator factorizeBuffered(ColumnSelectorFactory metricFactory) {
metricFactory.makeColumnValueSelector(fieldName), return new HdrHistogramBufferAggregator(
lowestDiscernibleValue, metricFactory.makeColumnValueSelector(fieldName),
highestTrackableValue, lowestDiscernibleValue,
numberOfSignificantValueDigits, highestTrackableValue,
autoResize, numberOfSignificantValueDigits,
getMaxIntermediateSize() autoResize,
); getMaxIntermediateSize()
} );
}
@Override
public Comparator getComparator() { @Override
return COMPARATOR; public Comparator getComparator() {
} return COMPARATOR;
}
@Override
public Object combine(Object lhs, Object rhs) { @Override
if(lhs == null){ public Object combine(Object lhs, Object rhs) {
return rhs; if(lhs == null){
}else if(rhs == null){ return rhs;
return lhs; }else if(rhs == null){
}else{ return lhs;
final HistogramUnion union = new HistogramUnion(lowestDiscernibleValue,highestTrackableValue,numberOfSignificantValueDigits,autoResize); }else{
union.update((HistogramSketch) lhs); final HistogramUnion union = new HistogramUnion(lowestDiscernibleValue,highestTrackableValue,numberOfSignificantValueDigits,autoResize);
union.update((HistogramSketch) rhs); union.update((HistogramSketch) lhs);
HistogramSketch result = union.getResult(); union.update((HistogramSketch) rhs);
return result; HistogramSketch result = union.getResult();
} return result;
} }
}
@Override
public AggregateCombiner makeAggregateCombiner() { @Override
return new ObjectAggregateCombiner<HistogramSketch>() { public AggregateCombiner makeAggregateCombiner() {
private HistogramUnion union = null; return new ObjectAggregateCombiner<HistogramSketch>() {
private HistogramUnion union = null;
@Override
public void reset(ColumnValueSelector selector) { @Override
//union.reset(); public void reset(ColumnValueSelector selector) {
union = null; //union.reset();
fold(selector); union = null;
} fold(selector);
}
@Override
public void fold(ColumnValueSelector selector) { @Override
HistogramSketch h = (HistogramSketch) selector.getObject(); public void fold(ColumnValueSelector selector) {
if(h != null){ HistogramSketch h = (HistogramSketch) selector.getObject();
if(union == null){ if(h != null){
union = new HistogramUnion(lowestDiscernibleValue,highestTrackableValue,numberOfSignificantValueDigits,autoResize); if(union == null){
} union = new HistogramUnion(lowestDiscernibleValue,highestTrackableValue,numberOfSignificantValueDigits,autoResize);
union.update(h); }
} union.update(h);
} }
}
@Override
public Class<HistogramSketch> classOfObject() { @Override
return HistogramSketch.class; public Class<HistogramSketch> classOfObject() {
} return HistogramSketch.class;
}
@Nullable
@Override @Nullable
public HistogramSketch getObject() { @Override
if(union == null){ public HistogramSketch getObject() {
return null; if(union == null){
}else{ return null;
HistogramSketch result = union.getResult(); }else{
/*if(result.getTotalCount() == 0){ HistogramSketch result = union.getResult();
return null; /*if(result.getTotalCount() == 0){
}*/ return null;
return result; }*/
} return result;
} }
}; }
} };
}
/*public Histogram geneHistogram() {
Histogram histogram = new Histogram(lowestDiscernibleValue, highestTrackableValue, numberOfSignificantValueDigits); /*public Histogram geneHistogram() {
histogram.setAutoResize(autoResize); Histogram histogram = new Histogram(lowestDiscernibleValue, highestTrackableValue, numberOfSignificantValueDigits);
return histogram; histogram.setAutoResize(autoResize);
}*/ return histogram;
}*/
@Override
public AggregatorFactory getCombiningFactory() { @Override
return new HdrHistogramMergeAggregatorFactory(name, name, lowestDiscernibleValue, highestTrackableValue, numberOfSignificantValueDigits, autoResize); public AggregatorFactory getCombiningFactory() {
} return new HdrHistogramMergeAggregatorFactory(name, name, lowestDiscernibleValue, highestTrackableValue, numberOfSignificantValueDigits, autoResize);
}
@Override
public AggregatorFactory getMergingFactory(AggregatorFactory other) throws AggregatorFactoryNotMergeableException { @Override
if (other.getName().equals(this.getName()) && other instanceof HdrHistogramAggregatorFactory) { public AggregatorFactory getMergingFactory(AggregatorFactory other) throws AggregatorFactoryNotMergeableException {
HdrHistogramAggregatorFactory castedOther = (HdrHistogramAggregatorFactory) other; if (other.getName().equals(this.getName()) && other instanceof HdrHistogramAggregatorFactory) {
HdrHistogramAggregatorFactory castedOther = (HdrHistogramAggregatorFactory) other;
return new HdrHistogramMergeAggregatorFactory(name, name,
Math.min(lowestDiscernibleValue, castedOther.lowestDiscernibleValue), return new HdrHistogramMergeAggregatorFactory(name, name,
Math.max(highestTrackableValue, castedOther.highestTrackableValue), Math.min(lowestDiscernibleValue, castedOther.lowestDiscernibleValue),
Math.max(numberOfSignificantValueDigits, castedOther.numberOfSignificantValueDigits), Math.max(highestTrackableValue, castedOther.highestTrackableValue),
autoResize || castedOther.autoResize Math.max(numberOfSignificantValueDigits, castedOther.numberOfSignificantValueDigits),
); autoResize || castedOther.autoResize
} else { );
throw new AggregatorFactoryNotMergeableException(this, other); } else {
} throw new AggregatorFactoryNotMergeableException(this, other);
} }
}
@Override
public List<AggregatorFactory> getRequiredColumns() { @Override
return Collections.singletonList( public List<AggregatorFactory> getRequiredColumns() {
new HdrHistogramAggregatorFactory( return Collections.singletonList(
fieldName, new HdrHistogramAggregatorFactory(
fieldName, fieldName,
lowestDiscernibleValue, highestTrackableValue, numberOfSignificantValueDigits, autoResize fieldName,
) lowestDiscernibleValue, highestTrackableValue, numberOfSignificantValueDigits, autoResize
); )
} );
}
@Override
public Object deserialize(Object object) { @Override
return HistogramUtils.deserializeHistogram(object); public AggregatorFactory withName(String newName) {
} return new HdrHistogramAggregatorFactory(newName, fieldName, lowestDiscernibleValue, highestTrackableValue, numberOfSignificantValueDigits, autoResize);
}
@Nullable
@Override @Override
public Object finalizeComputation(@Nullable Object object) { public Object deserialize(Object object) {
return object == null ? null : ((HistogramSketch) object).getTotalCount(); if (object == null) {
} return null;
}
@Override return HistogramUtils.deserializeHistogram(object);
@JsonProperty }
public String getName() {
return name; @Override
} public ColumnType getResultType() {
//return ColumnType.LONG;
@JsonProperty return getIntermediateType();
public String getFieldName() { }
return fieldName;
} @Nullable
@Override
@JsonProperty public Object finalizeComputation(@Nullable Object object) {
public long getLowestDiscernibleValue() { //return object == null ? null : ((HistogramSketch) object).getTotalCount();
return lowestDiscernibleValue; return object;
} }
@JsonProperty @Override
public long getHighestTrackableValue() { @JsonProperty
return highestTrackableValue; public String getName() {
} return name;
}
@JsonProperty
public int getNumberOfSignificantValueDigits() { @JsonProperty
return numberOfSignificantValueDigits; public String getFieldName() {
} return fieldName;
}
@JsonProperty
public boolean isAutoResize() { @JsonProperty
return autoResize; public long getLowestDiscernibleValue() {
} return lowestDiscernibleValue;
}
@Override
public String getTypeName() { @JsonProperty
return HdrHistogramModule.HDRHISTOGRAM_TYPE_NAME; public long getHighestTrackableValue() {
} return highestTrackableValue;
}
@Override
public List<String> requiredFields() { @JsonProperty
return Collections.singletonList(fieldName); public int getNumberOfSignificantValueDigits() {
} return numberOfSignificantValueDigits;
}
@Override @JsonProperty
public int getMaxIntermediateSize() { public boolean isAutoResize() {
if(!autoResize){ return autoResize;
/*Histogram histogram = new Histogram(lowestDiscernibleValue, highestTrackableValue, numberOfSignificantValueDigits); }
histogram.setAutoResize(autoResize);
return histogram.getNeededByteBufferCapacity();*/ /*
return HistogramSketch.getUpdatableSerializationBytes(lowestDiscernibleValue, highestTrackableValue, numberOfSignificantValueDigits); 没这个方法了, 新版本需要实现getIntermediateType方法
}else{ @Override
//return (1 << 10) * 512; public String getTypeName() {
return HistogramSketch.getUpdatableSerializationBytes(lowestDiscernibleValue, BUFFER_AUTO_RESIZE_HIGHEST, numberOfSignificantValueDigits); return HdrHistogramModule.HDRHISTOGRAM_TYPE_NAME;
} }*/
}
@Override
@Override public ColumnType getIntermediateType() {
public byte[] getCacheKey() { return HdrHistogramModule.TYPE;
return new CacheKeyBuilder(HdrHistogramModule.CACHE_TYPE_ID_OFFSET).appendByte(HdrHistogramModule.QUANTILES_HDRHISTOGRAM_BUILD_CACHE_TYPE_ID) }
.appendString(name).appendString(fieldName)
.appendDouble(lowestDiscernibleValue).appendDouble(highestTrackableValue) @Override
.appendInt(numberOfSignificantValueDigits).appendBoolean(autoResize) public List<String> requiredFields() {
.build(); return Collections.singletonList(fieldName);
} }
@Override
public boolean equals(final Object o){ @Override
if (this == o) { public int getMaxIntermediateSize() {
return true; return updatableSerializationBytes == 0? getUpdatableSerializationBytes():updatableSerializationBytes;
} }
if (o == null || !getClass().equals(o.getClass())) {
return false; private int getUpdatableSerializationBytes(){
} if(!autoResize){
/*Histogram histogram = new Histogram(lowestDiscernibleValue, highestTrackableValue, numberOfSignificantValueDigits);
HdrHistogramAggregatorFactory that = (HdrHistogramAggregatorFactory) o; histogram.setAutoResize(autoResize);
return name.equals(that.name) && fieldName.equals(that.fieldName) && return histogram.getNeededByteBufferCapacity();*/
lowestDiscernibleValue == that.lowestDiscernibleValue && return HistogramSketch.getUpdatableSerializationBytes(lowestDiscernibleValue, highestTrackableValue, numberOfSignificantValueDigits);
highestTrackableValue == that.highestTrackableValue && }else{
numberOfSignificantValueDigits == that.numberOfSignificantValueDigits && //return (1 << 10) * 512;
autoResize == that.autoResize return HistogramSketch.getUpdatableSerializationBytes(lowestDiscernibleValue, BUFFER_AUTO_RESIZE_HIGHEST, numberOfSignificantValueDigits);
; }
} }
@Override @Override
public int hashCode(){ public byte[] getCacheKey() {
return Objects.hash(name, fieldName, lowestDiscernibleValue, highestTrackableValue, numberOfSignificantValueDigits, autoResize); return new CacheKeyBuilder(HdrHistogramModule.CACHE_TYPE_ID_OFFSET).appendByte(HdrHistogramModule.QUANTILES_HDRHISTOGRAM_BUILD_CACHE_TYPE_ID)
} .appendString(name).appendString(fieldName)
.appendDouble(lowestDiscernibleValue).appendDouble(highestTrackableValue)
.appendInt(numberOfSignificantValueDigits).appendBoolean(autoResize)
@Override .build();
public String toString() { }
return getClass().getSimpleName() + "{" +
"name='" + name + '\'' + @Override
", fieldName='" + fieldName + '\'' + public boolean equals(final Object o){
", lowestDiscernibleValue=" + lowestDiscernibleValue + if (this == o) {
", highestTrackableValue=" + highestTrackableValue + return true;
", numberOfSignificantValueDigits=" + numberOfSignificantValueDigits + }
", autoResize=" + autoResize + if (o == null || !getClass().equals(o.getClass())) {
'}'; return false;
} }
}
HdrHistogramAggregatorFactory that = (HdrHistogramAggregatorFactory) o;
return name.equals(that.name) && fieldName.equals(that.fieldName) &&
lowestDiscernibleValue == that.lowestDiscernibleValue &&
highestTrackableValue == that.highestTrackableValue &&
numberOfSignificantValueDigits == that.numberOfSignificantValueDigits &&
autoResize == that.autoResize
;
}
@Override
public int hashCode(){
return Objects.hash(name, fieldName, lowestDiscernibleValue, highestTrackableValue, numberOfSignificantValueDigits, autoResize);
}
@Override
public String toString() {
return getClass().getSimpleName() + "{" +
"name='" + name + '\'' +
", fieldName='" + fieldName + '\'' +
", lowestDiscernibleValue=" + lowestDiscernibleValue +
", highestTrackableValue=" + highestTrackableValue +
", numberOfSignificantValueDigits=" + numberOfSignificantValueDigits +
", autoResize=" + autoResize +
'}';
}
}

7
druid-hdrhistogram/src/main/java/org/apache/druid/query/aggregation/sketch/HdrHistogram/HdrHistogramMergeAggregatorFactory.java
View File

@@ -1,9 +1,9 @@
package org.apache.druid.query.aggregation.sketch.HdrHistogram; package org.apache.druid.query.aggregation.sketch.HdrHistogram;
import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonProperty;
import org.HdrHistogram.Histogram;
import org.HdrHistogram.HistogramSketch; import org.HdrHistogram.HistogramSketch;
import org.apache.druid.query.aggregation.Aggregator; import org.apache.druid.query.aggregation.Aggregator;
import org.apache.druid.query.aggregation.AggregatorFactory;
import org.apache.druid.query.aggregation.BufferAggregator; import org.apache.druid.query.aggregation.BufferAggregator;
import org.apache.druid.query.cache.CacheKeyBuilder; import org.apache.druid.query.cache.CacheKeyBuilder;
import org.apache.druid.segment.ColumnSelectorFactory; import org.apache.druid.segment.ColumnSelectorFactory;
@@ -48,6 +48,11 @@ public class HdrHistogramMergeAggregatorFactory extends HdrHistogramAggregatorFa
); );
} }
@Override
public AggregatorFactory withName(String newName) {
return new HdrHistogramMergeAggregatorFactory(newName, fieldName, lowestDiscernibleValue, highestTrackableValue, numberOfSignificantValueDigits, autoResize);
}
@Override @Override
public byte[] getCacheKey() { public byte[] getCacheKey() {
return new CacheKeyBuilder(HdrHistogramModule.CACHE_TYPE_ID_OFFSET).appendByte(HdrHistogramModule.QUANTILES_HDRHISTOGRAM_MERGE_CACHE_TYPE_ID) return new CacheKeyBuilder(HdrHistogramModule.CACHE_TYPE_ID_OFFSET).appendByte(HdrHistogramModule.QUANTILES_HDRHISTOGRAM_MERGE_CACHE_TYPE_ID)

3
druid-hdrhistogram/src/main/java/org/apache/druid/query/aggregation/sketch/HdrHistogram/HdrHistogramModule.java
View File

@@ -7,13 +7,13 @@ import com.fasterxml.jackson.databind.jsontype.NamedType;
import com.fasterxml.jackson.databind.module.SimpleModule; import com.fasterxml.jackson.databind.module.SimpleModule;
import com.google.common.annotations.VisibleForTesting; import com.google.common.annotations.VisibleForTesting;
import com.google.inject.Binder; import com.google.inject.Binder;
import org.HdrHistogram.Histogram;
import org.HdrHistogram.HistogramSketch; import org.HdrHistogram.HistogramSketch;
import org.apache.druid.initialization.DruidModule; import org.apache.druid.initialization.DruidModule;
import org.apache.druid.query.aggregation.sketch.HdrHistogram.sql.HdrHistogramObjectSqlAggregator; import org.apache.druid.query.aggregation.sketch.HdrHistogram.sql.HdrHistogramObjectSqlAggregator;
import org.apache.druid.query.aggregation.sketch.HdrHistogram.sql.HdrHistogramPercentilesOperatorConversion; import org.apache.druid.query.aggregation.sketch.HdrHistogram.sql.HdrHistogramPercentilesOperatorConversion;
import org.apache.druid.query.aggregation.sketch.HdrHistogram.sql.HdrHistogramQuantileSqlAggregator; import org.apache.druid.query.aggregation.sketch.HdrHistogram.sql.HdrHistogramQuantileSqlAggregator;
import org.apache.druid.query.aggregation.sketch.HdrHistogram.sql.HdrHistogramQuantilesOperatorConversion; import org.apache.druid.query.aggregation.sketch.HdrHistogram.sql.HdrHistogramQuantilesOperatorConversion;
import org.apache.druid.segment.column.ColumnType;
import org.apache.druid.segment.serde.ComplexMetrics; import org.apache.druid.segment.serde.ComplexMetrics;
import org.apache.druid.sql.guice.SqlBindings; import org.apache.druid.sql.guice.SqlBindings;
@@ -29,6 +29,7 @@ public class HdrHistogramModule implements DruidModule {
public static final byte QUANTILES_HDRHISTOGRAM_TO_PERCENTILES_CACHE_TYPE_ID = 0x05; public static final byte QUANTILES_HDRHISTOGRAM_TO_PERCENTILES_CACHE_TYPE_ID = 0x05;
public static final String HDRHISTOGRAM_TYPE_NAME = "HdrHistogramSketch"; public static final String HDRHISTOGRAM_TYPE_NAME = "HdrHistogramSketch";
public static final ColumnType TYPE = ColumnType.ofComplex(HDRHISTOGRAM_TYPE_NAME);
public static final ObjectMapper objectMapper = new ObjectMapper(); public static final ObjectMapper objectMapper = new ObjectMapper();

232
druid-hdrhistogram/src/main/java/org/apache/druid/query/aggregation/sketch/HdrHistogram/HdrHistogramToPercentilesPostAggregator.java
View File

@@ -1,111 +1,121 @@
package org.apache.druid.query.aggregation.sketch.HdrHistogram; package org.apache.druid.query.aggregation.sketch.HdrHistogram;
import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.common.collect.Sets; import com.google.common.collect.Sets;
import org.HdrHistogram.HistogramSketch; import org.HdrHistogram.HistogramSketch;
import org.HdrHistogram.Percentile; import org.HdrHistogram.Percentile;
import org.apache.druid.java.util.common.IAE; import org.apache.druid.java.util.common.IAE;
import org.apache.druid.query.aggregation.AggregatorFactory; import org.apache.druid.query.aggregation.AggregatorFactory;
import org.apache.druid.query.aggregation.PostAggregator; import org.apache.druid.query.aggregation.PostAggregator;
import org.apache.druid.query.cache.CacheKeyBuilder; import org.apache.druid.query.cache.CacheKeyBuilder;
import org.apache.druid.segment.ColumnInspector;
import javax.annotation.Nullable; import org.apache.druid.segment.column.ColumnType;
import java.util.*;
import javax.annotation.Nullable;
public class HdrHistogramToPercentilesPostAggregator implements PostAggregator { import java.util.*;
private final String name;
private final String fieldName; public class HdrHistogramToPercentilesPostAggregator implements PostAggregator {
private final int percentileTicksPerHalfDistance; private final String name;
private final String fieldName;
@JsonCreator private final int percentileTicksPerHalfDistance;
public HdrHistogramToPercentilesPostAggregator(
@JsonProperty("name") String name, @JsonCreator
@JsonProperty("fieldName") String fieldName, public HdrHistogramToPercentilesPostAggregator(
@JsonProperty("percentileTicksPerHalfDistance") int percentileTicksPerHalfDistance @JsonProperty("name") String name,
){ @JsonProperty("fieldName") String fieldName,
this.name = name; @JsonProperty("percentileTicksPerHalfDistance") int percentileTicksPerHalfDistance
this.fieldName = fieldName; ){
this.percentileTicksPerHalfDistance = percentileTicksPerHalfDistance; this.name = name;
} this.fieldName = fieldName;
this.percentileTicksPerHalfDistance = percentileTicksPerHalfDistance;
@Override }
@JsonProperty
public String getName() { @Override
return name; public ColumnType getType(ColumnInspector signature){
} return ColumnType.STRING;
}
@JsonProperty
public String getFieldName() { @Override
return fieldName; @JsonProperty
} public String getName() {
return name;
@JsonProperty }
public int getPercentileTicksPerHalfDistance() {
return percentileTicksPerHalfDistance; @JsonProperty
} public String getFieldName() {
return fieldName;
@Nullable }
@Override
public Object compute(Map<String, Object> values) { @JsonProperty
HistogramSketch histogram = (HistogramSketch) values.get(fieldName); public int getPercentileTicksPerHalfDistance() {
List<Percentile> percentiles = histogram.percentileList(percentileTicksPerHalfDistance); return percentileTicksPerHalfDistance;
return HdrHistogramModule.toJson(percentiles); }
}
@Nullable
@Override @Override
public Comparator<double[]> getComparator() public Object compute(Map<String, Object> values) {
{ HistogramSketch histogram = (HistogramSketch) values.get(fieldName);
throw new IAE("Comparing arrays of quantiles is not supported"); if(histogram == null){
} return "[]"; //"[]"
}
@Override List<Percentile> percentiles = histogram.percentileList(percentileTicksPerHalfDistance);
public Set<String> getDependentFields() return HdrHistogramModule.toJson(percentiles);
{ }
return Sets.newHashSet(fieldName);
} @Override
public Comparator<double[]> getComparator()
@Override {
public PostAggregator decorate(Map<String, AggregatorFactory> aggregators) { throw new IAE("Comparing arrays of quantiles is not supported");
return this; }
}
@Override
@Override public Set<String> getDependentFields()
public byte[] getCacheKey() { {
CacheKeyBuilder builder = new CacheKeyBuilder(HdrHistogramModule.CACHE_TYPE_ID_OFFSET).appendByte(HdrHistogramModule.QUANTILES_HDRHISTOGRAM_TO_PERCENTILES_CACHE_TYPE_ID) return Sets.newHashSet(fieldName);
.appendString(fieldName); }
builder.appendInt(percentileTicksPerHalfDistance);
return builder.build(); @Override
} public PostAggregator decorate(Map<String, AggregatorFactory> aggregators) {
return this;
@Override }
public boolean equals(Object o) {
if (this == o) { @Override
return true; public byte[] getCacheKey() {
} CacheKeyBuilder builder = new CacheKeyBuilder(HdrHistogramModule.CACHE_TYPE_ID_OFFSET).appendByte(HdrHistogramModule.QUANTILES_HDRHISTOGRAM_TO_PERCENTILES_CACHE_TYPE_ID)
if (o == null || getClass() != o.getClass()) { .appendString(fieldName);
return false; builder.appendInt(percentileTicksPerHalfDistance);
} return builder.build();
HdrHistogramToPercentilesPostAggregator that = (HdrHistogramToPercentilesPostAggregator) o; }
return percentileTicksPerHalfDistance == that.percentileTicksPerHalfDistance && @Override
name.equals(that.name) && public boolean equals(Object o) {
fieldName.equals(that.fieldName); if (this == o) {
} return true;
}
@Override if (o == null || getClass() != o.getClass()) {
public int hashCode() { return false;
return Objects.hash(name, fieldName, percentileTicksPerHalfDistance); }
} HdrHistogramToPercentilesPostAggregator that = (HdrHistogramToPercentilesPostAggregator) o;
@Override return percentileTicksPerHalfDistance == that.percentileTicksPerHalfDistance &&
public String toString() { name.equals(that.name) &&
return "HdrHistogramToPercentilesPostAggregator{" + fieldName.equals(that.fieldName);
"name='" + name + '\'' + }
", fieldName='" + fieldName + '\'' +
", probabilitys=" + percentileTicksPerHalfDistance + @Override
'}'; public int hashCode() {
} return Objects.hash(name, fieldName, percentileTicksPerHalfDistance);
}
} @Override
public String toString() {
return "HdrHistogramToPercentilesPostAggregator{" +
"name='" + name + '\'' +
", fieldName='" + fieldName + '\'' +
", probabilitys=" + percentileTicksPerHalfDistance +
'}';
}
}

246
druid-hdrhistogram/src/main/java/org/apache/druid/query/aggregation/sketch/HdrHistogram/HdrHistogramToQuantilePostAggregator.java
View File

@@ -1,118 +1,128 @@
package org.apache.druid.query.aggregation.sketch.HdrHistogram; package org.apache.druid.query.aggregation.sketch.HdrHistogram;
import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.common.collect.Sets; import com.google.common.collect.Sets;
import org.HdrHistogram.Histogram; import org.HdrHistogram.Histogram;
import org.HdrHistogram.HistogramSketch; import org.HdrHistogram.HistogramSketch;
import org.apache.druid.java.util.common.IAE; import org.apache.druid.java.util.common.IAE;
import org.apache.druid.query.aggregation.AggregatorFactory; import org.apache.druid.query.aggregation.AggregatorFactory;
import org.apache.druid.query.aggregation.PostAggregator; import org.apache.druid.query.aggregation.PostAggregator;
import org.apache.druid.query.cache.CacheKeyBuilder; import org.apache.druid.query.cache.CacheKeyBuilder;
import org.apache.druid.segment.ColumnInspector;
import javax.annotation.Nullable; import org.apache.druid.segment.column.ColumnType;
import java.util.Comparator;
import java.util.Map; import javax.annotation.Nullable;
import java.util.Objects; import java.util.Comparator;
import java.util.Set; import java.util.Map;
import java.util.Objects;
public class HdrHistogramToQuantilePostAggregator implements PostAggregator { import java.util.Set;
private final String name;
private final String fieldName; public class HdrHistogramToQuantilePostAggregator implements PostAggregator {
private final float probability; private final String name;
private final String fieldName;
@JsonCreator private final float probability;
public HdrHistogramToQuantilePostAggregator(
@JsonProperty("name") String name, @JsonCreator
@JsonProperty("fieldName") String fieldName, public HdrHistogramToQuantilePostAggregator(
@JsonProperty("probability") float probability @JsonProperty("name") String name,
){ @JsonProperty("fieldName") String fieldName,
this.name = name; @JsonProperty("probability") float probability
this.fieldName = fieldName; ){
this.probability = probability; this.name = name;
this.fieldName = fieldName;
if (probability < 0 || probability > 1) { this.probability = probability;
throw new IAE("Illegal probability[%s], must be strictly between 0 and 1", probability);
} if (probability < 0 || probability > 1) {
} throw new IAE("Illegal probability[%s], must be strictly between 0 and 1", probability);
}
@Override }
public Set<String> getDependentFields() {
return Sets.newHashSet(fieldName); @Override
} public ColumnType getType(ColumnInspector signature){
return ColumnType.LONG;
@Override }
public Comparator getComparator() {
return new Comparator<Long>(){ @Override
@Override public Set<String> getDependentFields() {
public int compare(final Long a, final Long b){ return Sets.newHashSet(fieldName);
return Long.compare(a, b); }
}
}; @Override
} public Comparator getComparator() {
return new Comparator<Long>(){
@Nullable @Override
@Override public int compare(final Long a, final Long b){
public Object compute(Map<String, Object> values) { return Long.compare(a, b);
HistogramSketch histogram = (HistogramSketch) values.get(fieldName); }
return histogram.getValueAtPercentile(probability * 100); };
} }
@Override @Nullable
@JsonProperty @Override
public String getName() { public Object compute(Map<String, Object> values) {
return name; HistogramSketch histogram = (HistogramSketch) values.get(fieldName);
} if(histogram == null){
return null;
@JsonProperty }
public String getFieldName() { return histogram.getValueAtPercentile(probability * 100);
return fieldName; }
}
@Override
@JsonProperty @JsonProperty
public double getProbability() { public String getName() {
return probability; return name;
} }
@Override @JsonProperty
public PostAggregator decorate(Map<String, AggregatorFactory> aggregators) { public String getFieldName() {
return this; return fieldName;
} }
@Override @JsonProperty
public boolean equals(Object o) { public double getProbability() {
if (this == o) { return probability;
return true; }
}
if (o == null || getClass() != o.getClass()) { @Override
return false; public PostAggregator decorate(Map<String, AggregatorFactory> aggregators) {
} return this;
HdrHistogramToQuantilePostAggregator that = (HdrHistogramToQuantilePostAggregator) o; }
return Float.compare(that.probability, probability) == 0 && @Override
name.equals(that.name) && public boolean equals(Object o) {
fieldName.equals(that.fieldName); if (this == o) {
} return true;
}
@Override if (o == null || getClass() != o.getClass()) {
public int hashCode() { return false;
return Objects.hash(name, fieldName, probability); }
} HdrHistogramToQuantilePostAggregator that = (HdrHistogramToQuantilePostAggregator) o;
@Override return Float.compare(that.probability, probability) == 0 &&
public String toString() { name.equals(that.name) &&
return "HdrHistogramToQuantilePostAggregator{" + fieldName.equals(that.fieldName);
"name='" + name + '\'' + }
", fieldName='" + fieldName + '\'' +
", probability=" + probability + @Override
'}'; public int hashCode() {
} return Objects.hash(name, fieldName, probability);
}
@Override
public byte[] getCacheKey() { @Override
return new CacheKeyBuilder(HdrHistogramModule.CACHE_TYPE_ID_OFFSET).appendByte(HdrHistogramModule.QUANTILES_HDRHISTOGRAM_TO_QUANTILE_CACHE_TYPE_ID) public String toString() {
.appendString(fieldName) return "HdrHistogramToQuantilePostAggregator{" +
.appendFloat(probability) "name='" + name + '\'' +
.build(); ", fieldName='" + fieldName + '\'' +
} ", probability=" + probability +
} '}';
}
@Override
public byte[] getCacheKey() {
return new CacheKeyBuilder(HdrHistogramModule.CACHE_TYPE_ID_OFFSET).appendByte(HdrHistogramModule.QUANTILES_HDRHISTOGRAM_TO_QUANTILE_CACHE_TYPE_ID)
.appendString(fieldName)
.appendFloat(probability)
.build();
}
}

239
druid-hdrhistogram/src/main/java/org/apache/druid/query/aggregation/sketch/HdrHistogram/HdrHistogramToQuantilesPostAggregator.java
View File

@@ -1,114 +1,125 @@
package org.apache.druid.query.aggregation.sketch.HdrHistogram; package org.apache.druid.query.aggregation.sketch.HdrHistogram;
import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.common.collect.Sets; import com.google.common.collect.Sets;
import org.HdrHistogram.Histogram; import org.HdrHistogram.Histogram;
import org.HdrHistogram.HistogramSketch; import org.HdrHistogram.HistogramSketch;
import org.apache.druid.java.util.common.IAE; import org.apache.druid.java.util.common.IAE;
import org.apache.druid.query.aggregation.AggregatorFactory; import org.apache.druid.query.aggregation.AggregatorFactory;
import org.apache.druid.query.aggregation.PostAggregator; import org.apache.druid.query.aggregation.PostAggregator;
import org.apache.druid.query.cache.CacheKeyBuilder; import org.apache.druid.query.cache.CacheKeyBuilder;
import org.apache.druid.segment.ColumnInspector;
import javax.annotation.Nullable; import org.apache.druid.segment.column.ColumnType;
import java.util.*;
import javax.annotation.Nullable;
public class HdrHistogramToQuantilesPostAggregator implements PostAggregator { import java.util.*;
private final String name;
private final String fieldName; public class HdrHistogramToQuantilesPostAggregator implements PostAggregator {
private final float[] probabilitys; private final String name;
private final String fieldName;
@JsonCreator private final float[] probabilitys;
public HdrHistogramToQuantilesPostAggregator(
@JsonProperty("name") String name, @JsonCreator
@JsonProperty("fieldName") String fieldName, public HdrHistogramToQuantilesPostAggregator(
@JsonProperty("probabilitys") float[] probabilitys @JsonProperty("name") String name,
){ @JsonProperty("fieldName") String fieldName,
this.name = name; @JsonProperty("probabilitys") float[] probabilitys
this.fieldName = fieldName; ){
this.probabilitys = probabilitys; this.name = name;
} this.fieldName = fieldName;
this.probabilitys = probabilitys;
@Override }
@JsonProperty
public String getName() { @Override
return name; public ColumnType getType(ColumnInspector signature){
} return ColumnType.LONG_ARRAY;
}
@JsonProperty
public String getFieldName() { @Override
return fieldName; @JsonProperty
} public String getName() {
return name;
@JsonProperty }
public float[] getProbabilitys() {
return probabilitys; @JsonProperty
} public String getFieldName() {
return fieldName;
@Nullable }
@Override
public Object compute(Map<String, Object> values) { @JsonProperty
HistogramSketch histogram = (HistogramSketch) values.get(fieldName); public float[] getProbabilitys() {
final long[] counts = new long[probabilitys.length]; return probabilitys;
for (int i = 0; i < probabilitys.length; i++) { }
counts[i] = histogram.getValueAtPercentile(probabilitys[i] * 100);
} @Nullable
return counts; @Override
} public Object compute(Map<String, Object> values) {
HistogramSketch histogram = (HistogramSketch) values.get(fieldName);
@Override if(histogram == null){
public Comparator<double[]> getComparator() //return null;
{ return new Long[probabilitys.length];
throw new IAE("Comparing arrays of quantiles is not supported"); }
} final Long[] counts = new Long[probabilitys.length];
for (int i = 0; i < probabilitys.length; i++) {
@Override counts[i] = histogram.getValueAtPercentile(probabilitys[i] * 100);
public Set<String> getDependentFields() }
{ return counts;
return Sets.newHashSet(fieldName); }
}
@Override
@Override public Comparator<double[]> getComparator()
public PostAggregator decorate(Map<String, AggregatorFactory> aggregators) { {
return this; throw new IAE("Comparing arrays of quantiles is not supported");
} }
@Override @Override
public byte[] getCacheKey() { public Set<String> getDependentFields()
CacheKeyBuilder builder = new CacheKeyBuilder(HdrHistogramModule.CACHE_TYPE_ID_OFFSET).appendByte(HdrHistogramModule.QUANTILES_HDRHISTOGRAM_TO_QUANTILES_CACHE_TYPE_ID) {
.appendString(fieldName); return Sets.newHashSet(fieldName);
for (float probability : probabilitys) { }
builder.appendFloat(probability);
} @Override
return builder.build(); public PostAggregator decorate(Map<String, AggregatorFactory> aggregators) {
} return this;
}
@Override
public boolean equals(Object o) { @Override
if (this == o) { public byte[] getCacheKey() {
return true; CacheKeyBuilder builder = new CacheKeyBuilder(HdrHistogramModule.CACHE_TYPE_ID_OFFSET).appendByte(HdrHistogramModule.QUANTILES_HDRHISTOGRAM_TO_QUANTILES_CACHE_TYPE_ID)
} .appendString(fieldName);
if (o == null || getClass() != o.getClass()) { for (float probability : probabilitys) {
return false; builder.appendFloat(probability);
} }
HdrHistogramToQuantilesPostAggregator that = (HdrHistogramToQuantilesPostAggregator) o; return builder.build();
}
return Arrays.equals(probabilitys, that.probabilitys) &&
name.equals(that.name) && @Override
fieldName.equals(that.fieldName); public boolean equals(Object o) {
} if (this == o) {
return true;
@Override }
public int hashCode() { if (o == null || getClass() != o.getClass()) {
return Objects.hash(name, fieldName, Arrays.hashCode(probabilitys)); return false;
} }
HdrHistogramToQuantilesPostAggregator that = (HdrHistogramToQuantilesPostAggregator) o;
@Override
public String toString() { return Arrays.equals(probabilitys, that.probabilitys) &&
return "HdrHistogramToQuantilesPostAggregator{" + name.equals(that.name) &&
"name='" + name + '\'' + fieldName.equals(that.fieldName);
", fieldName='" + fieldName + '\'' + }
", probabilitys=" + Arrays.toString(probabilitys) +
'}'; @Override
} public int hashCode() {
} return Objects.hash(name, fieldName, Arrays.hashCode(probabilitys));
}
@Override
public String toString() {
return "HdrHistogramToQuantilesPostAggregator{" +
"name='" + name + '\'' +
", fieldName='" + fieldName + '\'' +
", probabilitys=" + Arrays.toString(probabilitys) +
'}';
}
}

13
druid-hdrhistogram/src/main/java/org/apache/druid/query/aggregation/sketch/HdrHistogram/sql/HdrHistogramObjectSqlAggregator.java
View File

@@ -18,6 +18,7 @@ import org.apache.druid.query.aggregation.AggregatorFactory;
import org.apache.druid.query.aggregation.sketch.HdrHistogram.HdrHistogramAggregatorFactory; import org.apache.druid.query.aggregation.sketch.HdrHistogram.HdrHistogramAggregatorFactory;
import org.apache.druid.query.aggregation.sketch.HdrHistogram.HdrHistogramMergeAggregatorFactory; import org.apache.druid.query.aggregation.sketch.HdrHistogram.HdrHistogramMergeAggregatorFactory;
import org.apache.druid.segment.VirtualColumn; import org.apache.druid.segment.VirtualColumn;
import org.apache.druid.segment.column.ColumnType;
import org.apache.druid.segment.column.RowSignature; import org.apache.druid.segment.column.RowSignature;
import org.apache.druid.segment.column.ValueType; import org.apache.druid.segment.column.ValueType;
import org.apache.druid.sql.calcite.aggregation.Aggregation; import org.apache.druid.sql.calcite.aggregation.Aggregation;
@@ -118,11 +119,11 @@ public class HdrHistogramObjectSqlAggregator implements SqlAggregator {
} }
// No existing match found. Create a new one. // No existing match found. Create a new one.
final List<VirtualColumn> virtualColumns = new ArrayList<>(); // 新版本删除了final List<VirtualColumn> virtualColumns = new ArrayList<>();
if (input.isDirectColumnAccess()) { if (input.isDirectColumnAccess()) {
// 参数是Histogram对象 // 参数是Histogram对象
if (rowSignature.getColumnType(input.getDirectColumn()).orElse(null) == ValueType.COMPLEX) { if (rowSignature.getColumnType(input.getDirectColumn()).map(type -> type.is(ValueType.COMPLEX)).orElse(false)) {
aggregatorFactory = new HdrHistogramMergeAggregatorFactory( aggregatorFactory = new HdrHistogramMergeAggregatorFactory(
histogramName, histogramName,
input.getDirectColumn(), input.getDirectColumn(),
@@ -142,12 +143,11 @@ public class HdrHistogramObjectSqlAggregator implements SqlAggregator {
); );
} }
} else { } else {
final VirtualColumn virtualColumn = final String virtualColumnName =
virtualColumnRegistry.getOrCreateVirtualColumnForExpression(plannerContext, input, SqlTypeName.BIGINT); virtualColumnRegistry.getOrCreateVirtualColumnForExpression(input, ColumnType.LONG);
virtualColumns.add(virtualColumn);
aggregatorFactory = new HdrHistogramAggregatorFactory( aggregatorFactory = new HdrHistogramAggregatorFactory(
histogramName, histogramName,
virtualColumn.getOutputName(), virtualColumnName,
lowestDiscernibleValue, lowestDiscernibleValue,
highestTrackableValue, highestTrackableValue,
numberOfSignificantValueDigits, numberOfSignificantValueDigits,
@@ -156,7 +156,6 @@ public class HdrHistogramObjectSqlAggregator implements SqlAggregator {
} }
return Aggregation.create( return Aggregation.create(
virtualColumns,
ImmutableList.of(aggregatorFactory), ImmutableList.of(aggregatorFactory),
null null
); );

11
druid-hdrhistogram/src/main/java/org/apache/druid/query/aggregation/sketch/HdrHistogram/sql/HdrHistogramPercentilesOperatorConversion.java
View File

@@ -14,16 +14,16 @@ import org.apache.druid.query.aggregation.sketch.HdrHistogram.HdrHistogramToPerc
import org.apache.druid.query.aggregation.PostAggregator; import org.apache.druid.query.aggregation.PostAggregator;
import org.apache.druid.query.aggregation.post.FieldAccessPostAggregator; import org.apache.druid.query.aggregation.post.FieldAccessPostAggregator;
import org.apache.druid.segment.column.RowSignature; import org.apache.druid.segment.column.RowSignature;
import org.apache.druid.sql.calcite.expression.DirectOperatorConversion;
import org.apache.druid.sql.calcite.expression.DruidExpression; import org.apache.druid.sql.calcite.expression.DruidExpression;
import org.apache.druid.sql.calcite.expression.OperatorConversions; import org.apache.druid.sql.calcite.expression.OperatorConversions;
import org.apache.druid.sql.calcite.expression.PostAggregatorVisitor; import org.apache.druid.sql.calcite.expression.PostAggregatorVisitor;
import org.apache.druid.sql.calcite.expression.SqlOperatorConversion;
import org.apache.druid.sql.calcite.planner.PlannerContext; import org.apache.druid.sql.calcite.planner.PlannerContext;
import javax.annotation.Nullable; import javax.annotation.Nullable;
import java.util.List; import java.util.List;
public class HdrHistogramPercentilesOperatorConversion extends DirectOperatorConversion { public class HdrHistogramPercentilesOperatorConversion implements SqlOperatorConversion {
private static final String FUNCTION_NAME = "HDR_GET_PERCENTILES"; private static final String FUNCTION_NAME = "HDR_GET_PERCENTILES";
private static final SqlFunction SQL_FUNCTION = OperatorConversions private static final SqlFunction SQL_FUNCTION = OperatorConversions
.operatorBuilder(StringUtils.toUpperCase(FUNCTION_NAME)) .operatorBuilder(StringUtils.toUpperCase(FUNCTION_NAME))
@@ -32,10 +32,6 @@ public class HdrHistogramPercentilesOperatorConversion extends DirectOperatorCon
.returnTypeInference(ReturnTypes.explicit(SqlTypeName.VARCHAR)) .returnTypeInference(ReturnTypes.explicit(SqlTypeName.VARCHAR))
.build(); .build();
public HdrHistogramPercentilesOperatorConversion() {
super(SQL_FUNCTION, FUNCTION_NAME);
}
@Override @Override
public SqlOperator calciteOperator() public SqlOperator calciteOperator()
{ {
@@ -66,7 +62,8 @@ public class HdrHistogramPercentilesOperatorConversion extends DirectOperatorCon
plannerContext, plannerContext,
rowSignature, rowSignature,
operands.get(0), operands.get(0),
postAggregatorVisitor postAggregatorVisitor,
true
); );
if (postAgg == null) { if (postAgg == null) {

33
druid-hdrhistogram/src/main/java/org/apache/druid/query/aggregation/sketch/HdrHistogram/sql/HdrHistogramQuantileSqlAggregator.java
View File

@@ -16,6 +16,7 @@ import org.apache.druid.query.aggregation.sketch.HdrHistogram.HdrHistogramAggreg
import org.apache.druid.query.aggregation.sketch.HdrHistogram.HdrHistogramMergeAggregatorFactory; import org.apache.druid.query.aggregation.sketch.HdrHistogram.HdrHistogramMergeAggregatorFactory;
import org.apache.druid.query.aggregation.sketch.HdrHistogram.HdrHistogramToQuantilePostAggregator; import org.apache.druid.query.aggregation.sketch.HdrHistogram.HdrHistogramToQuantilePostAggregator;
import org.apache.druid.segment.VirtualColumn; import org.apache.druid.segment.VirtualColumn;
import org.apache.druid.segment.column.ColumnType;
import org.apache.druid.segment.column.RowSignature; import org.apache.druid.segment.column.RowSignature;
import org.apache.druid.segment.column.ValueType; import org.apache.druid.segment.column.ValueType;
import org.apache.druid.segment.virtual.ExpressionVirtualColumn; import org.apache.druid.segment.virtual.ExpressionVirtualColumn;
@@ -141,22 +142,16 @@ public class HdrHistogramQuantileSqlAggregator implements SqlAggregator {
// Check input for equivalence. // Check input for equivalence.
final boolean inputMatches; final boolean inputMatches;
final VirtualColumn virtualInput = existing.getVirtualColumns() final DruidExpression virtualInput =
.stream() virtualColumnRegistry.findVirtualColumnExpressions(theFactory.requiredFields())
.filter( .stream()
virtualColumn -> .findFirst()
virtualColumn.getOutputName() .orElse(null);
.equals(theFactory.getFieldName())
)
.findFirst()
.orElse(null);
if (virtualInput == null) { if (virtualInput == null) {
inputMatches = input.isDirectColumnAccess() inputMatches = input.isDirectColumnAccess() && input.getDirectColumn().equals(theFactory.getFieldName());
&& input.getDirectColumn().equals(theFactory.getFieldName());
} else { } else {
inputMatches = ((ExpressionVirtualColumn) virtualInput).getExpression() inputMatches = virtualInput.equals(input);
.equals(input.getExpression());
} }
final boolean matches = inputMatches final boolean matches = inputMatches
@@ -177,11 +172,11 @@ public class HdrHistogramQuantileSqlAggregator implements SqlAggregator {
} }
// No existing match found. Create a new one. // No existing match found. Create a new one.
final List<VirtualColumn> virtualColumns = new ArrayList<>(); //final List<VirtualColumn> virtualColumns = new ArrayList<>();
if (input.isDirectColumnAccess()) { if (input.isDirectColumnAccess()) {
// 参数是Histogram对象 // 参数是Histogram对象
if (rowSignature.getColumnType(input.getDirectColumn()).orElse(null) == ValueType.COMPLEX) { if (rowSignature.getColumnType(input.getDirectColumn()).map(type -> type.is(ValueType.COMPLEX)).orElse(false)) {
aggregatorFactory = new HdrHistogramMergeAggregatorFactory( aggregatorFactory = new HdrHistogramMergeAggregatorFactory(
histogramName, histogramName,
input.getDirectColumn(), input.getDirectColumn(),
@@ -201,12 +196,11 @@ public class HdrHistogramQuantileSqlAggregator implements SqlAggregator {
); );
} }
} else { } else {
final VirtualColumn virtualColumn = final String virtualColumnName =
virtualColumnRegistry.getOrCreateVirtualColumnForExpression(plannerContext, input, SqlTypeName.BIGINT); virtualColumnRegistry.getOrCreateVirtualColumnForExpression(input, ColumnType.LONG);
virtualColumns.add(virtualColumn);
aggregatorFactory = new HdrHistogramAggregatorFactory( aggregatorFactory = new HdrHistogramAggregatorFactory(
histogramName, histogramName,
virtualColumn.getOutputName(), virtualColumnName,
lowestDiscernibleValue, lowestDiscernibleValue,
highestTrackableValue, highestTrackableValue,
numberOfSignificantValueDigits, numberOfSignificantValueDigits,
@@ -234,7 +228,6 @@ public class HdrHistogramQuantileSqlAggregator implements SqlAggregator {
} }
return Aggregation.create( return Aggregation.create(
virtualColumns,
ImmutableList.of(aggregatorFactory), ImmutableList.of(aggregatorFactory),
new HdrHistogramToQuantilePostAggregator(name, histogramName, probability) new HdrHistogramToQuantilePostAggregator(name, histogramName, probability)
); );

54
druid-hdrhistogram/src/main/java/org/apache/druid/query/aggregation/sketch/HdrHistogram/sql/HdrHistogramQuantilesOperatorConversion.java
View File

@@ -62,50 +62,30 @@ public class HdrHistogramQuantilesOperatorConversion implements SqlOperatorConve
{ {
final List<RexNode> operands = ((RexCall) rexNode).getOperands(); final List<RexNode> operands = ((RexCall) rexNode).getOperands();
final float[] args = new float[operands.size() - 1]; final float[] args = new float[operands.size() - 1];
PostAggregator postAgg = null;
int operandCounter = 0; // 新版本直接就从第一个参数取
for (RexNode operand : operands) { final PostAggregator inputSketchPostAgg = OperatorConversions.toPostAggregator(
final PostAggregator convertedPostAgg = OperatorConversions.toPostAggregator( plannerContext,
plannerContext, rowSignature,
rowSignature, operands.get(0),
operand, postAggregatorVisitor,
postAggregatorVisitor true
); );
if (convertedPostAgg == null) {
if (operandCounter > 0) { if (inputSketchPostAgg == null) {
try { return null;
if (!operand.isA(SqlKind.LITERAL)) {
return null;
}
float arg = ((Number) RexLiteral.value(operand)).floatValue();
args[operandCounter - 1] = arg;
}
catch (ClassCastException cce) {
return null;
}
} else {
return null;
}
} else {
if (operandCounter == 0) {
postAgg = convertedPostAgg;
} else {
if (!operand.isA(SqlKind.LITERAL)) {
return null;
}
}
}
operandCounter++;
} }
if (postAgg == null) { // 直接解析
return null; for (int i = 1; i < operands.size(); i++) {
RexNode operand = operands.get(i);
float arg = ((Number) RexLiteral.value(operand)).floatValue();
args[i - 1] = arg;
} }
return new HdrHistogramToQuantilesPostAggregator( return new HdrHistogramToQuantilesPostAggregator(
postAggregatorVisitor.getOutputNamePrefix() + postAggregatorVisitor.getAndIncrementCounter(), postAggregatorVisitor.getOutputNamePrefix() + postAggregatorVisitor.getAndIncrementCounter(),
((FieldAccessPostAggregator)postAgg).getFieldName(), ((FieldAccessPostAggregator)inputSketchPostAgg).getFieldName(),
args args
); );
} }

10
druid-hdrhistogram/src/test/java/org/apache/druid/query/aggregation/sketch/HdrHistogram/HdrHistogramBufferAggregatorTest.java
View File

@@ -2,17 +2,13 @@ package org.apache.druid.query.aggregation.sketch.HdrHistogram;
import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableMap;
import org.HdrHistogram.*; import org.HdrHistogram.*;
import org.apache.datasketches.theta.Sketches;
import org.apache.datasketches.theta.UpdateSketch;
import org.apache.druid.data.input.MapBasedRow; import org.apache.druid.data.input.MapBasedRow;
import org.apache.druid.query.aggregation.AggregatorFactory; import org.apache.druid.query.aggregation.AggregatorFactory;
import org.apache.druid.query.aggregation.BufferAggregator; import org.apache.druid.query.aggregation.BufferAggregator;
import org.apache.druid.query.aggregation.TestLongColumnSelector; import org.apache.druid.query.aggregation.TestLongColumnSelector;
import org.apache.druid.query.aggregation.TestObjectColumnSelector; import org.apache.druid.query.aggregation.TestObjectColumnSelector;
import org.apache.druid.query.aggregation.datasketches.theta.SketchHolder;
import org.apache.druid.query.aggregation.datasketches.theta.SketchMergeAggregatorFactory;
import org.apache.druid.query.groupby.epinephelinae.GrouperTestUtil; import org.apache.druid.query.groupby.epinephelinae.GrouperTestUtil;
import org.apache.druid.query.groupby.epinephelinae.TestColumnSelectorFactory; import org.apache.druid.query.groupby.epinephelinae.GroupByTestColumnSelectorFactory;
import org.apache.druid.segment.ColumnSelectorFactory; import org.apache.druid.segment.ColumnSelectorFactory;
import org.junit.Assert; import org.junit.Assert;
import org.junit.Test; import org.junit.Test;
@@ -230,7 +226,7 @@ public class HdrHistogramBufferAggregatorTest {
@Test @Test
public void testMergeAggregatorRelocate() { public void testMergeAggregatorRelocate() {
final TestColumnSelectorFactory columnSelectorFactory = GrouperTestUtil.newColumnSelectorFactory(); final GroupByTestColumnSelectorFactory columnSelectorFactory = GrouperTestUtil.newColumnSelectorFactory();
HistogramSketch histogram = new HistogramSketch(3); HistogramSketch histogram = new HistogramSketch(3);
for (int i = 0; i < 100000; i++) { for (int i = 0; i < 100000; i++) {
histogram.recordValue(i); histogram.recordValue(i);
@@ -252,7 +248,7 @@ public class HdrHistogramBufferAggregatorTest {
@Test @Test
public void testAggregatorRelocate() { public void testAggregatorRelocate() {
final TestColumnSelectorFactory columnSelectorFactory = GrouperTestUtil.newColumnSelectorFactory(); final GroupByTestColumnSelectorFactory columnSelectorFactory = GrouperTestUtil.newColumnSelectorFactory();
HistogramSketch histogram = new HistogramSketch(3); HistogramSketch histogram = new HistogramSketch(3);
for (int i = 0; i < 100000; i++) { for (int i = 0; i < 100000; i++) {
histogram.recordValue(i); histogram.recordValue(i);

501
druid-hdrhistogram/src/test/java/org/apache/druid/query/aggregation/sketch/HdrHistogram/sql/HdrHistogramQuantileSqlAggregatorTest.java
View File

@@ -1,12 +1,15 @@
package org.apache.druid.query.aggregation.sketch.HdrHistogram.sql; package org.apache.druid.query.aggregation.sketch.HdrHistogram.sql;
import com.alibaba.fastjson2.JSON;
import com.fasterxml.jackson.databind.Module; import com.fasterxml.jackson.databind.Module;
import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet; import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Iterables; import com.google.common.collect.Iterables;
import com.google.inject.Injector;
import org.apache.calcite.schema.SchemaPlus; import org.apache.calcite.schema.SchemaPlus;
import org.apache.druid.data.input.InputRow; import org.apache.druid.data.input.InputRow;
import org.apache.druid.guice.DruidInjectorBuilder;
import org.apache.druid.java.util.common.granularity.Granularities; import org.apache.druid.java.util.common.granularity.Granularities;
import org.apache.druid.java.util.common.io.Closer; import org.apache.druid.java.util.common.io.Closer;
import org.apache.druid.query.Druids; import org.apache.druid.query.Druids;
@@ -27,66 +30,49 @@ import org.apache.druid.query.spec.MultipleIntervalSegmentSpec;
import org.apache.druid.segment.IndexBuilder; import org.apache.druid.segment.IndexBuilder;
import org.apache.druid.segment.QueryableIndex; import org.apache.druid.segment.QueryableIndex;
import org.apache.druid.segment.TestHelper; import org.apache.druid.segment.TestHelper;
import org.apache.druid.segment.column.ColumnType;
import org.apache.druid.segment.column.ValueType; import org.apache.druid.segment.column.ValueType;
import org.apache.druid.segment.incremental.IncrementalIndexSchema; import org.apache.druid.segment.incremental.IncrementalIndexSchema;
import org.apache.druid.segment.join.JoinableFactoryWrapper;
import org.apache.druid.segment.virtual.ExpressionVirtualColumn; import org.apache.druid.segment.virtual.ExpressionVirtualColumn;
import org.apache.druid.segment.writeout.OffHeapMemorySegmentWriteOutMediumFactory; import org.apache.druid.segment.writeout.OffHeapMemorySegmentWriteOutMediumFactory;
import org.apache.druid.server.QueryStackTests; import org.apache.druid.server.QueryStackTests;
import org.apache.druid.server.security.AuthTestUtils; import org.apache.druid.server.security.AuthTestUtils;
import org.apache.druid.server.security.AuthenticationResult; import org.apache.druid.server.security.AuthenticationResult;
import org.apache.druid.sql.SqlLifecycle; import org.apache.druid.sql.calcite.BaseCalciteQueryTest;
import org.apache.druid.sql.SqlLifecycleFactory; import org.apache.druid.sql.calcite.QueryTestBuilder;
import org.apache.druid.sql.calcite.QueryTestRunner;
import org.apache.druid.sql.calcite.filtration.Filtration; import org.apache.druid.sql.calcite.filtration.Filtration;
import org.apache.druid.sql.calcite.planner.DruidOperatorTable; import org.apache.druid.sql.calcite.planner.DruidOperatorTable;
import org.apache.druid.sql.calcite.planner.PlannerConfig; import org.apache.druid.sql.calcite.planner.PlannerConfig;
import org.apache.druid.sql.calcite.planner.PlannerContext; import org.apache.druid.sql.calcite.planner.PlannerContext;
import org.apache.druid.sql.calcite.planner.PlannerFactory; import org.apache.druid.sql.calcite.planner.PlannerFactory;
import org.apache.druid.sql.calcite.util.CalciteTestBase; import org.apache.druid.sql.calcite.util.*;
import org.apache.druid.sql.calcite.util.CalciteTests;
import org.apache.druid.sql.calcite.util.QueryLogHook;
import org.apache.druid.sql.calcite.util.SpecificSegmentsQuerySegmentWalker;
import org.apache.druid.timeline.DataSegment; import org.apache.druid.timeline.DataSegment;
import org.apache.druid.timeline.partition.LinearShardSpec; import org.apache.druid.timeline.partition.LinearShardSpec;
import org.junit.*; import org.junit.*;
import org.junit.rules.TemporaryFolder; import org.junit.rules.TemporaryFolder;
import java.io.File;
import java.io.IOException; import java.io.IOException;
import java.util.Arrays; import java.util.*;
import java.util.List;
import java.util.Map;
public class HdrHistogramQuantileSqlAggregatorTest extends CalciteTestBase { public class HdrHistogramQuantileSqlAggregatorTest extends BaseCalciteQueryTest {
private static final String DATA_SOURCE = "foo"; @Override
public void gatherProperties(Properties properties)
private static QueryRunnerFactoryConglomerate conglomerate; {
private static Closer resourceCloser; super.gatherProperties(properties);
private static AuthenticationResult authenticationResult = CalciteTests.REGULAR_USER_AUTH_RESULT;
private static final Map<String, Object> QUERY_CONTEXT_DEFAULT = ImmutableMap.of(
PlannerContext.CTX_SQL_QUERY_ID, "dummy"
);
@Rule
public TemporaryFolder temporaryFolder = new TemporaryFolder();
@Rule
public QueryLogHook queryLogHook = QueryLogHook.create();
private SpecificSegmentsQuerySegmentWalker walker;
private SqlLifecycleFactory sqlLifecycleFactory;
@BeforeClass
public static void setUpClass() {
resourceCloser = Closer.create();
conglomerate = QueryStackTests.createQueryRunnerFactoryConglomerate(resourceCloser);
} }
@AfterClass @Override
public static void tearDownClass() throws IOException { public void configureGuice(DruidInjectorBuilder builder)
resourceCloser.close(); {
super.configureGuice(builder);
builder.addModule(new HdrHistogramModule());
} }
public static final List<InputRow> ROWS1 = ImmutableList.of( public static final List<InputRow> ROWS1 = ImmutableList.of(
CalciteTests.createRow( TestDataBuilder.createRow(
ImmutableMap.<String, Object>builder() ImmutableMap.<String, Object>builder()
.put("t", "2000-01-01") .put("t", "2000-01-01")
.put("m1", "1") .put("m1", "1")
@@ -96,7 +82,7 @@ public class HdrHistogramQuantileSqlAggregatorTest extends CalciteTestBase {
.put("dim3", ImmutableList.of("a", "b")) .put("dim3", ImmutableList.of("a", "b"))
.build() .build()
), ),
CalciteTests.createRow( TestDataBuilder.createRow(
ImmutableMap.<String, Object>builder() ImmutableMap.<String, Object>builder()
.put("t", "2000-01-02") .put("t", "2000-01-02")
.put("m1", "2.0") .put("m1", "2.0")
@@ -106,7 +92,7 @@ public class HdrHistogramQuantileSqlAggregatorTest extends CalciteTestBase {
.put("dim3", ImmutableList.of("b", "c")) .put("dim3", ImmutableList.of("b", "c"))
.build() .build()
), ),
CalciteTests.createRow( TestDataBuilder.createRow(
ImmutableMap.<String, Object>builder() ImmutableMap.<String, Object>builder()
.put("t", "2000-01-03") .put("t", "2000-01-03")
.put("m1", "3.0") .put("m1", "3.0")
@@ -116,7 +102,7 @@ public class HdrHistogramQuantileSqlAggregatorTest extends CalciteTestBase {
.put("dim3", ImmutableList.of("d")) .put("dim3", ImmutableList.of("d"))
.build() .build()
), ),
CalciteTests.createRow( TestDataBuilder.createRow(
ImmutableMap.<String, Object>builder() ImmutableMap.<String, Object>builder()
.put("t", "2001-01-01") .put("t", "2001-01-01")
.put("m1", "4.0") .put("m1", "4.0")
@@ -126,7 +112,7 @@ public class HdrHistogramQuantileSqlAggregatorTest extends CalciteTestBase {
.put("dim3", ImmutableList.of("")) .put("dim3", ImmutableList.of(""))
.build() .build()
), ),
CalciteTests.createRow( TestDataBuilder.createRow(
ImmutableMap.<String, Object>builder() ImmutableMap.<String, Object>builder()
.put("t", "2001-01-02") .put("t", "2001-01-02")
.put("m1", "5.0") .put("m1", "5.0")
@@ -136,7 +122,7 @@ public class HdrHistogramQuantileSqlAggregatorTest extends CalciteTestBase {
.put("dim3", ImmutableList.of()) .put("dim3", ImmutableList.of())
.build() .build()
), ),
CalciteTests.createRow( TestDataBuilder.createRow(
ImmutableMap.<String, Object>builder() ImmutableMap.<String, Object>builder()
.put("t", "2001-01-03") .put("t", "2001-01-03")
.put("m1", "6.0") .put("m1", "6.0")
@@ -146,15 +132,20 @@ public class HdrHistogramQuantileSqlAggregatorTest extends CalciteTestBase {
) )
); );
@Before @SuppressWarnings("resource")
public void setUp() throws Exception { @Override
public SpecificSegmentsQuerySegmentWalker createQuerySegmentWalker(
final QueryRunnerFactoryConglomerate conglomerate,
final JoinableFactoryWrapper joinableFactory,
final Injector injector
) throws IOException{
HdrHistogramModule.registerSerde(); HdrHistogramModule.registerSerde();
for (Module mod : new HdrHistogramModule().getJacksonModules()) { for (Module mod : new HdrHistogramModule().getJacksonModules()) {
CalciteTests.getJsonMapper().registerModule(mod); CalciteTests.getJsonMapper().registerModule(mod);
TestHelper.JSON_MAPPER.registerModule(mod); TestHelper.JSON_MAPPER.registerModule(mod);
} }
//final QueryableIndex index = TestHelper.getTestIndexIO().loadIndex(new File("D:/doc/datas/testIndex-6201298"));
final QueryableIndex index = IndexBuilder.create() /*final QueryableIndex index = IndexBuilder.create()
.tmpDir(temporaryFolder.newFolder()) .tmpDir(temporaryFolder.newFolder())
.segmentWriteOutMediumFactory(OffHeapMemorySegmentWriteOutMediumFactory.instance()) .segmentWriteOutMediumFactory(OffHeapMemorySegmentWriteOutMediumFactory.instance())
.schema( .schema(
@@ -176,81 +167,183 @@ public class HdrHistogramQuantileSqlAggregatorTest extends CalciteTestBase {
) )
//.rows(CalciteTests.ROWS1) //.rows(CalciteTests.ROWS1)
.rows(ROWS1) .rows(ROWS1)
.buildMMappedIndex(); .buildMMappedIndex();*/
walker = new SpecificSegmentsQuerySegmentWalker(conglomerate).add( String[] files = new String[]{
DataSegment.builder() "D:\\doc\\datas\\statistics_rule_segments\\2023-10-16T00_00_00.000Z_2023-10-17T00_00_00.000Z\\2023-10-16T07_51_47.981Z\\0\\17a457e4-599d-49c2-86e7-6655851bb99a\\index",
.dataSource(DATA_SOURCE) "D:\\doc\\datas\\statistics_rule_segments\\2023-10-15T00_00_00.000Z_2023-10-16T00_00_00.000Z\\2023-10-15T00_00_04.240Z\\15\\9a766f6c-779d-4f9f-9ff5-6a12c19b8c6c\\index"
.interval(index.getDataInterval()) };
.version("1") files = new String[]{
.shardSpec(new LinearShardSpec(0)) "D:/doc/datas/testIndex-6201298"
.size(0) };
.build(), SpecificSegmentsQuerySegmentWalker walker = new SpecificSegmentsQuerySegmentWalker(conglomerate);
index
);
final PlannerConfig plannerConfig = new PlannerConfig(); for (int i = 0; i < files.length; i++) {
final DruidOperatorTable operatorTable = new DruidOperatorTable( QueryableIndex index = TestHelper.getTestIndexIO().loadIndex(new File(files[i]));
ImmutableSet.of( return walker.add(
new HdrHistogramQuantileSqlAggregator(), DataSegment.builder()
new HdrHistogramObjectSqlAggregator() .dataSource(CalciteTests.DATASOURCE1)
), .interval(index.getDataInterval())
ImmutableSet.of( .version("1")
new HdrHistogramQuantilesOperatorConversion(), .shardSpec(new LinearShardSpec(i))
new HdrHistogramPercentilesOperatorConversion() .size(0)
) .build(),
); index
SchemaPlus rootSchema = );
CalciteTests.createMockRootSchema(conglomerate, walker, plannerConfig, AuthTestUtils.TEST_AUTHORIZER_MAPPER); }
sqlLifecycleFactory = CalciteTests.createSqlLifecycleFactory( return walker;
new PlannerFactory(
rootSchema,
CalciteTests.createMockQueryLifecycleFactory(walker, conglomerate),
operatorTable,
CalciteTests.createExprMacroTable(),
plannerConfig,
AuthTestUtils.TEST_AUTHORIZER_MAPPER,
CalciteTests.getJsonMapper(),
CalciteTests.DRUID_SCHEMA_NAME
)
);
} }
@After @Test
public void tearDown() throws Exception { public void testCount0() throws Exception {
walker.close(); String sql = "select count(1) cnt, APPROX_QUANTILE_HDR(hist_m1, 0.5, 1, 100, 2) from druid.foo where dim1 = 'aaa'";
walker = null; QueryTestBuilder builder = testBuilder().sql(sql).skipVectorize();
builder.run();
QueryTestRunner.QueryResults queryResults = builder.results();
List<Object[]> results = queryResults.results;
for (Object[] result : results) {
System.out.println(Arrays.toString(result));
}
}
@Test
public void testSqlQueryError() throws Exception {
String sql = "select min(__time) min_time,max(__time) max_time, HDR_HISTOGRAM(latency_ms_sketch) hdr from druid.foo";
QueryTestBuilder builder = testBuilder().sql(sql).skipVectorize();
builder.run();
QueryTestRunner.QueryResults queryResults = builder.results();
List<Object[]> results = queryResults.results;
for (Object[] result : results) {
System.out.println(Arrays.toString(result));
}
} }
@Test @Test
public void testSqlQuery() throws Exception { public void testSqlQuery() throws Exception {
SqlLifecycle sqlLifecycle = sqlLifecycleFactory.factorize(); String[] columns = new String[]{"__time", "dim1", "dim2", "dim3", "cnt", "hist_m1", "m1"};
String sql = "select * from druid.foo"; String sql = "select " + String.join(",", columns) + " from druid.foo";
final List<Object[]> results = QueryTestBuilder builder = testBuilder().sql(sql);
sqlLifecycle.runSimple(sql, QUERY_CONTEXT_DEFAULT, DEFAULT_PARAMETERS, authenticationResult).toList(); builder.run();
QueryTestRunner.QueryResults queryResults = builder.results();
List<Object[]> results = queryResults.results;
for (Object[] result : results) {
Map row = new LinkedHashMap();
for (int i = 0; i < result.length; i++) {
row.put(columns[i], result[i]);
}
System.out.println(JSON.toJSONString(row));
// System.out.println(Arrays.toString(result));
}
for (int i = 0; i < columns.length; i++) {
Object[] values = new Object[results.size()];
for (int j = 0; j < results.size(); j++) {
values[j] = results.get(j)[i];
}
System.out.println(columns[i] + ":" + Arrays.toString(values));
}
}
@Test
public void testSqlQuery3() throws Exception {
//cannotVectorize();
//String sql = "select HLLD_ESTIMATE(HLLD(hll_dim1)) from druid.foo where dim1 = ''";
String sql = "select HDR_HISTOGRAM(hist_m1) hdr from druid.foo ";
QueryTestBuilder builder = testBuilder().sql(sql).skipVectorize();
builder.run();
QueryTestRunner.QueryResults queryResults = builder.results();
List<Object[]> results = queryResults.results;
for (Object[] result : results) {
System.out.println(Arrays.toString(result));
}
}
@Test
public void testSqlQuery4() throws Exception {
//cannotVectorize();
//String sql = "select HLLD_ESTIMATE(HLLD(hll_dim1)) from druid.foo where dim1 = ''";
String sql = "select APPROX_QUANTILE_HDR (hdr, 0.95) as p95th_tcp_latency_ms from (select HDR_HISTOGRAM(hist_m1) hdr from druid.foo) t ";
QueryTestBuilder builder = testBuilder().sql(sql).skipVectorize();
builder.run();
QueryTestRunner.QueryResults queryResults = builder.results();
List<Object[]> results = queryResults.results;
for (Object[] result : results) { for (Object[] result : results) {
System.out.println(Arrays.toString(result)); System.out.println(Arrays.toString(result));
} }
} }
@Test @Test
public void testGroup() throws Exception { public void testSqlQuery5() throws Exception {
SqlLifecycle sqlLifecycle = sqlLifecycleFactory.factorize(); //cannotVectorize();
String sql = "select cnt, APPROX_QUANTILE_HDR(hist_m1, 0.5, 1, 100, 2) from druid.foo group by cnt"; //String sql = "select HLLD_ESTIMATE(HLLD(hll_dim1)) from druid.foo where dim1 = ''";
final List<Object[]> results = String sql = "select dim1, APPROX_QUANTILE_HDR (hdr, 0.95) as p95th_tcp_latency_ms from (select dim1, HDR_HISTOGRAM(hist_m1) hdr from druid.foo group by dim1) t group by dim1";
sqlLifecycle.runSimple(sql, QUERY_CONTEXT_DEFAULT, DEFAULT_PARAMETERS, authenticationResult).toList(); QueryTestBuilder builder = testBuilder().sql(sql).skipVectorize();
builder.run();
QueryTestRunner.QueryResults queryResults = builder.results();
List<Object[]> results = queryResults.results;
for (Object[] result : results) { for (Object[] result : results) {
System.out.println(Arrays.toString(result)); System.out.println(Arrays.toString(result));
} }
} }
@Test
public void testSqlQuery6() throws Exception {
//cannotVectorize();
//String sql = "select HLLD_ESTIMATE(HLLD(hll_dim1)) from druid.foo where dim1 = ''";
//String sql = "select dim1, APPROX_QUANTILE_HDR (hdr, 0.95) as p95th_tcp_latency_ms from (select dim1, HDR_HISTOGRAM(hist_m1) hdr from druid.foo group by dim1 limit 10) t group by dim1";
String sql = "select dim1, HDR_GET_QUANTILES(HDR_HISTOGRAM(hdr), 0.95) as p95th_tcp_latency_ms from (select dim1, HDR_HISTOGRAM(hist_m1) hdr from druid.foo group by dim1 limit 10) t group by dim1";
QueryTestBuilder builder = testBuilder().sql(sql).skipVectorize();
builder.run();
QueryTestRunner.QueryResults queryResults = builder.results();
List<Object[]> results = queryResults.results;
for (Object[] result : results) {
System.out.println(Arrays.toString(result));
}
}
@Test
public void testGroup() throws Exception {
String sql = "select cnt, APPROX_QUANTILE_HDR(hist_m1, 0.5, 1, 100, 2) from druid.foo group by cnt";
QueryTestBuilder builder = testBuilder().sql(sql).skipVectorize();
builder.run();
QueryTestRunner.QueryResults queryResults = builder.results();
List<Object[]> results = queryResults.results;
for (Object[] result : results) {
System.out.println(Arrays.toString(result));
}
}
@Test @Test
public void testGroup2() throws Exception { public void testGroup2() throws Exception {
SqlLifecycle sqlLifecycle = sqlLifecycleFactory.factorize();
String sql = "select HDR_HISTOGRAM(hist_m1) from druid.foo"; String sql = "select HDR_HISTOGRAM(hist_m1) from druid.foo";
final List<Object[]> results = QueryTestBuilder builder = testBuilder().sql(sql).skipVectorize();
sqlLifecycle.runSimple(sql, QUERY_CONTEXT_DEFAULT, DEFAULT_PARAMETERS, authenticationResult).toList(); builder.run();
QueryTestRunner.QueryResults queryResults = builder.results();
List<Object[]> results = queryResults.results;
for (Object[] result : results) {
System.out.println(Arrays.toString(result));
}
}
@Test
public void testGroup3() throws Exception {
String sql = "select APPROX_QUANTILE_HDR(h, 0.5) from(select HDR_HISTOGRAM(hist_m1) h from druid.foo) t";
QueryTestBuilder builder = testBuilder().sql(sql).skipVectorize();
builder.run();
QueryTestRunner.QueryResults queryResults = builder.results();
List<Object[]> results = queryResults.results;
for (Object[] result : results) {
System.out.println(Arrays.toString(result));
}
}
@Test
public void testGroup4() throws Exception {
String sql = "select hdr_get_quantiles(h, 0.1, 0.2, 0.3, 0.5, 0.9, 0.99, 1) from(select HDR_HISTOGRAM(hist_m1) h from druid.foo) t";
QueryTestBuilder builder = testBuilder().sql(sql).skipVectorize();
builder.run();
QueryTestRunner.QueryResults queryResults = builder.results();
List<Object[]> results = queryResults.results;
for (Object[] result : results) { for (Object[] result : results) {
System.out.println(Arrays.toString(result)); System.out.println(Arrays.toString(result));
} }
@@ -258,10 +351,11 @@ public class HdrHistogramQuantileSqlAggregatorTest extends CalciteTestBase {
@Test @Test
public void testSqlQueryGeneHdr() throws Exception { public void testSqlQueryGeneHdr() throws Exception {
SqlLifecycle sqlLifecycle = sqlLifecycleFactory.factorize();
String sql = "select HDR_HISTOGRAM(hist_m1, 1, 100, 2), HDR_HISTOGRAM(cnt, 1, 100, 2) from druid.foo"; String sql = "select HDR_HISTOGRAM(hist_m1, 1, 100, 2), HDR_HISTOGRAM(cnt, 1, 100, 2) from druid.foo";
final List<Object[]> results = QueryTestBuilder builder = testBuilder().sql(sql).skipVectorize();
sqlLifecycle.runSimple(sql, QUERY_CONTEXT_DEFAULT, DEFAULT_PARAMETERS, authenticationResult).toList(); builder.run();
QueryTestRunner.QueryResults queryResults = builder.results();
List<Object[]> results = queryResults.results;
for (Object[] result : results) { for (Object[] result : results) {
System.out.println(Arrays.toString(result)); System.out.println(Arrays.toString(result));
} }
@@ -269,11 +363,12 @@ public class HdrHistogramQuantileSqlAggregatorTest extends CalciteTestBase {
@Test @Test
public void testSqlQueryGeneHdr2() throws Exception { public void testSqlQueryGeneHdr2() throws Exception {
SqlLifecycle sqlLifecycle = sqlLifecycleFactory.factorize();
// HDR_HISTOGRAM(hist_m1, 1, 100, 2), // HDR_HISTOGRAM(hist_m1, 1, 100, 2),
String sql = "select HDR_GET_QUANTILES(HDR_HISTOGRAM(m1, 1, 100, 2), 0.1, 0.2, 0.3, 0.5, 0.9, 1) from druid.foo"; String sql = "select HDR_GET_QUANTILES(HDR_HISTOGRAM(m1, 1, 100, 2), 0.1, 0.2, 0.3, 0.5, 0.9, 1) from druid.foo";
final List<Object[]> results = QueryTestBuilder builder = testBuilder().sql(sql).skipVectorize();
sqlLifecycle.runSimple(sql, QUERY_CONTEXT_DEFAULT, DEFAULT_PARAMETERS, authenticationResult).toList(); builder.run();
QueryTestRunner.QueryResults queryResults = builder.results();
List<Object[]> results = queryResults.results;
for (Object[] result : results) { for (Object[] result : results) {
System.out.println(Arrays.toString(result)); System.out.println(Arrays.toString(result));
} }
@@ -281,44 +376,47 @@ public class HdrHistogramQuantileSqlAggregatorTest extends CalciteTestBase {
@Test @Test
public void testSqlQueryGeneHdrArgs() throws Exception { public void testSqlQueryGeneHdrArgs() throws Exception {
SqlLifecycle sqlLifecycle = sqlLifecycleFactory.factorize();
String sql = "select HDR_GET_QUANTILEs(HDR_HISTOGRAM(m1), 0.1, 0.2, 0.3, 0.5, 0.9, 1), " String sql = "select HDR_GET_QUANTILEs(HDR_HISTOGRAM(m1), 0.1, 0.2, 0.3, 0.5, 0.9, 1), "
+ "HDR_GET_QUANTILEs(HDR_HISTOGRAM(m1, 2), 0.1, 0.2, 0.3, 0.5, 0.9, 1) ,\n" + "HDR_GET_QUANTILEs(HDR_HISTOGRAM(m1, 2), 0.1, 0.2, 0.3, 0.5, 0.9, 1) ,\n"
+ "HDR_GET_QUANTILEs(HDR_HISTOGRAM(m1, 1, 110, 2), 0.1, 0.2, 0.3, 0.5, 0.9, 1) ,\n" + "HDR_GET_QUANTILEs(HDR_HISTOGRAM(m1, 1, 110, 2), 0.1, 0.2, 0.3, 0.5, 0.9, 1) ,\n"
+ "HDR_GET_QUANTILEs(HDR_HISTOGRAM(m1, 1, 110, 2, false), 0.1, 0.2, 0.3, 0.5, 0.9, 1) \n" + "HDR_GET_QUANTILEs(HDR_HISTOGRAM(m1, 1, 110, 2, false), 0.1, 0.2, 0.3, 0.5, 0.9, 1) \n"
+ "from druid.foo"; + "from druid.foo";
final List<Object[]> results = QueryTestBuilder builder = testBuilder().sql(sql).skipVectorize();
sqlLifecycle.runSimple(sql, QUERY_CONTEXT_DEFAULT, DEFAULT_PARAMETERS, authenticationResult).toList(); builder.run();
QueryTestRunner.QueryResults queryResults = builder.results();
List<Object[]> results = queryResults.results;
for (Object[] result : results) { for (Object[] result : results) {
System.out.println(Arrays.toString(result)); System.out.println(Arrays.toString(result));
} }
} }
@Test @Test
public void testSqlQueryGeneHdrArgs2() throws Exception { public void testSqlQueryGeneHdrArgs2() throws Exception {
SqlLifecycle sqlLifecycle = sqlLifecycleFactory.factorize();
String sql = "select APPROX_QUANTILE_HDR(m1, 0.1), " String sql = "select APPROX_QUANTILE_HDR(m1, 0.1), "
+ "APPROX_QUANTILE_HDR(m1, 0.1, 2) ,\n" + "APPROX_QUANTILE_HDR(m1, 0.1, 2) ,\n"
+ "APPROX_QUANTILE_HDR(m1, 0.1, 1, 110, 2) ,\n" + "APPROX_QUANTILE_HDR(m1, 0.1, 1, 110, 2) ,\n"
+ "APPROX_QUANTILE_HDR(m1, 0.1, 1, 110, 2, false)\n" + "APPROX_QUANTILE_HDR(m1, 0.1, 1, 110, 2, false)\n"
+ "from druid.foo"; + "from druid.foo";
final List<Object[]> results = QueryTestBuilder builder = testBuilder().sql(sql).skipVectorize();
sqlLifecycle.runSimple(sql, QUERY_CONTEXT_DEFAULT, DEFAULT_PARAMETERS, authenticationResult).toList(); builder.run();
for (Object[] result : results) { QueryTestRunner.QueryResults queryResults = builder.results();
System.out.println(Arrays.toString(result)); List<Object[]> results = queryResults.results;
} for (Object[] result : results) {
System.out.println(Arrays.toString(result));
}
} }
@Test @Test
public void testSqlQueryGeneHdr3() throws Exception { public void testSqlQueryGeneHdr3() throws Exception {
SqlLifecycle sqlLifecycle = sqlLifecycleFactory.factorize();
// 函数不区分大小写 // 函数不区分大小写
// HDR_HISTOGRAM(hist_m1, 1, 100, 2), // HDR_HISTOGRAM(hist_m1, 1, 100, 2),
//String sql = "select HDR_GET_PERCENTILES(HDR_HISTOGRAM(m1, 1, 100, 2)) from druid.foo"; //String sql = "select HDR_GET_PERCENTILES(HDR_HISTOGRAM(m1, 1, 100, 2)) from druid.foo";
//String sql = "select hdr_get_percentiles(hdr_histogram(m1, 1, 100, 2)) from druid.foo"; //String sql = "select hdr_get_percentiles(hdr_histogram(m1, 1, 100, 2)) from druid.foo";
String sql = "select hdr_get_percentiles(hdr_histogram(hist_m1, 1, 100, 2)) from druid.foo"; String sql = "select hdr_get_percentiles(hdr_histogram(hist_m1, 1, 100, 2)) from druid.foo";
final List<Object[]> results = QueryTestBuilder builder = testBuilder().sql(sql).skipVectorize();
sqlLifecycle.runSimple(sql, QUERY_CONTEXT_DEFAULT, DEFAULT_PARAMETERS, authenticationResult).toList(); builder.run();
QueryTestRunner.QueryResults queryResults = builder.results();
List<Object[]> results = queryResults.results;
for (Object[] result : results) { for (Object[] result : results) {
System.out.println(Arrays.toString(result)); System.out.println(Arrays.toString(result));
} }
@@ -326,7 +424,6 @@ public class HdrHistogramQuantileSqlAggregatorTest extends CalciteTestBase {
@Test @Test
public void testSqlQueryQuantiles() throws Exception { public void testSqlQueryQuantiles() throws Exception {
SqlLifecycle sqlLifecycle = sqlLifecycleFactory.factorize();
String sql = "SELECT\n" String sql = "SELECT\n"
+ "APPROX_QUANTILE_HDR(m1, 0.01, 1, 100, 2),\n" + "APPROX_QUANTILE_HDR(m1, 0.01, 1, 100, 2),\n"
+ "APPROX_QUANTILE_HDR(m1, 0.5, 1, 100, 2),\n" + "APPROX_QUANTILE_HDR(m1, 0.5, 1, 100, 2),\n"
@@ -338,9 +435,10 @@ public class HdrHistogramQuantileSqlAggregatorTest extends CalciteTestBase {
+ "APPROX_QUANTILE_HDR(m1, 0.999, 1, 100, 2) FILTER(WHERE dim1 = 'abc'),\n" + "APPROX_QUANTILE_HDR(m1, 0.999, 1, 100, 2) FILTER(WHERE dim1 = 'abc'),\n"
+ "APPROX_QUANTILE_HDR(cnt, 0.5, 1, 100, 2)\n" + "APPROX_QUANTILE_HDR(cnt, 0.5, 1, 100, 2)\n"
+ "FROM foo"; + "FROM foo";
final List<Object[]> results = QueryTestBuilder builder = testBuilder().sql(sql).skipVectorize();
sqlLifecycle.runSimple(sql, QUERY_CONTEXT_DEFAULT, DEFAULT_PARAMETERS, authenticationResult).toList(); builder.run();
System.out.println(sql); QueryTestRunner.QueryResults queryResults = builder.results();
List<Object[]> results = queryResults.results;
for (Object[] result : results) { for (Object[] result : results) {
System.out.println(Arrays.toString(result)); System.out.println(Arrays.toString(result));
} }
@@ -348,7 +446,6 @@ public class HdrHistogramQuantileSqlAggregatorTest extends CalciteTestBase {
@Test @Test
public void testSqlQueryQuantilesOnComplexColumn() throws Exception { public void testSqlQueryQuantilesOnComplexColumn() throws Exception {
SqlLifecycle sqlLifecycle = sqlLifecycleFactory.factorize();
String sql = "SELECT\n" String sql = "SELECT\n"
+ "APPROX_QUANTILE_HDR(hist_m1, 0.01, 1, 100, 2),\n" + "APPROX_QUANTILE_HDR(hist_m1, 0.01, 1, 100, 2),\n"
+ "APPROX_QUANTILE_HDR(hist_m1, 0.5, 1, 100, 2),\n" + "APPROX_QUANTILE_HDR(hist_m1, 0.5, 1, 100, 2),\n"
@@ -358,9 +455,10 @@ public class HdrHistogramQuantileSqlAggregatorTest extends CalciteTestBase {
+ "APPROX_QUANTILE_HDR(hist_m1, 0.999, 1, 100, 2) FILTER(WHERE dim1 <> 'abc'),\n" + "APPROX_QUANTILE_HDR(hist_m1, 0.999, 1, 100, 2) FILTER(WHERE dim1 <> 'abc'),\n"
+ "APPROX_QUANTILE_HDR(hist_m1, 0.999, 1, 100, 2) FILTER(WHERE dim1 = 'abc')\n" + "APPROX_QUANTILE_HDR(hist_m1, 0.999, 1, 100, 2) FILTER(WHERE dim1 = 'abc')\n"
+ "FROM foo"; + "FROM foo";
final List<Object[]> results = QueryTestBuilder builder = testBuilder().sql(sql).skipVectorize();
sqlLifecycle.runSimple(sql, QUERY_CONTEXT_DEFAULT, DEFAULT_PARAMETERS, authenticationResult).toList(); builder.run();
System.out.println(sql); QueryTestRunner.QueryResults queryResults = builder.results();
List<Object[]> results = queryResults.results;
for (Object[] result : results) { for (Object[] result : results) {
System.out.println(Arrays.toString(result)); System.out.println(Arrays.toString(result));
} }
@@ -373,7 +471,6 @@ public class HdrHistogramQuantileSqlAggregatorTest extends CalciteTestBase {
@Test @Test
public void testQuantileOnFloatAndLongs() throws Exception { public void testQuantileOnFloatAndLongs() throws Exception {
SqlLifecycle sqlLifecycle = sqlLifecycleFactory.factorize();
String sql = "SELECT\n" String sql = "SELECT\n"
+ "APPROX_QUANTILE_HDR(m1, 0.01, 1, 100, 2),\n" + "APPROX_QUANTILE_HDR(m1, 0.01, 1, 100, 2),\n"
+ "APPROX_QUANTILE_HDR(m1, 0.5, 1, 100, 2),\n" + "APPROX_QUANTILE_HDR(m1, 0.5, 1, 100, 2),\n"
@@ -385,60 +482,55 @@ public class HdrHistogramQuantileSqlAggregatorTest extends CalciteTestBase {
+ "APPROX_QUANTILE_HDR(m1, 0.999, 1, 100, 2) FILTER(WHERE dim1 = 'abc'),\n" + "APPROX_QUANTILE_HDR(m1, 0.999, 1, 100, 2) FILTER(WHERE dim1 = 'abc'),\n"
+ "APPROX_QUANTILE_HDR(cnt, 0.5, 1, 100, 2)\n" + "APPROX_QUANTILE_HDR(cnt, 0.5, 1, 100, 2)\n"
+ "FROM foo"; + "FROM foo";
final List<Object[]> results = QueryTestBuilder builder = testBuilder().sql(sql).skipVectorize();
sqlLifecycle.runSimple(sql, QUERY_CONTEXT_DEFAULT, DEFAULT_PARAMETERS, authenticationResult).toList(); builder = builder.expectedQueries(Collections.singletonList(Druids.newTimeseriesQueryBuilder()
System.out.println(sql); .dataSource(CalciteTests.DATASOURCE1)
.intervals(new MultipleIntervalSegmentSpec(ImmutableList.of(Filtration.eternity())))
.granularity(Granularities.ALL)
.virtualColumns(
new ExpressionVirtualColumn(
"v0",
"(\"m1\" * 2)",
ColumnType.LONG,
TestExprMacroTable.INSTANCE
)
)
.aggregators(ImmutableList.of(
new HdrHistogramAggregatorFactory("a0:agg", "m1", 1L, 100L, 2, true),
new HdrHistogramAggregatorFactory("a4:agg", "v0", 1L, 100L, 2, true),
new FilteredAggregatorFactory(
new HdrHistogramAggregatorFactory("a5:agg", "m1", 1L, 100L, 2, true),
new SelectorDimFilter("dim1", "abc", null)
),
new FilteredAggregatorFactory(
new HdrHistogramAggregatorFactory("a6:agg", "m1", 1L, 100L, 2, true),
new NotDimFilter(new SelectorDimFilter("dim1", "abc", null))
),
new HdrHistogramAggregatorFactory("a8:agg", "cnt", 1L, 100L, 2, true)
))
.postAggregators(
new HdrHistogramToQuantilePostAggregator("a0", "a0:agg", 0.01f),
new HdrHistogramToQuantilePostAggregator("a1", "a0:agg", 0.50f),
new HdrHistogramToQuantilePostAggregator("a2", "a0:agg", 0.98f),
new HdrHistogramToQuantilePostAggregator("a3", "a0:agg", 0.99f),
new HdrHistogramToQuantilePostAggregator("a4", "a4:agg", 0.97f),
new HdrHistogramToQuantilePostAggregator("a5", "a5:agg", 0.99f),
new HdrHistogramToQuantilePostAggregator("a6", "a6:agg", 0.999f),
new HdrHistogramToQuantilePostAggregator("a7", "a5:agg", 0.999f),
new HdrHistogramToQuantilePostAggregator("a8", "a8:agg", 0.50f)
)
.context(QUERY_CONTEXT_DEFAULT)
.build()));
builder.run();
QueryTestRunner.QueryResults queryResults = builder.results();
List<Object[]> results = queryResults.results;
for (Object[] result : results) { for (Object[] result : results) {
System.out.println(Arrays.toString(result)); System.out.println(Arrays.toString(result));
} }
// Verify query
Assert.assertEquals(
Druids.newTimeseriesQueryBuilder()
.dataSource(CalciteTests.DATASOURCE1)
.intervals(new MultipleIntervalSegmentSpec(ImmutableList.of(Filtration.eternity())))
.granularity(Granularities.ALL)
.virtualColumns(
new ExpressionVirtualColumn(
"v0",
"(\"m1\" * 2)",
ValueType.LONG,
TestExprMacroTable.INSTANCE
)
)
.aggregators(ImmutableList.of(
new HdrHistogramAggregatorFactory("a0:agg", "m1", 1L, 100L, 2, true),
new HdrHistogramAggregatorFactory("a4:agg", "v0", 1L, 100L, 2, true),
new FilteredAggregatorFactory(
new HdrHistogramAggregatorFactory("a5:agg", "m1", 1L, 100L, 2, true),
new SelectorDimFilter("dim1", "abc", null)
),
new FilteredAggregatorFactory(
new HdrHistogramAggregatorFactory("a6:agg", "m1", 1L, 100L, 2, true),
new NotDimFilter(new SelectorDimFilter("dim1", "abc", null))
),
new HdrHistogramAggregatorFactory("a8:agg", "cnt", 1L, 100L, 2, true)
))
.postAggregators(
new HdrHistogramToQuantilePostAggregator("a0", "a0:agg", 0.01f),
new HdrHistogramToQuantilePostAggregator("a1", "a0:agg", 0.50f),
new HdrHistogramToQuantilePostAggregator("a2", "a0:agg", 0.98f),
new HdrHistogramToQuantilePostAggregator("a3", "a0:agg", 0.99f),
new HdrHistogramToQuantilePostAggregator("a4", "a4:agg", 0.97f),
new HdrHistogramToQuantilePostAggregator("a5", "a5:agg", 0.99f),
new HdrHistogramToQuantilePostAggregator("a6", "a6:agg", 0.999f),
new HdrHistogramToQuantilePostAggregator("a7", "a5:agg", 0.999f),
new HdrHistogramToQuantilePostAggregator("a8", "a8:agg", 0.50f)
)
.context(ImmutableMap.of("skipEmptyBuckets", true, PlannerContext.CTX_SQL_QUERY_ID, "dummy"))
.build(),
Iterables.getOnlyElement(queryLogHook.getRecordedQueries())
);
} }
@Test @Test
public void testQuantileOnComplexColumn() throws Exception{ public void testQuantileOnComplexColumn() throws Exception{
SqlLifecycle sqlLifecycle = sqlLifecycleFactory.factorize();
String sql = "SELECT\n" String sql = "SELECT\n"
+ "APPROX_QUANTILE_HDR(hist_m1, 0.01, 1, 100, 2),\n" + "APPROX_QUANTILE_HDR(hist_m1, 0.01, 1, 100, 2),\n"
+ "APPROX_QUANTILE_HDR(hist_m1, 0.5, 1, 100, 2),\n" + "APPROX_QUANTILE_HDR(hist_m1, 0.5, 1, 100, 2),\n"
@@ -448,43 +540,42 @@ public class HdrHistogramQuantileSqlAggregatorTest extends CalciteTestBase {
+ "APPROX_QUANTILE_HDR(hist_m1, 0.999, 1, 100, 2) FILTER(WHERE dim1 <> 'abc'),\n" + "APPROX_QUANTILE_HDR(hist_m1, 0.999, 1, 100, 2) FILTER(WHERE dim1 <> 'abc'),\n"
+ "APPROX_QUANTILE_HDR(hist_m1, 0.999, 1, 100, 2) FILTER(WHERE dim1 = 'abc')\n" + "APPROX_QUANTILE_HDR(hist_m1, 0.999, 1, 100, 2) FILTER(WHERE dim1 = 'abc')\n"
+ "FROM foo"; + "FROM foo";
final List<Object[]> results = QueryTestBuilder builder = testBuilder().sql(sql).skipVectorize();
sqlLifecycle.runSimple(sql, QUERY_CONTEXT_DEFAULT, DEFAULT_PARAMETERS, authenticationResult).toList(); builder = builder.expectedQueries(Collections.singletonList(Druids.newTimeseriesQueryBuilder()
System.out.println(sql); .dataSource(CalciteTests.DATASOURCE1)
.intervals(new MultipleIntervalSegmentSpec(ImmutableList.of(Filtration.eternity())))
.granularity(Granularities.ALL)
.aggregators(ImmutableList.of(
new HdrHistogramMergeAggregatorFactory("a0:agg", "hist_m1", 1L, 100L, 2, true),
new FilteredAggregatorFactory(
new HdrHistogramMergeAggregatorFactory("a4:agg", "hist_m1", 1L, 100L, 2, true),
new SelectorDimFilter("dim1", "abc", null)
),
new FilteredAggregatorFactory(
new HdrHistogramMergeAggregatorFactory("a5:agg", "hist_m1", 1L, 100L, 2, true),
new NotDimFilter(new SelectorDimFilter("dim1", "abc", null))
)
))
.postAggregators(
new HdrHistogramToQuantilePostAggregator("a0", "a0:agg", 0.01f),
new HdrHistogramToQuantilePostAggregator("a1", "a0:agg", 0.50f),
new HdrHistogramToQuantilePostAggregator("a2", "a0:agg", 0.98f),
new HdrHistogramToQuantilePostAggregator("a3", "a0:agg", 0.99f),
new HdrHistogramToQuantilePostAggregator("a4", "a4:agg", 0.99f),
new HdrHistogramToQuantilePostAggregator("a5", "a5:agg", 0.999f),
new HdrHistogramToQuantilePostAggregator("a6", "a4:agg", 0.999f)
)
.context(QUERY_CONTEXT_DEFAULT)
.build()));
builder.run();
QueryTestRunner.QueryResults queryResults = builder.results();
List<Object[]> results = queryResults.results;
for (Object[] result : results) { for (Object[] result : results) {
System.out.println(Arrays.toString(result)); System.out.println(Arrays.toString(result));
} }
// Verify query
Assert.assertEquals(
Druids.newTimeseriesQueryBuilder()
.dataSource(CalciteTests.DATASOURCE1)
.intervals(new MultipleIntervalSegmentSpec(ImmutableList.of(Filtration.eternity())))
.granularity(Granularities.ALL)
.aggregators(ImmutableList.of(
new HdrHistogramMergeAggregatorFactory("a0:agg", "hist_m1", 1L, 100L, 2, true),
new FilteredAggregatorFactory(
new HdrHistogramMergeAggregatorFactory("a4:agg", "hist_m1", 1L, 100L, 2, true),
new SelectorDimFilter("dim1", "abc", null)
),
new FilteredAggregatorFactory(
new HdrHistogramMergeAggregatorFactory("a5:agg", "hist_m1", 1L, 100L, 2, true),
new NotDimFilter(new SelectorDimFilter("dim1", "abc", null))
)
))
.postAggregators(
new HdrHistogramToQuantilePostAggregator("a0", "a0:agg", 0.01f),
new HdrHistogramToQuantilePostAggregator("a1", "a0:agg", 0.50f),
new HdrHistogramToQuantilePostAggregator("a2", "a0:agg", 0.98f),
new HdrHistogramToQuantilePostAggregator("a3", "a0:agg", 0.99f),
new HdrHistogramToQuantilePostAggregator("a4", "a4:agg", 0.99f),
new HdrHistogramToQuantilePostAggregator("a5", "a5:agg", 0.999f),
new HdrHistogramToQuantilePostAggregator("a6", "a4:agg", 0.999f)
)
.context(ImmutableMap.of("skipEmptyBuckets", true, PlannerContext.CTX_SQL_QUERY_ID, "dummy"))
.build(),
Iterables.getOnlyElement(queryLogHook.getRecordedQueries())
);
} }
private static PostAggregator makeFieldAccessPostAgg(String name) { private static PostAggregator makeFieldAccessPostAgg(String name) {

22
druid-hlld/pom.xml
View File

@@ -5,7 +5,7 @@
<modelVersion>4.0.0</modelVersion> <modelVersion>4.0.0</modelVersion>
<groupId>org.apache.druid.extensions</groupId> <groupId>org.apache.druid.extensions</groupId>
<artifactId>druid-hlld_0.18.1</artifactId> <artifactId>druid-hlld_26.0.0</artifactId>
<name>druid-hlld</name> <name>druid-hlld</name>
<version>1.0-SNAPSHOT</version> <version>1.0-SNAPSHOT</version>
@@ -14,7 +14,7 @@
<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding> <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
<maven.compiler.source>1.8</maven.compiler.source> <maven.compiler.source>1.8</maven.compiler.source>
<maven.compiler.target>1.8</maven.compiler.target> <maven.compiler.target>1.8</maven.compiler.target>
<druid.version>0.18.1</druid.version> <druid.version>26.0.0</druid.version>
</properties> </properties>
<dependencies> <dependencies>
@@ -33,6 +33,14 @@
</dependency> </dependency>
<!-- Tests --> <!-- Tests -->
<dependency>
<groupId>org.easymock</groupId>
<artifactId>easymock</artifactId>
<version>4.3</version>
<scope>test</scope>
</dependency>
<dependency> <dependency>
<groupId>org.apache.druid</groupId> <groupId>org.apache.druid</groupId>
<artifactId>druid-processing</artifactId> <artifactId>druid-processing</artifactId>
@@ -42,9 +50,17 @@
</dependency> </dependency>
<dependency> <dependency>
<groupId>org.apache.druid</groupId> <groupId>org.apache.druid</groupId>
<artifactId>druid-benchmarks</artifactId> <artifactId>druid-server</artifactId>
<version>${druid.version}</version> <version>${druid.version}</version>
<scope>test</scope> <scope>test</scope>
<type>test-jar</type>
</dependency>
<dependency>
<groupId>org.apache.druid</groupId>
<artifactId>druid-sql</artifactId>
<version>${druid.version}</version>
<type>test-jar</type>
<scope>test</scope>
</dependency> </dependency>
<dependency> <dependency>
<groupId>junit</groupId> <groupId>junit</groupId>

543
druid-hlld/src/main/java/org/apache/druid/query/aggregation/sketch/hlld/HllAggregatorFactory.java
View File

@@ -1,256 +1,287 @@
package org.apache.druid.query.aggregation.sketch.hlld; package org.apache.druid.query.aggregation.sketch.hlld;
import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonProperty;
import com.zdjz.galaxy.sketch.hlld.Hll; import com.zdjz.galaxy.sketch.hlld.Hll;
import com.zdjz.galaxy.sketch.hlld.HllUnion; import com.zdjz.galaxy.sketch.hlld.HllUnion;
import org.apache.druid.java.util.common.IAE; import org.apache.druid.java.util.common.IAE;
import org.apache.druid.java.util.common.logger.Logger; import org.apache.druid.java.util.common.logger.Logger;
import org.apache.druid.query.aggregation.*; import org.apache.druid.query.aggregation.*;
import org.apache.druid.query.cache.CacheKeyBuilder; import org.apache.druid.query.cache.CacheKeyBuilder;
import org.apache.druid.segment.ColumnSelectorFactory; import org.apache.druid.segment.ColumnSelectorFactory;
import org.apache.druid.segment.ColumnValueSelector; import org.apache.druid.segment.ColumnValueSelector;
import org.apache.druid.segment.column.ColumnType;
import javax.annotation.Nullable;
import java.util.Collections; import javax.annotation.Nullable;
import java.util.Comparator; import java.util.Collections;
import java.util.List; import java.util.Comparator;
import java.util.Objects; import java.util.List;
import java.util.Objects;
public class HllAggregatorFactory extends AggregatorFactory {
private static final Logger LOG = new Logger(HllAggregatorFactory.class); public class HllAggregatorFactory extends AggregatorFactory {
public static final boolean DEFAULT_ROUND = false; private static final Logger LOG = new Logger(HllAggregatorFactory.class);
public static final int DEFAULT_PRECISION = 12; public static final boolean DEFAULT_ROUND = false;
public static final int DEFAULT_PRECISION = 12;
static final Comparator<Hll> COMPARATOR = Comparator.nullsFirst(Comparator.comparingDouble(Hll::size));
static final Comparator<Hll> COMPARATOR = Comparator.nullsFirst(Comparator.comparingDouble(Hll::size));
protected final String name;
protected final String fieldName; protected final String name;
protected final int precision; protected final String fieldName;
protected final boolean round; protected final int precision;
protected final boolean round;
public HllAggregatorFactory( protected final int updatableSerializationBytes;
@JsonProperty("name") final String name,
@JsonProperty("fieldName") final String fieldName, public HllAggregatorFactory(
@JsonProperty("precision") @Nullable final Integer precision, @JsonProperty("name") final String name,
@JsonProperty("round") @Nullable final Boolean round @JsonProperty("fieldName") final String fieldName,
) { @JsonProperty("precision") @Nullable final Integer precision,
if (name == null) { @JsonProperty("round") @Nullable final Boolean round
throw new IAE("Must have a valid, non-null aggregator name"); ) {
} if (name == null) {
if (fieldName == null) { throw new IAE("Must have a valid, non-null aggregator name");
throw new IAE("Parameter fieldName must be specified"); }
} if (fieldName == null) {
this.name = name; throw new IAE("Parameter fieldName must be specified");
this.fieldName = fieldName; }
this.precision = precision == null ? DEFAULT_PRECISION : precision; this.name = name;
this.round = round == null ? DEFAULT_ROUND : round; this.fieldName = fieldName;
} this.precision = precision == null ? DEFAULT_PRECISION : precision;
this.round = round == null ? DEFAULT_ROUND : round;
@Override this.updatableSerializationBytes = getUpdatableSerializationBytes();
public Aggregator factorize(ColumnSelectorFactory columnSelectorFactory) { }
final ColumnValueSelector<Object> selector = columnSelectorFactory.makeColumnValueSelector(fieldName);
return new HllAggregator(selector, precision); @Override
} public Aggregator factorize(ColumnSelectorFactory columnSelectorFactory) {
final ColumnValueSelector<Object> selector = columnSelectorFactory.makeColumnValueSelector(fieldName);
@Override return new HllAggregator(selector, precision);
public BufferAggregator factorizeBuffered(ColumnSelectorFactory columnSelectorFactory) { }
final ColumnValueSelector<Object> selector = columnSelectorFactory.makeColumnValueSelector(fieldName);
return new HllBufferAggregator( @Override
selector, public BufferAggregator factorizeBuffered(ColumnSelectorFactory columnSelectorFactory) {
precision final ColumnValueSelector<Object> selector = columnSelectorFactory.makeColumnValueSelector(fieldName);
); return new HllBufferAggregator(
} selector,
precision
@Override );
public Comparator getComparator() { }
return COMPARATOR;
} @Override
public Comparator getComparator() {
@Override return COMPARATOR;
public Object combine(Object lhs, Object rhs) { }
if(lhs == null){
return rhs; @Override
}else if(rhs == null){ public Object combine(Object lhs, Object rhs) {
return lhs; if(lhs == null){
}else{ return rhs;
final HllUnion union = new HllUnion(precision); }else if(rhs == null){
union.update((Hll) lhs); return lhs;
union.update((Hll) rhs); }else{
Hll result = union.getResult(); final HllUnion union = new HllUnion(precision);
return result; union.update((Hll) lhs);
} union.update((Hll) rhs);
} Hll result = union.getResult();
return result;
@Override }
public AggregateCombiner makeAggregateCombiner() { }
return new ObjectAggregateCombiner<Hll>() {
private HllUnion union = null; @Override
public AggregateCombiner makeAggregateCombiner() {
@Override return new ObjectAggregateCombiner<Hll>() {
public void reset(ColumnValueSelector selector) { private HllUnion union = null;
//LOG.error("HllAggregateCombiner reset:" + "-" + Thread.currentThread().getId() + "-" + this);
//union.reset(); @Override
union = null; public void reset(ColumnValueSelector selector) {
fold(selector); //LOG.error("HllAggregateCombiner reset:" + "-" + Thread.currentThread().getId() + "-" + this);
} //union.reset();
union = null;
@Override fold(selector);
public void fold(ColumnValueSelector selector) { }
//LOG.error("HllAggregateCombiner fold:" + "-" + Thread.currentThread().getId() + "-" + this);
final Hll hll = (Hll) selector.getObject(); @Override
if(hll != null){ public void fold(ColumnValueSelector selector) {
if(union == null){ //LOG.error("HllAggregateCombiner fold:" + "-" + Thread.currentThread().getId() + "-" + this);
union = new HllUnion(precision); final Hll hll = (Hll) selector.getObject();
} if(hll != null){
union.update(hll); if(union == null){
}else{ union = new HllUnion(precision);
//LOG.error("HllAggregateCombiner fold_null:" + "-" + Thread.currentThread().getId() + "-" + this); }
} union.update(hll);
} }else{
//LOG.error("HllAggregateCombiner fold_null:" + "-" + Thread.currentThread().getId() + "-" + this);
@Override }
public Class<Hll> classOfObject() { }
return Hll.class;
} @Override
public Class<Hll> classOfObject() {
@Nullable return Hll.class;
@Override }
public Hll getObject() {
//LOG.error("HllAggregateCombiner get:" + "-" + Thread.currentThread().getId() + "-" + this); @Nullable
if(union == null){ @Override
return null; public Hll getObject() {
}else{ //LOG.error("HllAggregateCombiner get:" + "-" + Thread.currentThread().getId() + "-" + this);
Hll result = union.getResult(); if(union == null){
/*if(result.size() == 0){ return null;
return null; }else{
}*/ Hll result = union.getResult();
return result; /*if(result.size() == 0){
} return null;
} }*/
}; return result;
} }
}
@Override };
public AggregatorFactory getCombiningFactory() { }
// 千万不能写错，好大一个坑
return new HllMergeAggregatorFactory(name, name, precision, round); @Override
} public AggregatorFactory getCombiningFactory() {
// 千万不能写错，好大一个坑
@Override return new HllMergeAggregatorFactory(name, name, precision, round);
public AggregatorFactory getMergingFactory(AggregatorFactory other) throws AggregatorFactoryNotMergeableException { }
if (other.getName().equals(this.getName()) && other instanceof HllAggregatorFactory) {
HllAggregatorFactory castedOther = (HllAggregatorFactory) other; @Override
public AggregatorFactory getMergingFactory(AggregatorFactory other) throws AggregatorFactoryNotMergeableException {
return new HllMergeAggregatorFactory(name, name, if (other.getName().equals(this.getName()) && other instanceof HllAggregatorFactory) {
Math.max(precision, castedOther.precision), HllAggregatorFactory castedOther = (HllAggregatorFactory) other;
round || castedOther.round
); return new HllMergeAggregatorFactory(name, name,
} else { Math.max(precision, castedOther.precision),
throw new AggregatorFactoryNotMergeableException(this, other); round || castedOther.round
} );
} }
@Override throw new AggregatorFactoryNotMergeableException(this, other);
public List<AggregatorFactory> getRequiredColumns() { }
return Collections.singletonList(
new HllAggregatorFactory(fieldName, fieldName, precision, round) @Override
); public List<AggregatorFactory> getRequiredColumns() {
} return Collections.singletonList(
new HllAggregatorFactory(fieldName, fieldName, precision, round)
@Override );
public Object deserialize(Object object) { }
return HllUtils.deserializeHll(object);
} @Override
public AggregatorFactory withName(String newName) {
@Nullable return new HllAggregatorFactory(newName, fieldName, precision, round);
@Override }
public Object finalizeComputation(@Nullable Object object) {
if (object == null) { @Override
return null; public Object deserialize(Object object) {
} if (object == null) {
final Hll hll = (Hll) object; return null;
final double estimate = hll.size(); }
return HllUtils.deserializeHll(object);
if (round) { }
return Math.round(estimate);
} else { @Override
return estimate; public ColumnType getResultType() {
} //return round ? ColumnType.LONG : ColumnType.DOUBLE;
} return getIntermediateType();
}
@Override
@JsonProperty @Nullable
public String getName() { @Override
return name; public Object finalizeComputation(@Nullable Object object) {
} if (object == null) {
return null;
@JsonProperty }
public String getFieldName() {
return fieldName; return object;
}
/*final Hll hll = (Hll) object;
@JsonProperty final double estimate = hll.size();
public int getPrecision() {
return precision; if (round) {
} return Math.round(estimate);
} else {
@JsonProperty return estimate;
public boolean isRound() { }*/
return round; }
}
@Override
@Override @JsonProperty
public String getTypeName() { public String getName() {
return HllModule.HLLD_BUILD_TYPE_NAME; return name;
} }
@Override @JsonProperty
public List<String> requiredFields() { public String getFieldName() {
return Collections.singletonList(fieldName); return fieldName;
} }
@Override @JsonProperty
public int getMaxIntermediateSize() { public int getPrecision() {
return Hll.getUpdatableSerializationBytes(precision); return precision;
} }
@Override @JsonProperty
public byte[] getCacheKey() { public boolean isRound() {
return new CacheKeyBuilder(HllModule.CACHE_TYPE_ID_OFFSET).appendByte(HllModule.HLLD_BUILD_CACHE_TYPE_ID) return round;
.appendString(name).appendString(fieldName) }
.appendInt(precision).appendBoolean(round)
.build(); /*
} 没这个方法了, 新版本需要实现getIntermediateType方法
@Override
@Override public String getTypeName() {
public boolean equals(final Object o){ return HllModule.HLLD_BUILD_TYPE_NAME;
if (this == o) { }*/
return true;
} @Override
if (o == null || !getClass().equals(o.getClass())) { public ColumnType getIntermediateType() {
return false; return HllModule.BUILD_TYPE;
} }
HllAggregatorFactory that = (HllAggregatorFactory) o; @Override
return name.equals(that.name) && fieldName.equals(that.fieldName) && public List<String> requiredFields() {
precision == that.precision && return Collections.singletonList(fieldName);
round == that.round }
;
} @Override
public int getMaxIntermediateSize() {
@Override return updatableSerializationBytes == 0? getUpdatableSerializationBytes():updatableSerializationBytes;
public int hashCode(){ }
return Objects.hash(name, fieldName, precision, round);
} protected int getUpdatableSerializationBytes(){
return Hll.getUpdatableSerializationBytes(precision);
}
@Override
public String toString() { @Override
return getClass().getSimpleName() + "{" + public byte[] getCacheKey() {
"name='" + name + '\'' + return new CacheKeyBuilder(HllModule.CACHE_TYPE_ID_OFFSET).appendByte(HllModule.HLLD_BUILD_CACHE_TYPE_ID)
", fieldName='" + fieldName + '\'' + .appendString(name).appendString(fieldName)
", precision=" + precision + .appendInt(precision).appendBoolean(round)
", round=" + round + .build();
'}'; }
}
} @Override
public boolean equals(final Object o){
if (this == o) {
return true;
}
if (o == null || !getClass().equals(o.getClass())) {
return false;
}
HllAggregatorFactory that = (HllAggregatorFactory) o;
return name.equals(that.name) && fieldName.equals(that.fieldName) &&
precision == that.precision &&
round == that.round
;
}
@Override
public int hashCode(){
return Objects.hash(name, fieldName, precision, round);
}
@Override
public String toString() {
return getClass().getSimpleName() + "{" +
"name='" + name + '\'' +
", fieldName='" + fieldName + '\'' +
", precision=" + precision +
", round=" + round +
'}';
}
}

132
druid-hlld/src/main/java/org/apache/druid/query/aggregation/sketch/hlld/HllMergeAggregatorFactory.java
View File

@@ -1,59 +1,73 @@
package org.apache.druid.query.aggregation.sketch.hlld; package org.apache.druid.query.aggregation.sketch.hlld;
import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonProperty;
import com.zdjz.galaxy.sketch.hlld.Hll; import com.zdjz.galaxy.sketch.hlld.Hll;
import com.zdjz.galaxy.sketch.hlld.HllUnion; import com.zdjz.galaxy.sketch.hlld.HllUnion;
import org.apache.druid.query.aggregation.Aggregator; import org.apache.druid.query.aggregation.Aggregator;
import org.apache.druid.query.aggregation.BufferAggregator; import org.apache.druid.query.aggregation.AggregatorFactory;
import org.apache.druid.query.cache.CacheKeyBuilder; import org.apache.druid.query.aggregation.BufferAggregator;
import org.apache.druid.segment.ColumnSelectorFactory; import org.apache.druid.query.cache.CacheKeyBuilder;
import org.apache.druid.segment.ColumnValueSelector; import org.apache.druid.segment.ColumnSelectorFactory;
import org.apache.druid.segment.ColumnValueSelector;
import javax.annotation.Nullable; import org.apache.druid.segment.column.ColumnType;
public class HllMergeAggregatorFactory extends HllAggregatorFactory{ import javax.annotation.Nullable;
public HllMergeAggregatorFactory(
@JsonProperty("name") final String name, public class HllMergeAggregatorFactory extends HllAggregatorFactory{
@JsonProperty("fieldName") final String fieldName, public HllMergeAggregatorFactory(
@JsonProperty("precision") @Nullable final Integer precision, @JsonProperty("name") final String name,
@JsonProperty("round") @Nullable final Boolean round @JsonProperty("fieldName") final String fieldName,
) { @JsonProperty("precision") @Nullable final Integer precision,
super(name, fieldName, precision, round); @JsonProperty("round") @Nullable final Boolean round
} ) {
super(name, fieldName, precision, round);
@Override }
public String getTypeName(){
return HllModule.HLLD_TYPE_NAME; /*
} 没这个方法了, 新版本需要实现getIntermediateType方法
@Override
@Override public String getTypeName(){
public Aggregator factorize(ColumnSelectorFactory metricFactory) { return HllModule.HLLD_TYPE_NAME;
final ColumnValueSelector<Hll> selector = metricFactory.makeColumnValueSelector(getFieldName()); }*/
return new HllMergeAggregator(
selector, @Override
precision public ColumnType getIntermediateType() {
); return HllModule.TYPE;
} }
@Override @Override
public BufferAggregator factorizeBuffered(ColumnSelectorFactory columnSelectorFactory) { public Aggregator factorize(ColumnSelectorFactory metricFactory) {
final ColumnValueSelector<Hll> selector = columnSelectorFactory.makeColumnValueSelector(getFieldName()); final ColumnValueSelector<Hll> selector = metricFactory.makeColumnValueSelector(getFieldName());
return new HllMergeBufferAggregator( return new HllMergeAggregator(
selector, selector,
precision precision
); );
} }
@Override @Override
public byte[] getCacheKey() { public BufferAggregator factorizeBuffered(ColumnSelectorFactory columnSelectorFactory) {
return new CacheKeyBuilder(HllModule.CACHE_TYPE_ID_OFFSET).appendByte(HllModule.HLLD_MERGE_CACHE_TYPE_ID) final ColumnValueSelector<Hll> selector = columnSelectorFactory.makeColumnValueSelector(getFieldName());
.appendString(name).appendString(fieldName) return new HllMergeBufferAggregator(
.appendInt(precision).appendBoolean(round) selector,
.build(); precision
} );
}
@Override
public int getMaxIntermediateSize() { @Override
return HllUnion.getUpdatableSerializationBytes(precision); public AggregatorFactory withName(String newName) {
} return new HllMergeAggregatorFactory(newName, fieldName, precision, round);
} }
@Override
public byte[] getCacheKey() {
return new CacheKeyBuilder(HllModule.CACHE_TYPE_ID_OFFSET).appendByte(HllModule.HLLD_MERGE_CACHE_TYPE_ID)
.appendString(name).appendString(fieldName)
.appendInt(precision).appendBoolean(round)
.build();
}
@Override
protected int getUpdatableSerializationBytes() {
return HllUnion.getUpdatableSerializationBytes(precision);
}
}

4
druid-hlld/src/main/java/org/apache/druid/query/aggregation/sketch/hlld/HllModule.java
View File

@@ -10,6 +10,7 @@ import org.apache.druid.initialization.DruidModule;
import org.apache.druid.query.aggregation.sketch.hlld.sql.HllApproxCountDistinctSqlAggregator; import org.apache.druid.query.aggregation.sketch.hlld.sql.HllApproxCountDistinctSqlAggregator;
import org.apache.druid.query.aggregation.sketch.hlld.sql.HllEstimateOperatorConversion; import org.apache.druid.query.aggregation.sketch.hlld.sql.HllEstimateOperatorConversion;
import org.apache.druid.query.aggregation.sketch.hlld.sql.HllObjectSqlAggregator; import org.apache.druid.query.aggregation.sketch.hlld.sql.HllObjectSqlAggregator;
import org.apache.druid.segment.column.ColumnType;
import org.apache.druid.segment.serde.ComplexMetrics; import org.apache.druid.segment.serde.ComplexMetrics;
import org.apache.druid.sql.guice.SqlBindings; import org.apache.druid.sql.guice.SqlBindings;
@@ -24,6 +25,9 @@ public class HllModule implements DruidModule {
public static final String HLLD_TYPE_NAME = "HLLDSketch"; public static final String HLLD_TYPE_NAME = "HLLDSketch";
public static final String HLLD_BUILD_TYPE_NAME = "HLLDSketchBuild"; public static final String HLLD_BUILD_TYPE_NAME = "HLLDSketchBuild";
public static final ColumnType TYPE = ColumnType.ofComplex(HLLD_TYPE_NAME);
public static final ColumnType BUILD_TYPE = ColumnType.ofComplex(HLLD_BUILD_TYPE_NAME);
@Override @Override
public void configure(Binder binder) { public void configure(Binder binder) {

217
druid-hlld/src/main/java/org/apache/druid/query/aggregation/sketch/hlld/HllToEstimatePostAggregator.java
View File

@@ -1,103 +1,114 @@
package org.apache.druid.query.aggregation.sketch.hlld; package org.apache.druid.query.aggregation.sketch.hlld;
import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonProperty;
import com.zdjz.galaxy.sketch.hlld.Hll; import com.zdjz.galaxy.sketch.hlld.Hll;
import org.apache.druid.query.aggregation.AggregatorFactory; import org.apache.druid.query.aggregation.AggregatorFactory;
import org.apache.druid.query.aggregation.PostAggregator; import org.apache.druid.query.aggregation.PostAggregator;
import org.apache.druid.query.aggregation.post.ArithmeticPostAggregator; import org.apache.druid.query.aggregation.post.ArithmeticPostAggregator;
import org.apache.druid.query.cache.CacheKeyBuilder; import org.apache.druid.query.cache.CacheKeyBuilder;
import org.apache.druid.segment.ColumnInspector;
import java.util.Comparator; import org.apache.druid.segment.column.ColumnType;
import java.util.Map;
import java.util.Objects; import java.util.Comparator;
import java.util.Set; import java.util.Map;
import java.util.Objects;
public class HllToEstimatePostAggregator implements PostAggregator { import java.util.Set;
private final String name;
private final PostAggregator field; public class HllToEstimatePostAggregator implements PostAggregator {
private final boolean round; private final String name;
private final PostAggregator field;
@JsonCreator private final boolean round;
public HllToEstimatePostAggregator(
@JsonProperty("name") final String name, @JsonCreator
@JsonProperty("field") final PostAggregator field, public HllToEstimatePostAggregator(
@JsonProperty("round") boolean round @JsonProperty("name") final String name,
) { @JsonProperty("field") final PostAggregator field,
this.name = name; @JsonProperty("round") boolean round
this.field = field; ) {
this.round = round; this.name = name;
} this.field = field;
this.round = round;
@Override }
@JsonProperty
public String getName() { // 新版本需要实现的方法
return name; @Override
} public ColumnType getType(ColumnInspector signature) {
return round ? ColumnType.LONG : ColumnType.DOUBLE;
@JsonProperty }
public PostAggregator getField() {
return field; @Override
} @JsonProperty
public String getName() {
@JsonProperty return name;
public boolean isRound() { }
return round;
} @JsonProperty
public PostAggregator getField() {
@Override return field;
public Set<String> getDependentFields() { }
return field.getDependentFields();
} @JsonProperty
public boolean isRound() {
@Override return round;
public Comparator<Double> getComparator() { }
return ArithmeticPostAggregator.DEFAULT_COMPARATOR;
} @Override
public Set<String> getDependentFields() {
@Override return field.getDependentFields();
public Object compute(final Map<String, Object> combinedAggregators) { }
final Hll sketch = (Hll) field.compute(combinedAggregators);
return round ? Math.round(sketch.size()) : sketch.size(); @Override
} public Comparator<Double> getComparator() {
return ArithmeticPostAggregator.DEFAULT_COMPARATOR;
@Override }
public PostAggregator decorate(final Map<String, AggregatorFactory> aggregators) {
return this; @Override
} public Object compute(final Map<String, Object> combinedAggregators) {
final Hll sketch = (Hll) field.compute(combinedAggregators);
@Override if(sketch == null){
public String toString() { return round ? 0L: 0D;
return "HllToEstimatePostAggregator{" + }
"name='" + name + '\'' + return round ? Math.round(sketch.size()) : sketch.size();
", field=" + field + }
", round=" + round +
'}'; @Override
} public PostAggregator decorate(final Map<String, AggregatorFactory> aggregators) {
return this;
@Override }
public boolean equals(final Object o) {
if (this == o) { @Override
return true; public String toString() {
} return "HllToEstimatePostAggregator{" +
if (!(o instanceof HllToEstimatePostAggregator)) { "name='" + name + '\'' +
return false; ", field=" + field +
} ", round=" + round +
'}';
final HllToEstimatePostAggregator that = (HllToEstimatePostAggregator) o; }
return name.equals(that.name) && field.equals(that.field) && round == that.round;
} @Override
public boolean equals(final Object o) {
@Override if (this == o) {
public int hashCode() { return true;
return Objects.hash(name, field, round); }
} if (!(o instanceof HllToEstimatePostAggregator)) {
return false;
@Override }
public byte[] getCacheKey() {
CacheKeyBuilder builder = new CacheKeyBuilder(HllModule.CACHE_TYPE_ID_OFFSET).appendByte(HllModule.HLLD_TO_ESTIMATE_CACHE_TYPE_ID) final HllToEstimatePostAggregator that = (HllToEstimatePostAggregator) o;
.appendCacheable(field).appendBoolean(round); return name.equals(that.name) && field.equals(that.field) && round == that.round;
return builder.build(); }
}
@Override
} public int hashCode() {
return Objects.hash(name, field, round);
}
@Override
public byte[] getCacheKey() {
CacheKeyBuilder builder = new CacheKeyBuilder(HllModule.CACHE_TYPE_ID_OFFSET).appendByte(HllModule.HLLD_TO_ESTIMATE_CACHE_TYPE_ID)
.appendCacheable(field).appendBoolean(round);
return builder.build();
}
}

20
druid-hlld/src/main/java/org/apache/druid/query/aggregation/sketch/hlld/sql/HllApproxCountDistinctSqlAggregator.java
View File

@@ -5,36 +5,44 @@ import org.apache.calcite.sql.SqlFunctionCategory;
import org.apache.calcite.sql.SqlKind; import org.apache.calcite.sql.SqlKind;
import org.apache.calcite.sql.type.*; import org.apache.calcite.sql.type.*;
import org.apache.druid.query.aggregation.AggregatorFactory; import org.apache.druid.query.aggregation.AggregatorFactory;
import org.apache.druid.query.aggregation.post.FieldAccessPostAggregator;
import org.apache.druid.query.aggregation.post.FinalizingFieldAccessPostAggregator; import org.apache.druid.query.aggregation.post.FinalizingFieldAccessPostAggregator;
import org.apache.druid.segment.VirtualColumn; import org.apache.druid.query.aggregation.sketch.hlld.HllAggregatorFactory;
import org.apache.druid.query.aggregation.sketch.hlld.HllToEstimatePostAggregator;
import org.apache.druid.sql.calcite.aggregation.Aggregation; import org.apache.druid.sql.calcite.aggregation.Aggregation;
import java.util.Collections; import java.util.Collections;
import java.util.List;
public class HllApproxCountDistinctSqlAggregator extends HllBaseSqlAggregator { public class HllApproxCountDistinctSqlAggregator extends HllBaseSqlAggregator {
private static final SqlAggFunction FUNCTION_INSTANCE = new CPCSketchApproxCountDistinctSqlAggFunction(); private static final SqlAggFunction FUNCTION_INSTANCE = new CPCSketchApproxCountDistinctSqlAggFunction();
private static final String NAME = "APPROX_COUNT_DISTINCT_HLLD"; private static final String NAME = "APPROX_COUNT_DISTINCT_HLLD";
public HllApproxCountDistinctSqlAggregator(){
super(true);
}
@Override @Override
public SqlAggFunction calciteFunction() { public SqlAggFunction calciteFunction() {
return FUNCTION_INSTANCE; return FUNCTION_INSTANCE;
} }
// 新版本参数少了virtualColumns
@Override @Override
protected Aggregation toAggregation( protected Aggregation toAggregation(
String name, String name,
boolean finalizeAggregations, boolean finalizeAggregations,
List<VirtualColumn> virtualColumns,
AggregatorFactory aggregatorFactory AggregatorFactory aggregatorFactory
) { ) {
return Aggregation.create( return Aggregation.create(
virtualColumns,
Collections.singletonList(aggregatorFactory), Collections.singletonList(aggregatorFactory),
//感觉是否是最外层的函数吧 //感觉是否是最外层的函数吧
finalizeAggregations ? new FinalizingFieldAccessPostAggregator( finalizeAggregations ? new HllToEstimatePostAggregator(
name, name,
aggregatorFactory.getName() new FieldAccessPostAggregator(
aggregatorFactory.getName(),
aggregatorFactory.getName()
),
((HllAggregatorFactory)aggregatorFactory).isRound()
) : null ) : null
); );
} }

58
druid-hlld/src/main/java/org/apache/druid/query/aggregation/sketch/hlld/sql/HllBaseSqlAggregator.java
View File

@@ -2,6 +2,7 @@ package org.apache.druid.query.aggregation.sketch.hlld.sql;
import org.apache.calcite.rel.core.AggregateCall; import org.apache.calcite.rel.core.AggregateCall;
import org.apache.calcite.rel.core.Project; import org.apache.calcite.rel.core.Project;
import org.apache.calcite.rel.type.RelDataType;
import org.apache.calcite.rex.RexBuilder; import org.apache.calcite.rex.RexBuilder;
import org.apache.calcite.rex.RexLiteral; import org.apache.calcite.rex.RexLiteral;
import org.apache.calcite.rex.RexNode; import org.apache.calcite.rex.RexNode;
@@ -14,6 +15,7 @@ import org.apache.druid.query.aggregation.sketch.hlld.HllMergeAggregatorFactory;
import org.apache.druid.query.dimension.DefaultDimensionSpec; import org.apache.druid.query.dimension.DefaultDimensionSpec;
import org.apache.druid.query.dimension.DimensionSpec; import org.apache.druid.query.dimension.DimensionSpec;
import org.apache.druid.segment.VirtualColumn; import org.apache.druid.segment.VirtualColumn;
import org.apache.druid.segment.column.ColumnType;
import org.apache.druid.segment.column.RowSignature; import org.apache.druid.segment.column.RowSignature;
import org.apache.druid.segment.column.ValueType; import org.apache.druid.segment.column.ValueType;
import org.apache.druid.sql.calcite.aggregation.Aggregation; import org.apache.druid.sql.calcite.aggregation.Aggregation;
@@ -29,6 +31,13 @@ import java.util.ArrayList;
import java.util.List; import java.util.List;
public abstract class HllBaseSqlAggregator implements SqlAggregator { public abstract class HllBaseSqlAggregator implements SqlAggregator {
private final boolean finalizeSketch;
protected HllBaseSqlAggregator(boolean finalizeSketch){
this.finalizeSketch = finalizeSketch;
}
@Nullable @Nullable
@Override @Override
public Aggregation toDruidAggregation( public Aggregation toDruidAggregation(
@@ -93,13 +102,14 @@ public abstract class HllBaseSqlAggregator implements SqlAggregator {
round = HllAggregatorFactory.DEFAULT_ROUND; round = HllAggregatorFactory.DEFAULT_ROUND;
} }
final List<VirtualColumn> virtualColumns = new ArrayList<>(); // 新版本删除了final List<VirtualColumn> virtualColumns = new ArrayList<>();
final AggregatorFactory aggregatorFactory; final AggregatorFactory aggregatorFactory;
final String aggregatorName = finalizeAggregations ? Calcites.makePrefixedName(name, "a") : name; //final String aggregatorName = finalizeAggregations ? Calcites.makePrefixedName(name, "a") : name;
final String aggregatorName = finalizeSketch ? Calcites.makePrefixedName(name, "a") : name;
// 输入是Cpc，返回HllMergeAggregatorFactory // 输入是Hll，返回HllSketchMergeAggregatorFactory
if (columnArg.isDirectColumnAccess() if (columnArg.isDirectColumnAccess()
&& rowSignature.getColumnType(columnArg.getDirectColumn()).orElse(null) == ValueType.COMPLEX) { && rowSignature.getColumnType(columnArg.getDirectColumn()).map(type -> type.is(ValueType.COMPLEX)).orElse(false)) {
// 这就是具体的聚合函数吧 // 这就是具体的聚合函数吧
aggregatorFactory = new HllMergeAggregatorFactory( aggregatorFactory = new HllMergeAggregatorFactory(
aggregatorName, aggregatorName,
@@ -109,10 +119,10 @@ public abstract class HllBaseSqlAggregator implements SqlAggregator {
); );
} else { } else {
// 输入是regular column，HllBuildAggregatorFactory // 输入是regular column，HllBuildAggregatorFactory
final SqlTypeName sqlTypeName = columnRexNode.getType().getSqlTypeName(); final RelDataType dataType = columnRexNode.getType();
final ValueType inputType = Calcites.getValueTypeForSqlTypeName(sqlTypeName); final ColumnType inputType = Calcites.getColumnTypeForRelDataType(dataType);
if (inputType == null) { if (inputType == null) {
throw new ISE("Cannot translate sqlTypeName[%s] to Druid type for field[%s]", sqlTypeName, aggregatorName); throw new ISE("Cannot translate sqlTypeName[%s] to Druid type for field[%s]", dataType.getSqlTypeName(), aggregatorName);
} }
final DimensionSpec dimensionSpec; final DimensionSpec dimensionSpec;
@@ -120,27 +130,34 @@ public abstract class HllBaseSqlAggregator implements SqlAggregator {
if (columnArg.isDirectColumnAccess()) { if (columnArg.isDirectColumnAccess()) {
dimensionSpec = columnArg.getSimpleExtraction().toDimensionSpec(null, inputType); dimensionSpec = columnArg.getSimpleExtraction().toDimensionSpec(null, inputType);
} else { } else {
VirtualColumn virtualColumn = virtualColumnRegistry.getOrCreateVirtualColumnForExpression( String virtualColumnName = virtualColumnRegistry.getOrCreateVirtualColumnForExpression(
plannerContext,
columnArg, columnArg,
sqlTypeName dataType
); );
dimensionSpec = new DefaultDimensionSpec(virtualColumn.getOutputName(), null, inputType); dimensionSpec = new DefaultDimensionSpec(virtualColumnName, null, inputType);
virtualColumns.add(virtualColumn);
} }
aggregatorFactory = new HllAggregatorFactory( // 新版本的判断，输入是Hll
aggregatorName, if (inputType.is(ValueType.COMPLEX)) {
dimensionSpec.getDimension(), aggregatorFactory = new HllMergeAggregatorFactory(
precision, aggregatorName,
round dimensionSpec.getOutputName(),
); precision,
round
);
} else {
aggregatorFactory = new HllAggregatorFactory(
aggregatorName,
dimensionSpec.getDimension(),
precision,
round
);
}
} }
return toAggregation( return toAggregation(
name, name,
finalizeAggregations, finalizeSketch,
virtualColumns,
aggregatorFactory aggregatorFactory
); );
} }
@@ -148,7 +165,6 @@ public abstract class HllBaseSqlAggregator implements SqlAggregator {
protected abstract Aggregation toAggregation( protected abstract Aggregation toAggregation(
String name, String name,
boolean finalizeAggregations, boolean finalizeAggregations,
List<VirtualColumn> virtualColumns,
AggregatorFactory aggregatorFactory AggregatorFactory aggregatorFactory
); );
} }

16
druid-hlld/src/main/java/org/apache/druid/query/aggregation/sketch/hlld/sql/HllEstimateOperatorConversion.java
View File

@@ -13,16 +13,15 @@ import org.apache.druid.query.aggregation.PostAggregator;
import org.apache.druid.query.aggregation.sketch.hlld.HllAggregatorFactory; import org.apache.druid.query.aggregation.sketch.hlld.HllAggregatorFactory;
import org.apache.druid.query.aggregation.sketch.hlld.HllToEstimatePostAggregator; import org.apache.druid.query.aggregation.sketch.hlld.HllToEstimatePostAggregator;
import org.apache.druid.segment.column.RowSignature; import org.apache.druid.segment.column.RowSignature;
import org.apache.druid.sql.calcite.expression.DirectOperatorConversion; import org.apache.druid.sql.calcite.expression.*;
import org.apache.druid.sql.calcite.expression.DruidExpression;
import org.apache.druid.sql.calcite.expression.OperatorConversions;
import org.apache.druid.sql.calcite.expression.PostAggregatorVisitor;
import org.apache.druid.sql.calcite.planner.PlannerContext; import org.apache.druid.sql.calcite.planner.PlannerContext;
import javax.annotation.Nullable; import javax.annotation.Nullable;
import java.util.List; import java.util.List;
public class HllEstimateOperatorConversion extends DirectOperatorConversion { // postAggregator, toDruidExpression返回null。相当于post udf和普通udf是不一样的。
// 新版本直接修改了父类
public class HllEstimateOperatorConversion implements SqlOperatorConversion {
private static final String FUNCTION_NAME = "HLLD_ESTIMATE"; private static final String FUNCTION_NAME = "HLLD_ESTIMATE";
private static final SqlFunction SQL_FUNCTION = OperatorConversions private static final SqlFunction SQL_FUNCTION = OperatorConversions
.operatorBuilder(StringUtils.toUpperCase(FUNCTION_NAME)) .operatorBuilder(StringUtils.toUpperCase(FUNCTION_NAME))
@@ -32,9 +31,7 @@ public class HllEstimateOperatorConversion extends DirectOperatorConversion {
.returnTypeInference(ReturnTypes.DOUBLE) .returnTypeInference(ReturnTypes.DOUBLE)
.build(); .build();
public HllEstimateOperatorConversion() { // 新版本少了构造函数
super(SQL_FUNCTION, FUNCTION_NAME);
}
@Override @Override
public SqlOperator calciteOperator() { public SqlOperator calciteOperator() {
@@ -63,7 +60,8 @@ public class HllEstimateOperatorConversion extends DirectOperatorConversion {
plannerContext, plannerContext,
rowSignature, rowSignature,
operands.get(0), operands.get(0),
postAggregatorVisitor postAggregatorVisitor,
true // 新版本多了个参数
); );
if (firstOperand == null) { if (firstOperand == null) {

8
druid-hlld/src/main/java/org/apache/druid/query/aggregation/sketch/hlld/sql/HllObjectSqlAggregator.java
View File

@@ -5,16 +5,18 @@ import org.apache.calcite.sql.SqlFunctionCategory;
import org.apache.calcite.sql.SqlKind; import org.apache.calcite.sql.SqlKind;
import org.apache.calcite.sql.type.*; import org.apache.calcite.sql.type.*;
import org.apache.druid.query.aggregation.AggregatorFactory; import org.apache.druid.query.aggregation.AggregatorFactory;
import org.apache.druid.segment.VirtualColumn;
import org.apache.druid.sql.calcite.aggregation.Aggregation; import org.apache.druid.sql.calcite.aggregation.Aggregation;
import java.util.Collections; import java.util.Collections;
import java.util.List;
public class HllObjectSqlAggregator extends HllBaseSqlAggregator { public class HllObjectSqlAggregator extends HllBaseSqlAggregator {
private static final SqlAggFunction FUNCTION_INSTANCE = new CpcSketchSqlAggFunction(); private static final SqlAggFunction FUNCTION_INSTANCE = new CpcSketchSqlAggFunction();
private static final String NAME = "HLLD"; private static final String NAME = "HLLD";
public HllObjectSqlAggregator(){
super(false);
}
@Override @Override
public SqlAggFunction calciteFunction() { public SqlAggFunction calciteFunction() {
return FUNCTION_INSTANCE; return FUNCTION_INSTANCE;
@@ -24,11 +26,9 @@ public class HllObjectSqlAggregator extends HllBaseSqlAggregator {
protected Aggregation toAggregation( protected Aggregation toAggregation(
String name, String name,
boolean finalizeAggregations, boolean finalizeAggregations,
List<VirtualColumn> virtualColumns,
AggregatorFactory aggregatorFactory AggregatorFactory aggregatorFactory
) { ) {
return Aggregation.create( return Aggregation.create(
virtualColumns,
Collections.singletonList(aggregatorFactory), Collections.singletonList(aggregatorFactory),
null null
); );

740
druid-hlld/src/test/java/org/apache/druid/query/aggregation/sketch/hlld/sql/HllApproxCountDistinctSqlAggregatorTest.java
View File

@@ -1,311 +1,429 @@
package org.apache.druid.query.aggregation.sketch.hlld.sql; package org.apache.druid.query.aggregation.sketch.hlld.sql;
import com.fasterxml.jackson.databind.Module; import com.alibaba.fastjson2.JSON;
import com.google.common.collect.ImmutableMap; import com.fasterxml.jackson.databind.Module;
import com.google.common.collect.ImmutableSet; import com.google.inject.Injector;
import org.apache.calcite.schema.SchemaPlus; import org.apache.druid.guice.DruidInjectorBuilder;
import org.apache.druid.java.util.common.io.Closer; import org.apache.druid.query.QueryRunnerFactoryConglomerate;
import org.apache.druid.query.QueryRunnerFactoryConglomerate; import org.apache.druid.query.aggregation.sketch.hlld.HllModule;
import org.apache.druid.query.aggregation.CountAggregatorFactory; import org.apache.druid.segment.QueryableIndex;
import org.apache.druid.query.aggregation.DoubleSumAggregatorFactory; import org.apache.druid.segment.TestHelper;
import org.apache.druid.query.aggregation.sketch.hlld.HllAggregatorFactory; import org.apache.druid.segment.join.JoinableFactoryWrapper;
import org.apache.druid.query.aggregation.sketch.hlld.HllModule; import org.apache.druid.sql.calcite.BaseCalciteQueryTest;
import org.apache.druid.segment.IndexBuilder; import org.apache.druid.sql.calcite.QueryTestBuilder;
import org.apache.druid.segment.QueryableIndex; import org.apache.druid.sql.calcite.QueryTestRunner;
import org.apache.druid.segment.TestHelper; import org.apache.druid.sql.calcite.util.CalciteTests;
import org.apache.druid.segment.incremental.IncrementalIndexSchema; import org.apache.druid.sql.calcite.util.SpecificSegmentsQuerySegmentWalker;
import org.apache.druid.segment.writeout.OffHeapMemorySegmentWriteOutMediumFactory; import org.apache.druid.timeline.DataSegment;
import org.apache.druid.server.QueryStackTests; import org.apache.druid.timeline.partition.LinearShardSpec;
import org.apache.druid.server.security.AuthTestUtils; import org.junit.*;
import org.apache.druid.server.security.AuthenticationResult;
import org.apache.druid.sql.SqlLifecycle; import java.io.File;
import org.apache.druid.sql.SqlLifecycleFactory; import java.io.IOException;
import org.apache.druid.sql.calcite.planner.DruidOperatorTable; import java.util.*;
import org.apache.druid.sql.calcite.planner.PlannerConfig;
import org.apache.druid.sql.calcite.planner.PlannerContext; // 新版本父类直接变了，实现更简单了
import org.apache.druid.sql.calcite.planner.PlannerFactory; public class HllApproxCountDistinctSqlAggregatorTest extends BaseCalciteQueryTest {
import org.apache.druid.sql.calcite.util.CalciteTestBase; private static final boolean ROUND = true;
import org.apache.druid.sql.calcite.util.CalciteTests;
import org.apache.druid.sql.calcite.util.QueryLogHook; @Override
import org.apache.druid.sql.calcite.util.SpecificSegmentsQuerySegmentWalker; public void gatherProperties(Properties properties)
import org.apache.druid.timeline.DataSegment; {
import org.apache.druid.timeline.partition.LinearShardSpec; super.gatherProperties(properties);
import org.junit.*; }
import org.junit.rules.TemporaryFolder;
@Override
import java.io.IOException; public void configureGuice(DruidInjectorBuilder builder)
import java.util.Arrays; {
import java.util.List; super.configureGuice(builder);
import java.util.Map; builder.addModule(new HllModule());
}
public class HllApproxCountDistinctSqlAggregatorTest extends CalciteTestBase {
private static final String DATA_SOURCE = "foo";
private static final boolean ROUND = true;
private static final Map<String, Object> QUERY_CONTEXT_DEFAULT = ImmutableMap.of( @SuppressWarnings("resource")
PlannerContext.CTX_SQL_QUERY_ID, "dummy" @Override
); public SpecificSegmentsQuerySegmentWalker createQuerySegmentWalker(
private static QueryRunnerFactoryConglomerate conglomerate; final QueryRunnerFactoryConglomerate conglomerate,
private static Closer resourceCloser; final JoinableFactoryWrapper joinableFactory,
private static AuthenticationResult authenticationResult = CalciteTests.REGULAR_USER_AUTH_RESULT; final Injector injector
) throws IOException
@Rule {
public TemporaryFolder temporaryFolder = new TemporaryFolder(); HllModule.registerSerde();
for (Module mod : new HllModule().getJacksonModules()) {
@Rule CalciteTests.getJsonMapper().registerModule(mod);
public QueryLogHook queryLogHook = QueryLogHook.create(TestHelper.JSON_MAPPER); TestHelper.JSON_MAPPER.registerModule(mod);
}
private SpecificSegmentsQuerySegmentWalker walker;
private SqlLifecycleFactory sqlLifecycleFactory; final QueryableIndex index = TestHelper.getTestIndexIO().loadIndex(new File("D:/doc/datas/testIndex-1369101812"));
//final QueryableIndex index = TestHelper.getTestIndexIO().loadIndex(new File("D:/doc/datas/9_index"));
@BeforeClass /*final QueryableIndex index = IndexBuilder.create()
public static void setUpClass() { .tmpDir(temporaryFolder.newFolder())
resourceCloser = Closer.create(); .segmentWriteOutMediumFactory(OffHeapMemorySegmentWriteOutMediumFactory.instance())
conglomerate = QueryStackTests.createQueryRunnerFactoryConglomerate(resourceCloser); .schema(
} new IncrementalIndexSchema.Builder()
.withMetrics(
@AfterClass new CountAggregatorFactory("cnt"),
public static void tearDownClass() throws IOException { new DoubleSumAggregatorFactory("m1", "m1"),
resourceCloser.close(); new HllAggregatorFactory(
} "hll_dim1",
"dim1",
@Before null,
public void setUp() throws Exception { ROUND
HllModule.registerSerde(); )
for (Module mod : new HllModule().getJacksonModules()) { )
CalciteTests.getJsonMapper().registerModule(mod); .withRollup(false)
TestHelper.JSON_MAPPER.registerModule(mod); .build()
} )
.rows(TestDataBuilder.ROWS1)
final QueryableIndex index = IndexBuilder.create() .buildMMappedIndex();*/
.tmpDir(temporaryFolder.newFolder())
.segmentWriteOutMediumFactory(OffHeapMemorySegmentWriteOutMediumFactory.instance()) return new SpecificSegmentsQuerySegmentWalker(conglomerate).add(
.schema( DataSegment.builder()
new IncrementalIndexSchema.Builder() .dataSource(CalciteTests.DATASOURCE1)
.withMetrics( .interval(index.getDataInterval())
new CountAggregatorFactory("cnt"), .version("1")
new DoubleSumAggregatorFactory("m1", "m1"), .shardSpec(new LinearShardSpec(0))
new HllAggregatorFactory( .size(0)
"hll_dim1", .build(),
"dim1", index
null, );
ROUND }
)
) @Test
.withRollup(false) public void testSqlQuery() throws Exception {
.build() // Can't vectorize due to SUBSTRING expression.
) cannotVectorize();
.rows(CalciteTests.ROWS1)
.buildMMappedIndex(); String[] columns = new String[]{"__time", "dim1", "dim2", "dim3", "cnt", "hll_dim1", "m1"};
walker = new SpecificSegmentsQuerySegmentWalker(conglomerate).add( String sql = "select " + String.join(",", columns) + " from druid.foo";
DataSegment.builder() QueryTestBuilder builder = testBuilder().sql(sql);
.dataSource(DATA_SOURCE) builder.run();
.interval(index.getDataInterval()) QueryTestRunner.QueryResults queryResults = builder.results();
.version("1") List<Object[]> results = queryResults.results;
.shardSpec(new LinearShardSpec(0)) for (Object[] result : results) {
.size(0) Map row = new LinkedHashMap();
.build(), for (int i = 0; i < result.length; i++) {
index row.put(columns[i], result[i]);
); }
System.out.println(JSON.toJSONString(row));
final PlannerConfig plannerConfig = new PlannerConfig(); // System.out.println(Arrays.toString(result));
final DruidOperatorTable operatorTable = new DruidOperatorTable( }
ImmutableSet.of(
new HllApproxCountDistinctSqlAggregator(), for (int i = 0; i < columns.length; i++) {
new HllObjectSqlAggregator() Object[] values = new Object[results.size()];
), for (int j = 0; j < results.size(); j++) {
ImmutableSet.of( values[j] = results.get(j)[i];
new HllEstimateOperatorConversion() }
) System.out.println(columns[i] + ":" + Arrays.toString(values));
); }
}
SchemaPlus rootSchema = CalciteTests.createMockRootSchema(conglomerate, walker, plannerConfig, AuthTestUtils.TEST_AUTHORIZER_MAPPER);
sqlLifecycleFactory = CalciteTests.createSqlLifecycleFactory( @Test
new PlannerFactory( public void testSqlQuery11() throws Exception {
rootSchema, // Can't vectorize due to SUBSTRING expression.
CalciteTests.createMockQueryLifecycleFactory(walker, conglomerate), //cannotVectorize();
operatorTable,
CalciteTests.createExprMacroTable(),
plannerConfig, String sql = "select HLLD(hll_dim1) hll_dim1 from (select hll_dim1 from druid.foo limit 5) t ";
AuthTestUtils.TEST_AUTHORIZER_MAPPER, //sql = "select HLLD(hll_dim1) hll_dim1 from druid.foo t ";
CalciteTests.getJsonMapper(), QueryTestBuilder builder = testBuilder().sql(sql).skipVectorize();;
CalciteTests.DRUID_SCHEMA_NAME builder.run();
) QueryTestRunner.QueryResults queryResults = builder.results();
); List<Object[]> results = queryResults.results;
} for (Object[] result : results) {
System.out.println(Arrays.toString(result));
@After }
public void tearDown() throws Exception { }
walker.close();
walker = null; @Test
} public void testSqlQuery12() throws Exception {
// Can't vectorize due to SUBSTRING expression.
@Test cannotVectorize();
public void testSqlQuery() throws Exception {
SqlLifecycle sqlLifecycle = sqlLifecycleFactory.factorize(); String sql = "select * from (select * from druid.foo limit 6) t where __time >= '1970-12-15 07:00:28' and __time < '2023-12-15 08:10:28' ";
String sql = "select * from druid.foo"; QueryTestBuilder builder = testBuilder().sql(sql);
final List<Object[]> results = builder.run();
sqlLifecycle.runSimple(sql, QUERY_CONTEXT_DEFAULT, DEFAULT_PARAMETERS, authenticationResult).toList(); QueryTestRunner.QueryResults queryResults = builder.results();
for (Object[] result : results) { List<Object[]> results = queryResults.results;
System.out.println(Arrays.toString(result)); for (Object[] result : results) {
} System.out.println(Arrays.toString(result));
} }
}
@Test
public void testSqlQuery2() throws Exception { @Test
SqlLifecycle sqlLifecycle = sqlLifecycleFactory.factorize(); public void testSqlQuery1() throws Exception {
String sql = "select HLLD_ESTIMATE(HLLD(hll_dim1)) from druid.foo where dim1 = ''"; // Can't vectorize due to SUBSTRING expression.
final List<Object[]> results = cannotVectorize();
sqlLifecycle.runSimple(sql, QUERY_CONTEXT_DEFAULT, DEFAULT_PARAMETERS, authenticationResult).toList();
for (Object[] result : results) { String sql = "select dim1 from druid.foo";
System.out.println(Arrays.toString(result)); QueryTestBuilder builder = testBuilder().sql(sql);
} builder.run();
} QueryTestRunner.QueryResults queryResults = builder.results();
List<Object[]> results = queryResults.results;
@Test for (Object[] result : results) {
public void testAgg() throws Exception { System.out.println(Arrays.toString(result));
SqlLifecycle sqlLifecycle = sqlLifecycleFactory.factorize(); }
}
final String sql = "SELECT\n"
+ " SUM(cnt),\n" @Test
+ " APPROX_COUNT_DISTINCT_HLLD(hll_dim1)\n" public void testSqlQuery2() throws Exception {
+ "FROM druid.foo"; //cannotVectorize();
//String sql = "select HLLD_ESTIMATE(HLLD(hll_dim1)) from druid.foo where dim1 = '1'";
final List<Object[]> results = // Caused by: org.apache.calcite.sql.validate.SqlValidatorException: Aggregate expressions cannot be nested
sqlLifecycle.runSimple(sql, QUERY_CONTEXT_DEFAULT, DEFAULT_PARAMETERS, authenticationResult).toList(); //String sql = "select HLLD_ESTIMATE(HLLD(hll_dim1)), APPROX_COUNT_DISTINCT_HLLD(HLLD(hll_dim1)), HLLD(hll_dim1) from druid.foo";
for (Object[] result : results) { String sql = "select HLLD_ESTIMATE(HLLD(hll_dim1)), APPROX_COUNT_DISTINCT_HLLD(hll_dim1), HLLD(hll_dim1) from (select HLLD(hll_dim1) hll_dim1 from druid.foo) t";
System.out.println(Arrays.toString(result)); QueryTestBuilder builder = testBuilder().sql(sql).skipVectorize();
} builder.run();
QueryTestRunner.QueryResults queryResults = builder.results();
} List<Object[]> results = queryResults.results;
for (Object[] result : results) {
System.out.println(Arrays.toString(result));
@Test }
public void testDistinct() throws Exception { }
SqlLifecycle sqlLifecycle = sqlLifecycleFactory.factorize();
@Test
final String sql = "SELECT\n" public void testSqlQuery3() throws Exception {
+ " SUM(cnt),\n" //cannotVectorize();
+ " APPROX_COUNT_DISTINCT_HLLD(dim2),\n" // uppercase //String sql = "select HLLD_ESTIMATE(HLLD(hll_dim1)) from druid.foo where dim1 = ''";
+ " APPROX_COUNT_DISTINCT_HLLD(dim2) FILTER(WHERE dim2 <> ''),\n" // lowercase; also, filtered String sql = "select APPROX_COUNT_DISTINCT_HLLD(hll, 12) from (select HLLD(hll_dim1) hll from druid.foo where dim1 = '1') t ";
+ " APPROX_COUNT_DISTINCT_HLLD(SUBSTRING(dim2, 1, 1)),\n" // on extractionFn QueryTestBuilder builder = testBuilder().sql(sql).skipVectorize();
+ " APPROX_COUNT_DISTINCT_HLLD(SUBSTRING(dim2, 1, 1) || 'x'),\n" // on expression builder.run();
+ " APPROX_COUNT_DISTINCT_HLLD(hll_dim1, 16),\n" // on native HllSketch column QueryTestRunner.QueryResults queryResults = builder.results();
+ " APPROX_COUNT_DISTINCT_HLLD(hll_dim1)\n" // on native HllSketch column List<Object[]> results = queryResults.results;
+ "FROM druid.foo"; for (Object[] result : results) {
System.out.println(Arrays.toString(result));
final List<Object[]> results = }
sqlLifecycle.runSimple(sql, QUERY_CONTEXT_DEFAULT, DEFAULT_PARAMETERS, authenticationResult).toList(); }
for (Object[] result : results) {
System.out.println(Arrays.toString(result)); @Test
} public void testSqlQuery4() throws Exception {
//cannotVectorize();
} //String sql = "select HLLD_ESTIMATE(HLLD(hll_dim1)) from druid.foo where dim1 = ''";
String sql = "select APPROX_COUNT_DISTINCT_HLLD(hll, 12) from (select HLLD(hll_dim1) hll from druid.foo where dim1 = '1') t ";
@Test QueryTestBuilder builder = testBuilder().sql(sql).skipVectorize();
public void testDistinct2() throws Exception { builder.run();
SqlLifecycle sqlLifecycle = sqlLifecycleFactory.factorize(); QueryTestRunner.QueryResults queryResults = builder.results();
List<Object[]> results = queryResults.results;
final String sql = "SELECT\n" for (Object[] result : results) {
+ " SUM(cnt),\n" System.out.println(Arrays.toString(result));
+ " APPROX_COUNT_DISTINCT_HLLD(dim2),\n" }
+ " HLLD(dim2),\n" }
+ " HLLD(hll_dim1),\n"
+ " HLLD_ESTIMATE(HLLD(dim2)),\n" @Test
+ " HLLD_ESTIMATE(HLLD(dim2), true),\n" public void testSqlQuery5() throws Exception {
+ " HLLD_ESTIMATE(HLLD(dim1), true),\n" //cannotVectorize();
+ " HLLD_ESTIMATE(HLLD(hll_dim1)),\n" // on native HllSketch column //String sql = "select HLLD_ESTIMATE(HLLD(hll_dim1)) from druid.foo where dim1 = ''";
+ " APPROX_COUNT_DISTINCT_HLLD(hll_dim1)\n" // on native HllSketch column String sql = "select dim1,APPROX_COUNT_DISTINCT_HLLD(hll, 12) from (select dim1,HLLD(hll_dim1) hll from druid.foo where dim1 = '1' group by dim1) t group by dim1";
+ "FROM druid.foo"; QueryTestBuilder builder = testBuilder().sql(sql).skipVectorize();
builder.run();
final List<Object[]> results = QueryTestRunner.QueryResults queryResults = builder.results();
sqlLifecycle.runSimple(sql, QUERY_CONTEXT_DEFAULT, DEFAULT_PARAMETERS, authenticationResult).toList(); List<Object[]> results = queryResults.results;
for (Object[] result : results) { for (Object[] result : results) {
System.out.println(Arrays.toString(result)); System.out.println(Arrays.toString(result));
} }
}
}
@Test
@Test public void testSqlQuery6() throws Exception {
public void testDistinctDebug() throws Exception { //cannotVectorize();
SqlLifecycle sqlLifecycle = sqlLifecycleFactory.factorize(); //String sql = "select HLLD_ESTIMATE(HLLD(hll_dim1)) from druid.foo where dim1 = ''";
String sql = "select dim1,APPROX_COUNT_DISTINCT_HLLD(hll, 12) from (select dim1,HLLD(dim1) hll from druid.foo where dim1 = '1' group by dim1 limit 10) t group by dim1";
final String sql = "SELECT\n" //String sql = "select dim1,HLLD_ESTIMATE(HLLD(hll), false) from (select dim1,HLLD(dim1) hll from druid.foo where dim1 = '1' group by dim1 limit 10) t group by dim1";
+ " SUM(cnt),\n" QueryTestBuilder builder = testBuilder().sql(sql).skipVectorize();
+ " APPROX_COUNT_DISTINCT_HLLD(dim2)\n" builder.run();
+ "FROM druid.foo"; QueryTestRunner.QueryResults queryResults = builder.results();
List<Object[]> results = queryResults.results;
final List<Object[]> results = for (Object[] result : results) {
sqlLifecycle.runSimple(sql, QUERY_CONTEXT_DEFAULT, DEFAULT_PARAMETERS, authenticationResult).toList(); System.out.println(Arrays.toString(result));
for (Object[] result : results) { }
System.out.println(Arrays.toString(result)); }
}
@Test
} public void testSqlQuery62() throws Exception {
//cannotVectorize();
@Test //String sql = "select HLLD_ESTIMATE(HLLD(hll_dim1)) from druid.foo where dim1 = ''";
public void testDeser() throws Exception { String sql = "select dim1,APPROX_COUNT_DISTINCT_HLLD(hll) from (select dim1,HLLD(dim1) hll from druid.foo where dim1 = '1' group by dim1 limit 10) t group by dim1";
SqlLifecycle sqlLifecycle = sqlLifecycleFactory.factorize(); QueryTestBuilder builder = testBuilder().sql(sql).skipVectorize();
builder.run();
final String sql = "SELECT\n" QueryTestRunner.QueryResults queryResults = builder.results();
+ " APPROX_COUNT_DISTINCT_HLLD(hll_dim1) cnt\n" List<Object[]> results = queryResults.results;
+ "FROM druid.foo"; for (Object[] result : results) {
System.out.println(Arrays.toString(result));
final List<Object[]> results = }
sqlLifecycle.runSimple(sql, QUERY_CONTEXT_DEFAULT, DEFAULT_PARAMETERS, authenticationResult).toList(); }
for (Object[] result : results) {
System.out.println(Arrays.toString(result)); @Test
} public void testSqlQuery7() throws Exception {
//cannotVectorize();
} //String sql = "select HLLD_ESTIMATE(HLLD(hll_dim1)) from druid.foo where dim1 = ''";
String sql = "select dim1,APPROX_COUNT_DISTINCT_HLLD(hll, 12) from (select dim1,HLLD(dim1) hll from druid.foo where dim1 = '1' group by dim1) t group by dim1 limit 10";
QueryTestBuilder builder = testBuilder().sql(sql).skipVectorize();
@Test builder.run();
public void testGroupBy() throws Exception { QueryTestRunner.QueryResults queryResults = builder.results();
SqlLifecycle sqlLifecycle = sqlLifecycleFactory.factorize(); List<Object[]> results = queryResults.results;
for (Object[] result : results) {
final String sql = "SELECT cnt,\n" System.out.println(Arrays.toString(result));
+ " APPROX_COUNT_DISTINCT_HLLD(hll_dim1, 14) cnt2\n" }
+ "FROM druid.foo group by cnt"; }
final List<Object[]> results = @Test
sqlLifecycle.runSimple(sql, QUERY_CONTEXT_DEFAULT, DEFAULT_PARAMETERS, authenticationResult).toList(); public void testAgg() throws Exception {
for (Object[] result : results) { final String sql = "SELECT\n"
System.out.println(Arrays.toString(result)); + " SUM(cnt),\n"
} + " APPROX_COUNT_DISTINCT_HLLD(hll_dim1)\n"
+ "FROM druid.foo";
}
QueryTestBuilder builder = testBuilder().sql(sql).skipVectorize();
@Test builder.run();
public void testGroupBy1() throws Exception { QueryTestRunner.QueryResults queryResults = builder.results();
SqlLifecycle sqlLifecycle = sqlLifecycleFactory.factorize(); List<Object[]> results = queryResults.results;
for (Object[] result : results) {
final String sql = "SELECT __time,\n" System.out.println(Arrays.toString(result));
+ " APPROX_COUNT_DISTINCT_HLLD(hll_dim1, 14) cnt\n" }
+ "FROM druid.foo group by __time";
}
final List<Object[]> results =
sqlLifecycle.runSimple(sql, QUERY_CONTEXT_DEFAULT, DEFAULT_PARAMETERS, authenticationResult).toList(); @Test
for (Object[] result : results) { public void testDistinct() throws Exception {
System.out.println(Arrays.toString(result)); final String sql = "SELECT\n"
} + " SUM(cnt),\n"
+ " APPROX_COUNT_DISTINCT_HLLD(dim2),\n" // uppercase
} + " APPROX_COUNT_DISTINCT_HLLD(dim2) FILTER(WHERE dim2 <> ''),\n" // lowercase; also, filtered
+ " APPROX_COUNT_DISTINCT_HLLD(SUBSTRING(dim2, 1, 1)),\n" // on extractionFn
@Test + " APPROX_COUNT_DISTINCT_HLLD(SUBSTRING(dim2, 1, 1) || 'x'),\n" // on expression
public void testGroupBy2() throws Exception { + " APPROX_COUNT_DISTINCT_HLLD(hll_dim1, 16),\n" // on native HllSketch column
SqlLifecycle sqlLifecycle = sqlLifecycleFactory.factorize(); + " APPROX_COUNT_DISTINCT_HLLD(hll_dim1)\n" // on native HllSketch column
+ "FROM druid.foo";
final String sql = "SELECT __time,\n"
+ " APPROX_COUNT_DISTINCT_HLLD(hll_dim1, 14) cnt\n" QueryTestBuilder builder = testBuilder().sql(sql).skipVectorize();
+ "FROM druid.foo group by __time order by cnt desc"; builder.run();
QueryTestRunner.QueryResults queryResults = builder.results();
final List<Object[]> results = List<Object[]> results = queryResults.results;
sqlLifecycle.runSimple(sql, QUERY_CONTEXT_DEFAULT, DEFAULT_PARAMETERS, authenticationResult).toList(); for (Object[] result : results) {
for (Object[] result : results) { System.out.println(Arrays.toString(result));
System.out.println(Arrays.toString(result)); }
} }
} @Test
} public void testDistinct2() throws Exception {
final String sql = "SELECT\n"
+ " SUM(cnt),\n"
+ " APPROX_COUNT_DISTINCT_HLLD(dim2),\n"
+ " HLLD(dim2),\n"
+ " HLLD(hll_dim1),\n"
+ " HLLD_ESTIMATE(HLLD(dim2)),\n"
+ " HLLD_ESTIMATE(HLLD(dim2), true),\n"
+ " HLLD_ESTIMATE(HLLD(dim1), true),\n"
+ " HLLD_ESTIMATE(HLLD(hll_dim1)),\n" // on native HllSketch column
+ " APPROX_COUNT_DISTINCT_HLLD(hll_dim1)\n" // on native HllSketch column
+ "FROM druid.foo";
QueryTestBuilder builder = testBuilder().sql(sql).skipVectorize();
builder.run();
QueryTestRunner.QueryResults queryResults = builder.results();
List<Object[]> results = queryResults.results;
for (Object[] result : results) {
System.out.println(Arrays.toString(result));
}
}
@Test
public void testDistinctDebug2() throws Exception {
final String sql = "SELECT\n"
+ " dim1, dim2\n"
+ "FROM druid.foo";
QueryTestBuilder builder = testBuilder().sql(sql).skipVectorize();
builder.run();
QueryTestRunner.QueryResults queryResults = builder.results();
List<Object[]> results = queryResults.results;
for (Object[] result : results) {
System.out.println(Arrays.toString(result));
}
}
@Test
public void testDistinctDebug() throws Exception {
final String sql = "SELECT\n"
+ " SUM(cnt),\n"
+ " APPROX_COUNT_DISTINCT_HLLD(dim2)\n"
+ "FROM druid.foo";
QueryTestBuilder builder = testBuilder().sql(sql).skipVectorize();
builder.run();
QueryTestRunner.QueryResults queryResults = builder.results();
List<Object[]> results = queryResults.results;
for (Object[] result : results) {
System.out.println(Arrays.toString(result));
}
}
@Test
public void testDeser() throws Exception {
final String sql = "SELECT\n"
+ " APPROX_COUNT_DISTINCT_HLLD(hll_dim1) cnt\n"
+ "FROM druid.foo";
QueryTestBuilder builder = testBuilder().sql(sql).skipVectorize();
builder.run();
QueryTestRunner.QueryResults queryResults = builder.results();
List<Object[]> results = queryResults.results;
for (Object[] result : results) {
System.out.println(Arrays.toString(result));
}
}
@Test
public void testGroupBy() throws Exception {
final String sql = "SELECT cnt,\n"
+ " APPROX_COUNT_DISTINCT_HLLD(hll_dim1, 14) cnt2\n"
+ "FROM druid.foo group by cnt";
QueryTestBuilder builder = testBuilder().sql(sql).skipVectorize();
builder.run();
QueryTestRunner.QueryResults queryResults = builder.results();
List<Object[]> results = queryResults.results;
for (Object[] result : results) {
System.out.println(Arrays.toString(result));
}
}
@Test
public void testGroupBy1() throws Exception {
final String sql = "SELECT __time,\n"
+ " APPROX_COUNT_DISTINCT_HLLD(hll_dim1, 14) cnt\n"
+ "FROM druid.foo group by __time";
QueryTestBuilder builder = testBuilder().sql(sql).skipVectorize();
builder.run();
QueryTestRunner.QueryResults queryResults = builder.results();
List<Object[]> results = queryResults.results;
for (Object[] result : results) {
System.out.println(Arrays.toString(result));
}
}
@Test
public void testGroupBy2() throws Exception {
final String sql = "SELECT __time,\n"
+ " APPROX_COUNT_DISTINCT_HLLD(hll_dim1, 14) cnt\n"
+ "FROM druid.foo group by __time order by cnt desc";
QueryTestBuilder builder = testBuilder().sql(sql).skipVectorize();
builder.run();
QueryTestRunner.QueryResults queryResults = builder.results();
List<Object[]> results = queryResults.results;
for (Object[] result : results) {
System.out.println(Arrays.toString(result));
}
}
}