提交Live Traffic Chart重构后初版代码。（TSG-14799）
This commit is contained in:
qidaijie
248
src/test/java/com/zdjizhi/DatasketchesTest.java
Normal file
248
src/test/java/com/zdjizhi/DatasketchesTest.java
Normal file
@@ -0,0 +1,248 @@
|
||||
package com.zdjizhi;
|
||||
|
||||
import cn.hutool.json.JSONUtil;
|
||||
import com.alibaba.fastjson2.*;
|
||||
import com.zdjizhi.utils.JsonMapper;
|
||||
import org.apache.datasketches.hll.HllSketch;
|
||||
import org.apache.datasketches.hll.Union;
|
||||
import org.apache.kafka.clients.producer.KafkaProducer;
|
||||
import org.apache.kafka.clients.producer.ProducerRecord;
|
||||
import org.junit.Test;
|
||||
|
||||
import java.util.*;
|
||||
|
||||
/**
|
||||
* @author qidaijie
|
||||
* @Package com.zdjizhi
|
||||
* @Description:
|
||||
* @date 2023/3/217:17
|
||||
*/
|
||||
public class DatasketchesTest {
|
||||
|
||||
@Test
|
||||
public void HllSketchTest() {
|
||||
HashSet<String> strings = new HashSet<>();
|
||||
|
||||
HllSketch sketch = new HllSketch(12);
|
||||
|
||||
for (int i = 0; i < 50; i++) {
|
||||
String ip = "192.168.1." + i;
|
||||
sketch.update(ip);
|
||||
strings.add(ip);
|
||||
}
|
||||
|
||||
System.out.println(sketch.getEstimate() + "--" + strings.size());
|
||||
|
||||
HashSet<String> randomStrings = new HashSet<>();
|
||||
|
||||
HllSketch randomSketch = new HllSketch(12);
|
||||
|
||||
for (int i = 0; i < 50; i++) {
|
||||
String ip = makeIPv4Random();
|
||||
randomSketch.update(ip);
|
||||
randomStrings.add(ip);
|
||||
}
|
||||
|
||||
System.out.println(randomSketch.getEstimate() + "--" + randomStrings.size());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void HllSketchUnionTest() {
|
||||
HashSet<String> strings = new HashSet<>();
|
||||
|
||||
HllSketch sketch = new HllSketch(12);
|
||||
|
||||
for (int i = 0; i < 50; i++) {
|
||||
String ip = "192.168.1." + i;
|
||||
sketch.update(ip);
|
||||
strings.add(ip);
|
||||
}
|
||||
|
||||
HllSketch sketch2 = new HllSketch(12);
|
||||
|
||||
for (int i = 0; i < 10; i++) {
|
||||
String ip = "192.168.2." + i;
|
||||
sketch2.update(ip);
|
||||
strings.add(ip);
|
||||
}
|
||||
|
||||
Union union = new Union(12);
|
||||
|
||||
union.update(sketch);
|
||||
union.update(sketch2);
|
||||
HllSketch sketch_result = HllSketch.heapify(union.getResult().toCompactByteArray());
|
||||
|
||||
System.out.println(sketch.getEstimate() + "--" + strings.size());
|
||||
System.out.println(sketch2.getEstimate() + "--" + strings.size());
|
||||
System.out.println(sketch_result.getEstimate() + "--" + strings.size());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void HllSketchDruidTest() {
|
||||
HashMap<String, Object> dataMap = new HashMap<>();
|
||||
|
||||
HashSet<String> strings = new HashSet<>();
|
||||
|
||||
HllSketch sketch = new HllSketch(12);
|
||||
|
||||
for (int i = 0; i < 50; i++) {
|
||||
String ip = "192.168.1." + i;
|
||||
sketch.update(ip);
|
||||
strings.add(ip);
|
||||
}
|
||||
|
||||
HllSketch sketch2 = new HllSketch(12);
|
||||
|
||||
for (int i = 0; i < 10; i++) {
|
||||
String ip = "192.168.2." + i;
|
||||
sketch2.update(ip);
|
||||
strings.add(ip);
|
||||
}
|
||||
|
||||
Union union = new Union(12);
|
||||
|
||||
union.update(sketch);
|
||||
union.update(sketch2);
|
||||
HllSketch sketch_result1 = HllSketch.heapify(union.getResult().toCompactByteArray());
|
||||
|
||||
HllSketch sketch3 = new HllSketch(12);
|
||||
|
||||
for (int i = 0; i < 10; i++) {
|
||||
String ip = "192.168.3." + i;
|
||||
sketch3.update(ip);
|
||||
strings.add(ip);
|
||||
}
|
||||
|
||||
Union union2 = new Union(12);
|
||||
|
||||
union2.update(sketch_result1);
|
||||
union2.update(sketch3);
|
||||
HllSketch sketch_result2 = HllSketch.heapify(union2.getResult().toCompactByteArray());
|
||||
|
||||
System.out.println(sketch.getEstimate() + "--" + strings.size());
|
||||
System.out.println(sketch2.getEstimate() + "--" + strings.size());
|
||||
System.out.println(sketch3.getEstimate() + "--" + strings.size());
|
||||
System.out.println(sketch_result1.getEstimate() + "--" + strings.size());
|
||||
System.out.println(sketch_result2.getEstimate() + "--" + strings.size());
|
||||
|
||||
Result result = new Result();
|
||||
result.setC2s_pkt_num(10);
|
||||
result.setS2c_pkt_num(10);
|
||||
result.setC2s_byte_num(10);
|
||||
result.setS2c_byte_num(10);
|
||||
result.setStat_time(1679970031);
|
||||
result.setSchema_type("HLLSketchMergeTest");
|
||||
|
||||
//CompactByte
|
||||
result.setIp_object(sketch_result2.toCompactByteArray());
|
||||
// System.out.println(result.toString());
|
||||
//sendMessage(JsonMapper.toJsonString(result);
|
||||
|
||||
|
||||
//UpdatableByte
|
||||
result.setIp_object(sketch_result2.toUpdatableByteArray());
|
||||
// System.out.println(result.toString());
|
||||
//sendMessage(JsonMapper.toJsonString(result);
|
||||
|
||||
//Hashmap
|
||||
dataMap.put("app_name", "TEST");
|
||||
dataMap.put("protocol_stack_id", "HTTP");
|
||||
dataMap.put("vsys_id", 1);
|
||||
dataMap.put("stat_time", 1681370100);
|
||||
dataMap.put("client_ip_sketch", sketch_result2.toUpdatableByteArray());
|
||||
|
||||
System.out.println("Jackson:" + JsonMapper.toJsonString(dataMap));
|
||||
System.out.println("FastJson2:" + JSONObject.toJSONString(dataMap));
|
||||
System.out.println("Hutool:" + JSONUtil.toJsonStr(dataMap) + "\n\n");
|
||||
|
||||
|
||||
dataMap.put("client_ip_sketch", Base64.getEncoder().encode(sketch_result2.toUpdatableByteArray()));
|
||||
System.out.println("FastJson2 Byte(Base64):" + JSONObject.toJSONString(dataMap));
|
||||
System.out.println("Hutool Byte(Base64):" + JSONObject.toJSONString(dataMap));
|
||||
System.out.println(JSONUtil.toJsonStr(dataMap));
|
||||
|
||||
|
||||
// sendMessage(JSONObject.toJSONString(dataMap));
|
||||
}
|
||||
|
||||
|
||||
//随机生成ip
|
||||
private static String makeIPv4Random() {
|
||||
int v4_1 = new Random().nextInt(255) + 1;
|
||||
int v4_2 = new Random().nextInt(255);
|
||||
int v4_3 = new Random().nextInt(255);
|
||||
int v4_4 = new Random().nextInt(255);
|
||||
return v4_1 + "." + v4_2 + "." + v4_3 + "." + v4_4;
|
||||
}
|
||||
|
||||
private static void sendMessage(Object message) {
|
||||
Properties props = new Properties();
|
||||
//kafka地址
|
||||
props.put("bootstrap.servers", "192.168.44.12:9092");
|
||||
props.put("acks", "all");
|
||||
props.put("retries", 0);
|
||||
props.put("linger.ms", 1);
|
||||
props.put("buffer.memory", 67108864);
|
||||
// props.put("key.serializer", "org.apache.kafka.common.serialization.StringSerializer");
|
||||
props.put("key.serializer", "org.apache.kafka.common.serialization.ByteArraySerializer");
|
||||
// props.put("value.serializer", "org.apache.kafka.common.serialization.StringSerializer");
|
||||
props.put("value.serializer", "org.apache.kafka.common.serialization.ByteArraySerializer");
|
||||
KafkaProducer<String, Object> kafkaProducer = new KafkaProducer<String, Object>(props);
|
||||
|
||||
kafkaProducer.send(new ProducerRecord<String, Object>("TRAFFIC-PROTOCOL-TEST", message));
|
||||
|
||||
kafkaProducer.close();
|
||||
}
|
||||
}
|
||||
|
||||
class Result {
|
||||
|
||||
private String schema_type;
|
||||
private long c2s_byte_num;
|
||||
private long c2s_pkt_num;
|
||||
private long s2c_byte_num;
|
||||
private long s2c_pkt_num;
|
||||
private long stat_time;
|
||||
private byte[] ip_object;
|
||||
|
||||
public void setSchema_type(String schema_type) {
|
||||
this.schema_type = schema_type;
|
||||
}
|
||||
|
||||
public void setC2s_byte_num(long c2s_byte_num) {
|
||||
this.c2s_byte_num = c2s_byte_num;
|
||||
}
|
||||
|
||||
public void setC2s_pkt_num(long c2s_pkt_num) {
|
||||
this.c2s_pkt_num = c2s_pkt_num;
|
||||
}
|
||||
|
||||
public void setS2c_byte_num(long s2c_byte_num) {
|
||||
this.s2c_byte_num = s2c_byte_num;
|
||||
}
|
||||
|
||||
public void setS2c_pkt_num(long s2c_pkt_num) {
|
||||
this.s2c_pkt_num = s2c_pkt_num;
|
||||
}
|
||||
|
||||
public void setStat_time(long stat_time) {
|
||||
this.stat_time = stat_time;
|
||||
}
|
||||
|
||||
public void setIp_object(byte[] ip_object) {
|
||||
this.ip_object = ip_object;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return "Result{" +
|
||||
"schema_type='" + schema_type + '\'' +
|
||||
", c2s_byte_num=" + c2s_byte_num +
|
||||
", c2s_pkt_num=" + c2s_pkt_num +
|
||||
", s2c_byte_num=" + s2c_byte_num +
|
||||
", s2c_pkt_num=" + s2c_pkt_num +
|
||||
", stat_time=" + stat_time +
|
||||
", ip_object=" + Arrays.toString(ip_object) +
|
||||
'}';
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user