重构版本
This commit is contained in:
wangkuan
@@ -1,243 +1,193 @@
|
||||
package com.galaxy.tsg.function;
|
||||
|
||||
import com.galaxy.tsg.pojo.*;
|
||||
import org.apache.flink.api.java.tuple.Tuple4;
|
||||
import org.apache.flink.api.java.tuple.Tuple5;
|
||||
import org.apache.flink.streaming.api.functions.windowing.ProcessWindowFunction;
|
||||
import org.apache.flink.streaming.api.windowing.windows.TimeWindow;
|
||||
import org.apache.flink.util.Collector;
|
||||
|
||||
import java.util.PriorityQueue;
|
||||
|
||||
public class metricsCalculate extends ProcessWindowFunction<
|
||||
Entity, // 输入类型
|
||||
ResultEntity, // 输出类型
|
||||
Tuple4<String,Long,String,String>, // 键类型
|
||||
public class metricsCalculate extends ProcessWindowFunction<
|
||||
transformEntity, // 输入类型
|
||||
resultEntity, // 输出类型
|
||||
Tuple5<String, Long, String, String, String>, // 键类型
|
||||
TimeWindow> { // 窗口类型
|
||||
private final int topSize;
|
||||
private final String key;
|
||||
// Set<ObjectEntity> allSet = new TreeSet<ObjectEntity>() ;
|
||||
private PriorityQueue<SessionResultEntity> sessionOrderEntity ;
|
||||
private PriorityQueue<PacketResultEntity> packetOrderEntity ;
|
||||
private PriorityQueue<ByteResultEntity> byteOrderEntity ;
|
||||
private PriorityQueue<sessionResultEntity> sessionOrderEntity;
|
||||
private PriorityQueue<packetResultEntity> packetOrderEntity;
|
||||
private PriorityQueue<byteResultEntity> byteOrderEntity;
|
||||
|
||||
public metricsCalculate(int i,String key) {
|
||||
private long windowEndTimestamp;
|
||||
|
||||
public metricsCalculate(int i, String key) {
|
||||
this.key = key;
|
||||
this.topSize = i;
|
||||
this.sessionOrderEntity=new PriorityQueue<>();
|
||||
this.packetOrderEntity=new PriorityQueue<>();
|
||||
this.byteOrderEntity=new PriorityQueue<>();
|
||||
this.sessionOrderEntity = new PriorityQueue<>();
|
||||
this.packetOrderEntity = new PriorityQueue<>();
|
||||
this.byteOrderEntity = new PriorityQueue<>();
|
||||
}
|
||||
|
||||
@Override
|
||||
public void process(Tuple4<String,Long,String,String> s,
|
||||
public void process(Tuple5<String, Long, String, String, String> s,
|
||||
Context context,
|
||||
Iterable<Entity> elements, Collector<ResultEntity> out) throws Exception {
|
||||
Iterable<transformEntity> elements, Collector<resultEntity> out) throws Exception {
|
||||
|
||||
if (context.window().getEnd() == windowEndTimestamp) {
|
||||
|
||||
|
||||
|
||||
if (elements.iterator().hasNext()) {
|
||||
|
||||
for(Entity objectEntity : elements) {
|
||||
|
||||
if (sessionOrderEntity.size() < topSize) {
|
||||
sessionOrderEntity.add(enrichessionResult(context.window().getEnd() / 1000, objectEntity));
|
||||
} else {
|
||||
if (sessionOrderEntity.peek() != null) {
|
||||
SessionResultEntity res = sessionOrderEntity.peek();
|
||||
if (res.getSession_num() <= objectEntity.getCommon_sessions()) {
|
||||
sessionOrderEntity.poll();
|
||||
sessionOrderEntity.add(enrichessionResult(context.window().getEnd() / 1000, objectEntity));
|
||||
transformEntity objectTransformEntity = elements.iterator().next();
|
||||
|
||||
if (sessionOrderEntity.size() < topSize) {
|
||||
sessionOrderEntity.add(enrichessionResult(context.window().getEnd(), objectTransformEntity));
|
||||
} else {
|
||||
if (sessionOrderEntity.peek() != null) {
|
||||
sessionResultEntity res = sessionOrderEntity.peek();
|
||||
if (res.getSessions() <= objectTransformEntity.getSessions()) {
|
||||
sessionOrderEntity.poll();
|
||||
sessionOrderEntity.add(enrichessionResult(context.window().getEnd(), objectTransformEntity));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
if (packetOrderEntity.size() < topSize) {
|
||||
packetOrderEntity.add(enrichPacketResult(context.window().getEnd() / 1000, objectEntity));
|
||||
} else {
|
||||
if (packetOrderEntity.peek() != null) {
|
||||
PacketResultEntity res = packetOrderEntity.peek();
|
||||
if ((res.getS2c_pkt_num() + res.getC2s_pkt_num()) <= (objectEntity.getCommon_s2c_pkt_num() + objectEntity.getCommon_c2s_pkt_num())) {
|
||||
packetOrderEntity.poll();
|
||||
packetOrderEntity.add(enrichPacketResult(context.window().getEnd() / 1000, objectEntity));
|
||||
if (packetOrderEntity.size() < topSize) {
|
||||
packetOrderEntity.add(enrichPacketResult(context.window().getEnd(), objectTransformEntity));
|
||||
} else {
|
||||
if (packetOrderEntity.peek() != null) {
|
||||
packetResultEntity res = packetOrderEntity.peek();
|
||||
if ((res.getIn_pkts() + res.getOut_pkts()) <= (objectTransformEntity.getIn_pkts() + objectTransformEntity.getOut_pkts())) {
|
||||
packetOrderEntity.poll();
|
||||
packetOrderEntity.add(enrichPacketResult(context.window().getEnd() , objectTransformEntity));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (byteOrderEntity.size() < topSize) {
|
||||
byteOrderEntity.add(enrichByteResult(context.window().getEnd() / 1000, objectEntity));
|
||||
} else {
|
||||
if (byteOrderEntity.peek() != null) {
|
||||
ByteResultEntity res = byteOrderEntity.peek();
|
||||
if ((res.getS2c_byte_num() + res.getC2s_byte_num()) <= (objectEntity.getCommon_s2c_byte_num() + objectEntity.getCommon_c2s_byte_num())) {
|
||||
byteOrderEntity.poll();
|
||||
byteOrderEntity.add(enrichByteResult(context.window().getEnd() / 1000, objectEntity));
|
||||
if (byteOrderEntity.size() < topSize) {
|
||||
byteOrderEntity.add(enrichByteResult(context.window().getEnd(), objectTransformEntity));
|
||||
} else {
|
||||
if (byteOrderEntity.peek() != null) {
|
||||
byteResultEntity res = byteOrderEntity.peek();
|
||||
if ((res.getIn_bytes() + res.getOut_bytes()) <= (objectTransformEntity.getIn_bytes() + objectTransformEntity.getOut_bytes())) {
|
||||
byteOrderEntity.poll();
|
||||
byteOrderEntity.add(enrichByteResult(context.window().getEnd(), objectTransformEntity));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
} else {
|
||||
windowEndTimestamp = context.window().getEnd();
|
||||
|
||||
for (com.galaxy.tsg.pojo.sessionResultEntity sessionResultEntity : sessionOrderEntity) {
|
||||
|
||||
resultEntity en = new resultEntity();
|
||||
en.setOrder_by("sessions");
|
||||
en.setStat_time(context.window().getEnd());
|
||||
en.setSessionResultEntity(sessionResultEntity);
|
||||
out.collect(en);
|
||||
}
|
||||
sessionOrderEntity.clear();
|
||||
|
||||
|
||||
for (com.galaxy.tsg.pojo.packetResultEntity packetResultEntity : packetOrderEntity) {
|
||||
resultEntity en = new resultEntity();
|
||||
en.setOrder_by("packets");
|
||||
en.setStat_time(context.window().getEnd());
|
||||
en.setPacketResultEntity(packetResultEntity);
|
||||
out.collect(en);
|
||||
}
|
||||
packetOrderEntity.clear();
|
||||
|
||||
for (com.galaxy.tsg.pojo.byteResultEntity byteResultEntity : byteOrderEntity) {
|
||||
resultEntity en = new resultEntity();
|
||||
en.setOrder_by("bytes");
|
||||
en.setStat_time(context.window().getEnd());
|
||||
en.setByteResultEntity(byteResultEntity);
|
||||
out.collect(en);
|
||||
}
|
||||
byteOrderEntity.clear();
|
||||
|
||||
}
|
||||
|
||||
while (sessionOrderEntity.size() > 0) {
|
||||
SessionResultEntity obj = sessionOrderEntity.peek();
|
||||
//String jsonStr = JSONObject.toJSONString(en);
|
||||
ResultEntity en = new ResultEntity();
|
||||
en.setOrder_by("sessions");
|
||||
en.setStat_time(context.window().getEnd() / 1000);
|
||||
en.setSessionResultEntity(obj);
|
||||
out.collect(en);
|
||||
sessionOrderEntity.remove();
|
||||
}
|
||||
|
||||
|
||||
while (packetOrderEntity.size() > 0) {
|
||||
PacketResultEntity obj = packetOrderEntity.peek();
|
||||
ResultEntity en = new ResultEntity();
|
||||
en.setOrder_by("packets");
|
||||
en.setStat_time(context.window().getEnd() / 1000);
|
||||
en.setPacketResultEntity(obj);
|
||||
out.collect(en);
|
||||
packetOrderEntity.remove();
|
||||
}
|
||||
|
||||
while (byteOrderEntity.size() > 0) {
|
||||
ByteResultEntity obj = byteOrderEntity.peek();
|
||||
ResultEntity en = new ResultEntity();
|
||||
en.setOrder_by("bytes");
|
||||
en.setStat_time(context.window().getEnd() / 1000);
|
||||
en.setByteResultEntity(obj);
|
||||
out.collect(en);
|
||||
byteOrderEntity.remove();
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
public ByteResultEntity enrichByteResult(Long time,Entity objectEntity) {
|
||||
ByteResultEntity en = new ByteResultEntity();
|
||||
en.setVsys_id(objectEntity.getCommon_vsys_id());
|
||||
en.setStat_time(time);
|
||||
en.setSource(objectEntity.getCommon_client_ip());
|
||||
en.setSession_num(objectEntity.getCommon_sessions());
|
||||
public byteResultEntity enrichByteResult(Long time, transformEntity objectTransformEntity) {
|
||||
byteResultEntity en = new byteResultEntity();
|
||||
en.setVsys_id(objectTransformEntity.getVsys_id());
|
||||
en.setTimestamp(time);
|
||||
en.setSessions(objectTransformEntity.getSessions());
|
||||
en.setOrder_by("bytes");
|
||||
en.setDevice_group(objectEntity.getCommon_device_group());
|
||||
en.setData_center(objectEntity.getCommon_data_center());
|
||||
en.setS2c_pkt_num(objectEntity.getCommon_s2c_pkt_num());
|
||||
en.setC2s_pkt_num(objectEntity.getCommon_c2s_pkt_num());
|
||||
en.setS2c_byte_num(objectEntity.getCommon_s2c_byte_num());
|
||||
en.setC2s_byte_num(objectEntity.getCommon_c2s_byte_num());
|
||||
switch (key) {
|
||||
case "common_client_ip":
|
||||
en.setSource(objectEntity.getCommon_client_ip());
|
||||
break;
|
||||
case "common_server_ip":
|
||||
en.setDestination(objectEntity.getCommon_server_ip());
|
||||
break;
|
||||
case "common_internal_ip":
|
||||
en.setSource(objectEntity.getCommon_internal_ip());
|
||||
break;
|
||||
case "common_external_ip":
|
||||
en.setDestination(objectEntity.getCommon_external_ip());
|
||||
break;
|
||||
case "http_domain":
|
||||
en.setDomain(objectEntity.getHttp_domain());
|
||||
break;
|
||||
|
||||
case "common_subscriber_id":
|
||||
en.setSubscriber_id(objectEntity.getCommon_subscriber_id());
|
||||
break;
|
||||
en.setServer_ip(objectTransformEntity.getServer_ip());
|
||||
en.setClient_ip(objectTransformEntity.getClient_ip());
|
||||
en.setSubscriber_id(objectTransformEntity.getSubscriber_id());
|
||||
en.setFqdn(objectTransformEntity.getFqdn());
|
||||
en.setExternal_ip(objectTransformEntity.getExternal_ip());
|
||||
en.setInternal_ip(objectTransformEntity.getInternal_ip());
|
||||
en.setDomain(objectTransformEntity.getDomain());
|
||||
en.setDevice_group(objectTransformEntity.getDevice_group());
|
||||
en.setData_center(objectTransformEntity.getData_center());
|
||||
en.setIn_bytes(objectTransformEntity.getIn_bytes());
|
||||
en.setIn_pkts(objectTransformEntity.getIn_pkts());
|
||||
en.setOut_bytes(objectTransformEntity.getOut_bytes());
|
||||
en.setOut_pkts(objectTransformEntity.getOut_pkts());
|
||||
en.setKey_by(objectTransformEntity.getKey_by());
|
||||
|
||||
case "common_app_label":
|
||||
en.setApp_name(objectEntity.getCommon_app_label());
|
||||
break;
|
||||
|
||||
default:
|
||||
|
||||
}
|
||||
return en;
|
||||
|
||||
}
|
||||
public SessionResultEntity enrichessionResult(Long time,Entity objectEntity){
|
||||
|
||||
SessionResultEntity en =new SessionResultEntity();
|
||||
en.setVsys_id(objectEntity.getCommon_vsys_id());
|
||||
en.setStat_time(time);
|
||||
en.setSession_num(objectEntity.getCommon_sessions());
|
||||
public sessionResultEntity enrichessionResult(Long time, transformEntity objectTransformEntity) {
|
||||
|
||||
sessionResultEntity en = new sessionResultEntity();
|
||||
en.setVsys_id(objectTransformEntity.getVsys_id());
|
||||
en.setTimestamp(time);
|
||||
en.setSessions(objectTransformEntity.getSessions());
|
||||
en.setOrder_by("sessions");
|
||||
en.setDevice_group(objectEntity.getCommon_device_group());
|
||||
en.setData_center(objectEntity.getCommon_data_center());
|
||||
en.setS2c_pkt_num(objectEntity.getCommon_s2c_pkt_num());
|
||||
en.setC2s_pkt_num(objectEntity.getCommon_c2s_pkt_num());
|
||||
en.setS2c_byte_num(objectEntity.getCommon_s2c_byte_num());
|
||||
en.setC2s_byte_num(objectEntity.getCommon_c2s_byte_num());
|
||||
switch(key) {
|
||||
case "common_client_ip":
|
||||
en.setSource(objectEntity.getCommon_client_ip());
|
||||
break;
|
||||
case "common_server_ip":
|
||||
en.setDestination(objectEntity.getCommon_server_ip());
|
||||
break;
|
||||
case "common_internal_ip":
|
||||
en.setSource(objectEntity.getCommon_internal_ip());
|
||||
break;
|
||||
case "common_external_ip":
|
||||
en.setDestination(objectEntity.getCommon_external_ip());
|
||||
break;
|
||||
case "http_domain":
|
||||
en.setDomain(objectEntity.getHttp_domain());
|
||||
break;
|
||||
|
||||
case "common_subscriber_id":
|
||||
en.setSubscriber_id(objectEntity.getCommon_subscriber_id());
|
||||
break;
|
||||
|
||||
case "common_app_label":
|
||||
en.setApp_name(objectEntity.getCommon_app_label());
|
||||
break;
|
||||
|
||||
default:
|
||||
|
||||
|
||||
|
||||
}
|
||||
return en;
|
||||
en.setServer_ip(objectTransformEntity.getServer_ip());
|
||||
en.setClient_ip(objectTransformEntity.getClient_ip());
|
||||
en.setSubscriber_id(objectTransformEntity.getSubscriber_id());
|
||||
en.setFqdn(objectTransformEntity.getFqdn());
|
||||
en.setExternal_ip(objectTransformEntity.getExternal_ip());
|
||||
en.setInternal_ip(objectTransformEntity.getInternal_ip());
|
||||
en.setDomain(objectTransformEntity.getDomain());
|
||||
en.setDevice_group(objectTransformEntity.getDevice_group());
|
||||
en.setData_center(objectTransformEntity.getData_center());
|
||||
en.setIn_bytes(objectTransformEntity.getIn_bytes());
|
||||
en.setIn_pkts(objectTransformEntity.getIn_pkts());
|
||||
en.setOut_bytes(objectTransformEntity.getOut_bytes());
|
||||
en.setOut_pkts(objectTransformEntity.getOut_pkts());
|
||||
en.setKey_by(objectTransformEntity.getKey_by());
|
||||
return en;
|
||||
}
|
||||
public PacketResultEntity enrichPacketResult(Long time,Entity objectEntity){
|
||||
PacketResultEntity en =new PacketResultEntity();
|
||||
en.setVsys_id(objectEntity.getCommon_vsys_id());
|
||||
en.setStat_time(time);
|
||||
en.setSource(objectEntity.getCommon_client_ip());
|
||||
en.setSession_num(objectEntity.getCommon_sessions());
|
||||
|
||||
public packetResultEntity enrichPacketResult(Long time, transformEntity objectTransformEntity) {
|
||||
packetResultEntity en = new packetResultEntity();
|
||||
en.setVsys_id(objectTransformEntity.getVsys_id());
|
||||
en.setTimestamp(time);
|
||||
en.setSessions(objectTransformEntity.getSessions());
|
||||
en.setOrder_by("packets");
|
||||
en.setDevice_group(objectEntity.getCommon_device_group());
|
||||
en.setData_center(objectEntity.getCommon_data_center());
|
||||
en.setS2c_pkt_num(objectEntity.getCommon_s2c_pkt_num());
|
||||
en.setC2s_pkt_num(objectEntity.getCommon_c2s_pkt_num());
|
||||
en.setS2c_byte_num(objectEntity.getCommon_s2c_byte_num());
|
||||
en.setC2s_byte_num(objectEntity.getCommon_c2s_byte_num());
|
||||
switch(key) {
|
||||
case "common_client_ip":
|
||||
en.setSource(objectEntity.getCommon_client_ip());
|
||||
break;
|
||||
case "common_server_ip":
|
||||
en.setDestination(objectEntity.getCommon_server_ip());
|
||||
break;
|
||||
case "common_internal_ip":
|
||||
en.setSource(objectEntity.getCommon_internal_ip());
|
||||
break;
|
||||
case "common_external_ip":
|
||||
en.setDestination(objectEntity.getCommon_external_ip());
|
||||
break;
|
||||
case "common_subscriber_id":
|
||||
en.setSubscriber_id(objectEntity.getCommon_subscriber_id());
|
||||
break;
|
||||
en.setServer_ip(objectTransformEntity.getServer_ip());
|
||||
en.setClient_ip(objectTransformEntity.getClient_ip());
|
||||
en.setSubscriber_id(objectTransformEntity.getSubscriber_id());
|
||||
en.setFqdn(objectTransformEntity.getFqdn());
|
||||
en.setExternal_ip(objectTransformEntity.getExternal_ip());
|
||||
en.setInternal_ip(objectTransformEntity.getInternal_ip());
|
||||
en.setDomain(objectTransformEntity.getDomain());
|
||||
en.setDevice_group(objectTransformEntity.getDevice_group());
|
||||
en.setData_center(objectTransformEntity.getData_center());
|
||||
en.setIn_bytes(objectTransformEntity.getIn_bytes());
|
||||
en.setIn_pkts(objectTransformEntity.getIn_pkts());
|
||||
en.setOut_bytes(objectTransformEntity.getOut_bytes());
|
||||
en.setOut_pkts(objectTransformEntity.getOut_pkts());
|
||||
en.setKey_by(objectTransformEntity.getKey_by());
|
||||
|
||||
case "common_app_label":
|
||||
en.setApp_name(objectEntity.getCommon_app_label());
|
||||
break;
|
||||
|
||||
default:
|
||||
|
||||
|
||||
}
|
||||
return en;
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user