57 Commits

Author SHA1 Message Date
梁超
53c6c267e8 Merge branch 'hotfix/field-completion' into 'main'
[GAL-568] fix: add field 'rtp_originator_dir'

See merge request galaxy/tsg_olap/sip-rtp-correlation!25
2024-05-10 05:50:10 +00:00
chaochaoc
20e8421900 chore: update versions 2024-05-10 13:49:37 +08:00
chaochaoc
d01235e092 [GAL-568] fix: add field 'rtp_originator_dir' 2024-05-10 11:58:35 +08:00
梁超
12828291a9 Merge branch 'hotfix/app-name' into 'main'
fix: modify job name

See merge request galaxy/tsg_olap/sip-rtp-correlation!24
2023-12-18 07:52:56 +00:00
chaoc
f21e814763 chore: update versions 2023-12-18 15:47:41 +08:00
chaoc
11c2c641bb docs: add job name config 2023-12-18 15:47:30 +08:00
chaoc
59cabb4868 fix: add job name config 2023-12-18 15:47:23 +08:00
chaoc
b7c739a955 chore: update version 2023-12-15 10:32:17 +08:00
梁超
1f6ef08a30 Merge branch 'hotfix/illegal-flags' into 'release/1.2'
[GAL-444] fix: fix error caused by invalid flags

See merge request galaxy/tsg_olap/sip-rtp-correlation!23
2023-12-15 02:27:43 +00:00
chaoc
87abd1e2ca [GAL-444] fix: fix error caused by invalid flags 2023-12-15 10:24:06 +08:00
梁超
2542a8bfd2 Merge branch 'bugfix/repeat-voip' into 'main'
chore: update version

See merge request galaxy/tsg_olap/sip-rtp-correlation!21
2023-12-07 09:22:19 +00:00
chaoc
1c0259a95c chore: update version 2023-12-07 17:20:20 +08:00
梁超
37f49c40d5 Merge branch 'bugfix/repeat-voip' into 'main'
fix: repeated voip record

See merge request galaxy/tsg_olap/sip-rtp-correlation!20
2023-12-07 09:19:32 +00:00
chaoc
beef47df4c fix: add ipaddr non-empty check 2023-12-07 17:17:05 +08:00
chaoc
6d77d1c3c0 fix: remove rtp from state when fusion successfully 2023-12-07 17:15:11 +08:00
梁超
4179a0a887 Merge branch 'bugfix/some-err' into 'main'
[GAL-444] fix: output unmatched sip record

See merge request galaxy/tsg_olap/sip-rtp-correlation!19
2023-12-06 10:17:02 +00:00
chaoc
6ebefc9026 chore: update version 2023-12-06 18:15:37 +08:00
chaoc
701019c38a [GAL-444] fix: output unmatched sip record 2023-12-06 18:14:52 +08:00
chaoc
6ae7fdef06 [GAL-444] fix: use default '1' instead of '0' 2023-12-06 18:14:33 +08:00
梁超
e277117c6d Merge branch 'release/1.2' into 'main'
chore: add distribution management

See merge request galaxy/tsg_olap/sip-rtp-correlation!18
2023-12-05 09:53:19 +00:00
chaoc
d54c93b61d chore: add distribution management 2023-12-05 17:52:49 +08:00
梁超
3b06d3dfd5 Merge branch 'release/1.2' into 'main'
chore: update version

See merge request galaxy/tsg_olap/sip-rtp-correlation!17
2023-12-05 09:51:35 +00:00
chaoc
e718120be1 chore: update version 2023-12-05 17:47:51 +08:00
梁超
1dffb8fb6f Merge branch 'feature/field-update' into 'main'
[GAL-444] feat: field update

See merge request galaxy/tsg_olap/sip-rtp-correlation!16
2023-12-05 09:45:50 +00:00
chaoc
0994219ede fix: long value reader 2023-12-05 17:43:32 +08:00
chaoc
6f915e5162 fix: update flag 2023-12-04 17:19:32 +08:00
chaoc
35247d7414 fix: update field name 2023-12-04 17:08:53 +08:00
chaoc
64f19b528e chore: update version 2023-12-04 10:29:04 +08:00
chaoc
d0c3ebd60f feat: update stream dir field read method 2023-12-04 10:28:03 +08:00
chaoc
114c180742 feature: field update 2023-11-21 09:59:44 +08:00
梁超
e51d693fa0 Merge branch 'release/1.1' into 'main'
chore: update version

See merge request galaxy/tsg_olap/sip-rtp-correlation!15
2023-10-25 08:43:28 +00:00
chaoc
757808d313 chore: update version 2023-10-25 16:42:31 +08:00
梁超
e123233af9 Merge branch 'feature/config-key' into 'main'
[GAL-419] feat: modify config option name

See merge request galaxy/tsg_olap/sip-rtp-correlation!14
2023-10-25 08:40:51 +00:00
chaoc
ad3072cc7a docs: modify config option name 2023-10-25 16:35:30 +08:00
chaoc
617ddab7ab feat: modify config option name 2023-10-25 16:30:27 +08:00
梁超
45891bc734 Merge branch 'feature/internal-ip-config' into 'main'
feat: add config 'DETERMINE_INTRANET_IP_BE_ABNORMAL'

See merge request galaxy/tsg_olap/sip-rtp-correlation!13
2023-10-13 06:43:38 +00:00
chaoc
19e0bce58f chore: update version 2023-10-13 14:43:08 +08:00
chaoc
10ce6cfa07 feat: add config 'DETERMINE_INTRANET_IP_BE_ABNORMAL' 2023-10-13 14:38:23 +08:00
梁超
9d5d99974b Merge branch 'release/1.0' into 'main'
release: 1.0

See merge request galaxy/tsg_olap/sip-rtp-correlation!12
2023-10-12 06:52:09 +00:00
梁超
970977ba3c Merge branch 'hotfix/output-sip' into 'release/1.0'
hotfix: add sip record output

See merge request galaxy/tsg_olap/sip-rtp-correlation!11
2023-10-12 02:44:32 +00:00
chaoc
732d9f5aa9 style: update version 2023-10-12 10:42:48 +08:00
chaoc
96fa19aea1 fix: use event time timer 2023-10-12 10:36:44 +08:00
chaoc
1b7c33d078 fix: output sip record 2023-10-12 10:36:29 +08:00
liang chao
b9463f07ac Merge branch 'release/1.0' into 'main'
merge: 1.0-rc3

See merge request galaxy/tsg_olap/sip-rtp-correlation!10
2023-08-28 03:30:06 +00:00
liang chao
35e2807a91 Merge branch 'hotfix/rename' into 'release/1.0'
style: rename job name

See merge request galaxy/tsg_olap/sip-rtp-correlation!9
2023-08-28 03:29:10 +00:00
chaoc
2275f349d1 Merge remote-tracking branch 'origin/release/1.0' into hotfix/rename 2023-08-28 11:27:23 +08:00
chaoc
1fedfbe4b8 style: add plugin reproducible 2023-08-28 11:26:25 +08:00
chaoc
b2f15b3919 style: modify job name 2023-08-28 11:11:02 +08:00
liang chao
8fc8cc7c2d Merge branch 'hotfix/null-point-err' into 'release/1.0'
style: update versions

See merge request galaxy/tsg_olap/sip-rtp-correlation!8
2023-08-21 09:09:25 +00:00
chaoc
98bb843159 style: update versions 2023-08-21 17:08:51 +08:00
liang chao
edb044596e Merge branch 'hotfix/null-point-err' into 'release/1.0'
docs: add hotfix changelog

See merge request galaxy/tsg_olap/sip-rtp-correlation!7
2023-08-21 09:06:49 +00:00
chaoc
557156af79 docs: add hotfix changelog 2023-08-21 17:06:22 +08:00
liang chao
32a811fb1c Merge branch 'hotfix/null-point-err' into 'release/1.0'
fix(utils): fix null point err

See merge request galaxy/tsg_olap/sip-rtp-correlation!6
2023-08-21 08:59:11 +00:00
chaoc
36cbaebf0c fix(utils): fix null point err 2023-08-21 16:57:13 +08:00
liang chao
da572f4bd0 Merge branch 'release/1.0' into 'main'
merge: 1.0.rc1

See merge request galaxy/tsg_olap/sip-rtp-correlation!5
2023-08-16 03:05:22 +00:00
liang chao
77cdd73f02 Merge branch 'hotfix/no-collect-expire-data' into 'main'
fix: cannot collect data due to expiration

See merge request galaxy/tsg_olap/sip-rtp-correlation!4
2023-08-11 06:18:46 +00:00
liang chao
5481a7b9ee Merge branch 'feature/address-keyby-impl' into 'main'
feature: develop job using java

See merge request galaxy/tsg_olap/sip-rtp-correlation!3
2023-08-10 09:39:51 +00:00
13 changed files with 262 additions and 50 deletions

8
CHANGELOG.md Normal file
View File

@@ -0,0 +1,8 @@
# Changelog
### Hotfix
- [#5](https://git.mesalab.cn/galaxy/tsg_olap/sip-rtp-correlation/-/issues/5) 修复了由于 IPUtil 在判断 Ipv6 地址没有判空而引起的空指针异常
### Feature
- 输出 SIP Record
- [GAL-419](https://jira.geedge.net/browse/GAL-419) 增加配置项 `include.intranet.ip`, 是否对 SIP 协商主叫 IP 或被叫 IP 为内网地址的数据进行关联,不关联则输出到异常 Topic 中。

View File

@@ -22,17 +22,19 @@ flink run -c com.zdjizhi.flink.voip.CorrelateApp path/to/sip-rtp-correlation-<ve
## 配置项说明
| 配置项 | 类型 | 必需 | 默认值 | 描述 |
| --------------------------- | ------------------- | ---------- | ----------------------------------------------------------- | ----------------------------------------------------------- |
| source.kafka.topic | STRING | Y | | 将要读取的 Kafka Topic 名称,其包含 SIP 和 RTP 原始数据 |
| source.kafka.props.* | MAP<STRING, STRING> | Y | | 将要读取的 Kafka 的 Properties |
| sink.kafka.topic | STRING | Y | | 将合成的 VoIP 及 未关联成功的 RTP 数据写出的 Kafka Topic 名 |
| sink.kafka.props.* | MAP<STRING, STRING> | Y | | 数据输出的 Kafka 的 Properties |
| error.records.output.enable | STRING | N | False | 是否开启异常数据的输出 【IP 或 Port 为空】 |
| error.sink.kafka.topic | STRING | N | | 异常数据输出到的 Kafka Topic 名 |
| error.sink.kafka.props.* | MAP<STRING, STRING> | N | | 异常数据输出的 Kafka 的 Properties |
| sip.state.clear.interval.minutes | INT | N | 1 | SIP 单向流关联的窗口大小(单位:分钟) |
| rtp.state.clear.interval.minutes | INT | N | 6 | SIP 和 RTP 关联的窗口大小(单位:分钟) |
| 配置项 | 类型 | 必需 | 默认值 | 描述 |
|----------------------------------| ------------------- | ---------- | ---------------------------------------------------------- |-------------------------------------------|
| source.kafka.topic | STRING | Y | | 将要读取的 Kafka Topic 名称,其包含 SIP 和 RTP 原始数据 |
| source.kafka.props.* | MAP<STRING, STRING> | Y | | 将要读取的 Kafka 的 Properties |
| sink.kafka.topic | STRING | Y | | 将合成的 VoIP 及 未关联成功的 RTP 数据写出的 Kafka Topic 名 |
| sink.kafka.props.* | MAP<STRING, STRING> | Y | | 数据输出的 Kafka 的 Properties |
| error.records.output.enable | BOOLEAN | N | False | 是否开启异常数据的输出 【IP 或 Port 为空】 |
| include.intranet.ip | BOOLEAN | N | True | 是否对 SIP 协商主叫 IP 或被叫 IP 为内网地址的数据进行关联 |
| error.sink.kafka.topic | STRING | N | | 异常数据输出的 Kafka Topic 名 |
| error.sink.kafka.props.* | MAP<STRING, STRING> | N | | 异常数据输出的 Kafka 的 Properties |
| sip.state.clear.interval.minutes | INT | N | 1 | SIP 单向流关联的窗口大小(单位:分钟) |
| rtp.state.clear.interval.minutes | INT | N | 6 | SIP 和 RTP 关联的窗口大小(单位:分钟) |
| job.name | STRING | N | correlation_sip_rtp_session | Job 名 |

28
pom.xml
View File

@@ -7,7 +7,7 @@
<groupId>com.zdjizhi</groupId>
<artifactId>sip-rtp-correlation</artifactId>
<version>1.0-rc1</version>
<version>1.2.2</version>
<name>Flink : SIP-RTP : Correlation</name>
@@ -24,6 +24,18 @@
<jackson.version>2.13.2.20220328</jackson.version>
</properties>
<distributionManagement>
<repository>
<id>platform-releases</id>
<url>http://192.168.40.153:8099/content/repositories/platform-release</url>
<uniqueVersion>true</uniqueVersion>
</repository>
<snapshotRepository>
<id>platform-snapshots</id>
<url>http://192.168.40.153:8099/content/repositories/platform-snapshot</url>
</snapshotRepository>
</distributionManagement>
<dependencies>
<dependency>
<groupId>org.apache.flink</groupId>
@@ -285,6 +297,20 @@
</executions>
</plugin>
<plugin>
<groupId>io.github.zlika</groupId>
<artifactId>reproducible-build-maven-plugin</artifactId>
<version>0.2</version>
<executions>
<execution>
<goals>
<goal>strip-jar</goal>
</goals>
<phase>package</phase>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-shade-plugin</artifactId>

View File

@@ -4,8 +4,8 @@ import com.zdjizhi.flink.voip.conf.FusionConfiguration;
import com.zdjizhi.flink.voip.error.ErrorHandler;
import com.zdjizhi.flink.voip.formats.JsonNodeSerializationSchema;
import com.zdjizhi.flink.voip.functions.*;
import com.zdjizhi.flink.voip.records.Record;
import com.zdjizhi.flink.voip.records.SIPRecord;
import org.apache.flink.api.common.eventtime.SerializableTimestampAssigner;
import org.apache.flink.api.common.eventtime.WatermarkStrategy;
import org.apache.flink.api.java.functions.KeySelector;
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.api.java.utils.ParameterTool;
@@ -13,12 +13,13 @@ import org.apache.flink.configuration.Configuration;
import org.apache.flink.formats.json.JsonNodeDeserializationSchema;
import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.node.ObjectNode;
import org.apache.flink.streaming.api.datastream.DataStream;
import org.apache.flink.streaming.api.datastream.DataStreamSource;
import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.streaming.connectors.kafka.FlinkKafkaConsumer;
import org.apache.flink.streaming.connectors.kafka.FlinkKafkaProducer;
import java.time.Duration;
import static com.zdjizhi.flink.voip.conf.FusionConfigs.*;
/**
@@ -52,7 +53,13 @@ public class CorrelateApp {
fusionConfiguration
.getProperties(SOURCE_KAFKA_PROPERTIES_PREFIX));
final DataStreamSource<ObjectNode> sourceStream = env.addSource(kafkaConsumer);
final DataStream<ObjectNode> sourceStream = env.addSource(kafkaConsumer)
.assignTimestampsAndWatermarks(
WatermarkStrategy
.<ObjectNode>forBoundedOutOfOrderness(Duration.ofSeconds(5))
.withTimestampAssigner((SerializableTimestampAssigner<ObjectNode>)
(element, recordTimestamp) ->
element.get("start_timestamp_ms").asLong()));
final ErrorHandler errorHandler = new ErrorHandler(config);
@@ -92,8 +99,10 @@ public class CorrelateApp {
new JsonNodeSerializationSchema(),
fusionConfiguration.getProperties(SINK_KAFKA_PROPERTIES_PREFIX));
voIpOperator.addSink(producer);
voIpOperator
.union(sipDoubleDirOperator.getSideOutput(SIPPairingFunction.SIP_OUTPUT_TAG))
.addSink(producer);
env.execute("VoIP Fusion Job");
env.execute(config.get(JOB_NAME));
}
}

View File

@@ -56,6 +56,16 @@ public class FusionConfigs {
.withDescription("Enable or disable the output of error records. " +
"If set to true, the error records will be sent to the specified Kafka topic.");
/**
* Configuration option to determine whether to perform data correlate for intranet addresses.
*/
public static final ConfigOption<Boolean> INCLUDE_INTRANET_IP =
ConfigOptions.key("include.intranet.ip")
.booleanType()
.defaultValue(true)
.withDescription("Whether to perform data correlate for intranet addresses");
/**
* Configuration option for specifying the Kafka topic name where the error data will be sent.
* This configuration option is used when the output of error records is enabled.
@@ -85,4 +95,13 @@ public class FusionConfigs {
.intType()
.defaultValue(6)
.withDescription("The interval at which RTP state data should be cleared.");
/**
* Configuration option for specifying the name of a job.
*/
public static final ConfigOption<String> JOB_NAME =
ConfigOptions.key("job.name")
.stringType()
.defaultValue("correlation_sip_rtp_session")
.withDescription("The name of current job.");
}

View File

@@ -3,12 +3,12 @@ package com.zdjizhi.flink.voip.error;
import com.zdjizhi.flink.voip.conf.FusionConfigs;
import com.zdjizhi.flink.voip.conf.FusionConfiguration;
import com.zdjizhi.flink.voip.formats.JsonNodeSerializationSchema;
import com.zdjizhi.flink.voip.functions.FunctionHelper;
import com.zdjizhi.flink.voip.records.Record;
import com.zdjizhi.flink.voip.records.SIPRecord;
import com.zdjizhi.flink.voip.records.SchemaType;
import com.zdjizhi.flink.voip.records.StreamDir;
import com.zdjizhi.utils.IPUtil;
import org.apache.commons.lang3.StringUtils;
import org.apache.flink.api.common.typeinfo.TypeInformation;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.node.ObjectNode;
@@ -21,6 +21,8 @@ import org.apache.flink.util.OutputTag;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.function.Function;
/**
* The ErrorHandler class is responsible for handling and filtering error records from the input data stream.
* It checks for records that have invalid or meaningless addresses and ports, and outputs them to a separate stream if enabled.
@@ -88,35 +90,50 @@ public class ErrorHandler {
* The MeaninglessAddressProcessFunction is a custom ProcessFunction used to check for records
* with invalid or meaningless addresses and ports. It separates them into the invalid output tag if necessary.
*/
class MeaninglessAddressProcessFunction extends ProcessFunction<ObjectNode, ObjectNode> {
class MeaninglessAddressProcessFunction extends ProcessFunction<ObjectNode, ObjectNode> implements FunctionHelper {
private static final Logger LOG = LoggerFactory.getLogger(MeaninglessAddressProcessFunction.class);
private transient boolean includeIntranetIp;
@Override
public void open(Configuration parameters) throws Exception {
super.open(parameters);
final Configuration config = getGlobalConfiguration();
includeIntranetIp = config.get(FusionConfigs.INCLUDE_INTRANET_IP);
}
@Override
public void processElement(ObjectNode obj,
ProcessFunction<ObjectNode, ObjectNode>.Context ctx,
Collector<ObjectNode> out) throws Exception {
final Record record = new Record(obj);
// Check for invalid or meaningless addresses and ports
boolean cond1 = StringUtils.isNotBlank(record.getClientIp()) &&
StringUtils.isNotBlank(record.getServerIp()) &&
record.getClientPort() >= 0 &&
record.getServerPort() >= 0;
boolean cond1 = isIPAddress(record.getClientIp()) &&
isIPAddress(record.getServerIp()) &&
record.getClientPort() > 0 &&
record.getServerPort() > 0;
boolean cond8 = null != executeSafely(Record::getStreamDir, record);
final SIPRecord sipRecord = new SIPRecord(obj);
boolean cond2 = !IPUtil.isIPAddress(sipRecord.getOriginatorSdpConnectIp())
|| IPUtil.internalIp(sipRecord.getOriginatorSdpConnectIp());
boolean cond3 = !IPUtil.isIPAddress(sipRecord.getResponderSdpConnectIp())
|| IPUtil.internalIp(sipRecord.getResponderSdpConnectIp());
boolean cond4 = IPUtil.isIPAddress(sipRecord.getOriginatorSdpConnectIp())
|| IPUtil.isIPAddress(sipRecord.getResponderSdpConnectIp());
boolean cond2 = isIPAddress(sipRecord.getOriginatorSdpConnectIp())
|| isIPAddress(sipRecord.getResponderSdpConnectIp());
boolean cond3 = !isIPAddress(sipRecord.getResponderSdpConnectIp())
|| (includeIntranetIp || !isInternalIp(sipRecord.getResponderSdpConnectIp()));
boolean cond4 = !isIPAddress(sipRecord.getOriginatorSdpConnectIp())
|| (includeIntranetIp || !isInternalIp(sipRecord.getOriginatorSdpConnectIp()));
boolean cond5 = SchemaType.SIP.equals(sipRecord.getSchemaType());
boolean cond6 = StreamDir.DOUBLE == sipRecord.getStreamDir() &&
IPUtil.internalIp(sipRecord.getResponderSdpConnectIp()) &&
IPUtil.internalIp(sipRecord.getOriginatorSdpConnectIp());
boolean cond6 = StreamDir.DOUBLE == executeSafely(Record::getStreamDir, sipRecord) &&
(includeIntranetIp || !isInternalIp(sipRecord.getResponderSdpConnectIp())) &&
(includeIntranetIp || !isInternalIp(sipRecord.getOriginatorSdpConnectIp()));
boolean cond7 = isIPAddress(sipRecord.getOriginatorSdpConnectIp()) &&
isIPAddress(sipRecord.getResponderSdpConnectIp()) &&
sipRecord.getResponderSdpMediaPort() > 0 && sipRecord.getOriginatorSdpMediaPort() > 0;
// Both client and server addresses in the data are valid.
if (cond1 && (
if (cond1 && cond8 && (!cond5 || cond7) && (
// The address in the SIP one-way stream is valid and not an internal network address.
cond2 && cond3 && cond4 && cond5
// The coordinating addresses in the SIP double directional stream are valid
@@ -132,4 +149,29 @@ class MeaninglessAddressProcessFunction extends ProcessFunction<ObjectNode, Obje
}
}
}
// ======================================================================================
// ----------------------------------- private helper -----------------------------------
public static <T, R> R executeSafely(Function<T, R> function, T v) {
try {
return function.apply(v);
} catch (Exception e) {
return null;
}
}
private static boolean isIPAddress(final String ipaddr) {
if (null == ipaddr) {
return false;
}
return IPUtil.isIPAddress(ipaddr);
}
private static boolean isInternalIp(final String ipaddr) {
if (!isIPAddress(ipaddr)) {
return false;
}
return IPUtil.internalIp(ipaddr);
}
}

View File

@@ -26,7 +26,7 @@ public interface FunctionHelper extends RichFunction {
}
default void registerNextFireTimestamp(TimerService timerService, long interval) {
long current = timerService.currentProcessingTime();
timerService.registerProcessingTimeTimer(current + interval);
long current = timerService.currentWatermark();
timerService.registerEventTimeTimer(current + interval);
}
}

View File

@@ -7,11 +7,13 @@ import org.apache.flink.api.common.state.StateTtlConfig;
import org.apache.flink.api.common.state.ValueState;
import org.apache.flink.api.common.state.ValueStateDescriptor;
import org.apache.flink.api.common.time.Time;
import org.apache.flink.api.common.typeinfo.TypeInformation;
import org.apache.flink.api.java.tuple.Tuple3;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.node.ObjectNode;
import org.apache.flink.streaming.api.functions.KeyedProcessFunction;
import org.apache.flink.util.Collector;
import org.apache.flink.util.OutputTag;
/**
* A KeyedProcessFunction that pairs SIP records based on their addresses and stream direction.
@@ -23,6 +25,9 @@ import org.apache.flink.util.Collector;
public class SIPPairingFunction extends KeyedProcessFunction<Tuple3<Integer, String, Address>, ObjectNode, ObjectNode>
implements FunctionHelper {
public static final OutputTag<ObjectNode> SIP_OUTPUT_TAG =
new OutputTag<>("unmatched-sip", TypeInformation.of(ObjectNode.class));
private transient Time fireInterval;
private transient ValueState<ObjectNode> valueState;
@@ -63,7 +68,7 @@ public class SIPPairingFunction extends KeyedProcessFunction<Tuple3<Integer, Str
out.collect(value);
valueState.clear();
} else {
// If the address is not yet in the mapState.
// If the address is not yet in the valueState.
valueState.update(value);
}
} else {
@@ -77,6 +82,10 @@ public class SIPPairingFunction extends KeyedProcessFunction<Tuple3<Integer, Str
public void onTimer(long timestamp,
KeyedProcessFunction<Tuple3<Integer, String, Address>, ObjectNode, ObjectNode>.OnTimerContext ctx,
Collector<ObjectNode> out) throws Exception {
final ObjectNode value = valueState.value();
if (value != null) {
ctx.output(SIP_OUTPUT_TAG, value);
}
valueState.clear();
}
}

View File

@@ -1,9 +1,8 @@
package com.zdjizhi.flink.voip.functions;
import com.zdjizhi.flink.voip.conf.FusionConfigs;
import com.zdjizhi.flink.voip.records.Record;
import com.zdjizhi.flink.voip.records.SchemaType;
import com.zdjizhi.flink.voip.records.StreamDir;
import com.zdjizhi.flink.voip.records.*;
import org.apache.commons.lang3.StringUtils;
import org.apache.flink.api.common.functions.RuntimeContext;
import org.apache.flink.api.common.state.*;
import org.apache.flink.api.common.time.Time;
@@ -80,9 +79,12 @@ public class VoIPFusionFunction extends KeyedCoProcessFunction<Tuple2<Integer, A
final ObjectNode rtpObj = entry.getValue();
final Record rtpRecord = new Record(rtpObj);
completeOriginatorField(rtpRecord, new SIPRecord(sipObj));
rtpRecord.merge(sipObj)
.setString(Record.F_COMMON_SCHEMA_TYPE, SchemaType.VOIP.getValue());
out.collect(rtpObj);
iterator.remove();
switch (entry.getKey()) {
case S2C:
@@ -120,6 +122,8 @@ public class VoIPFusionFunction extends KeyedCoProcessFunction<Tuple2<Integer, A
final StreamDir streamDir = rtpRecord.getStreamDir();
if (null != info) {
completeOriginatorField(rtpRecord, new SIPRecord(info.getObj()));
rtpRecord.merge(info.getObj())
.setString(Record.F_COMMON_SCHEMA_TYPE, SchemaType.VOIP.getValue());
out.collect(rtpObj);
@@ -149,9 +153,28 @@ public class VoIPFusionFunction extends KeyedCoProcessFunction<Tuple2<Integer, A
KeyedCoProcessFunction<Tuple2<Integer, Address>, ObjectNode, ObjectNode, ObjectNode>.OnTimerContext ctx,
Collector<ObjectNode> out) throws Exception {
for (ObjectNode obj : rtpState.values()) {
final Record rtpRecord = new Record(obj);
rtpRecord.setInt(RTPRecord.F_ORIGINATOR_DIR, RTPRecord.OriginatorDir.UNKNOWN.getCode());
out.collect(obj);
}
rtpState.clear();
sipState.clear();
}
// ======================================================================
// PRIVATE HELPER
// ======================================================================
private void completeOriginatorField(final Record rtpRecord, final SIPRecord sipRecord) {
if (StringUtils.isNoneBlank(rtpRecord.getClientIp())) {
if (StringUtils.equals(sipRecord.getOriginatorSdpConnectIp(), rtpRecord.getClientIp())) {
rtpRecord.setInt(RTPRecord.F_ORIGINATOR_DIR, RTPRecord.OriginatorDir.C2S.getCode());
return;
} else if (StringUtils.equals(sipRecord.getResponderSdpConnectIp(), rtpRecord.getClientIp())) {
rtpRecord.setInt(RTPRecord.F_ORIGINATOR_DIR, RTPRecord.OriginatorDir.S2C.getCode());
return;
}
}
rtpRecord.setInt(RTPRecord.F_ORIGINATOR_DIR, RTPRecord.OriginatorDir.UNKNOWN.getCode());
}
}

View File

@@ -0,0 +1,27 @@
package com.zdjizhi.flink.voip.records;
import lombok.Getter;
import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.node.ObjectNode;
public class RTPRecord extends Record {
public static final String F_ORIGINATOR_DIR = "rtp_originator_dir";
public RTPRecord(ObjectNode obj) {
super(obj);
}
@Getter
public enum OriginatorDir {
UNKNOWN(0),
C2S(1),
S2C(2);
private final int code;
OriginatorDir(int code) {
this.code = code;
}
}
}

View File

@@ -20,31 +20,35 @@ public class Record {
/**
* 字段名:数据记录中的所属 vsys
*/
public static final String F_COMMON_VSYS_ID = "common_vsys_id";
public static final String F_COMMON_VSYS_ID = "vsys_id";
/**
* 字段名:数据记录中的字段类型
*/
public static final String F_COMMON_SCHEMA_TYPE = "common_schema_type";
public static final String F_COMMON_SCHEMA_TYPE = "decoded_as";
/**
* 字段名:数据记录中的流类型
*/
public static final String F_COMMON_STREAM_DIR = "common_stream_dir";
/**
* 字段名:数据记录中的流类型的 Flags
*/
public static final String F_FLAGS = "flags";
/**
* 字段名:数据记录中的服务端地址
*/
public static final String F_COMMON_SERVER_IP = "common_server_ip";
public static final String F_COMMON_SERVER_IP = "server_ip";
/**
* 字段名:数据记录中的服务端端口
*/
public static final String F_COMMON_SERVER_PORT = "common_server_port";
public static final String F_COMMON_SERVER_PORT = "server_port";
/**
* 字段名:数据记录中的客户端地址
*/
public static final String F_COMMON_CLIENT_IP = "common_client_ip";
public static final String F_COMMON_CLIENT_IP = "client_ip";
/**
* 字段名:数据记录中的客户端端口
*/
public static final String F_COMMON_CLIENT_PORT = "common_client_port";
public static final String F_COMMON_CLIENT_PORT = "client_port";
/**
* ObjectNode data.
@@ -57,7 +61,8 @@ public class Record {
* @return The VSys ID as an integer.
*/
public int getVSysID() {
return Record.getInt(obj, F_COMMON_VSYS_ID);
int v = Record.getInt(obj, F_COMMON_VSYS_ID);
return v == 0 ? 1 : v;
}
/**
@@ -75,7 +80,7 @@ public class Record {
* @return The stream direction.
*/
public final StreamDir getStreamDir() {
return StreamDir.of(Record.getInt(obj, F_COMMON_STREAM_DIR));
return StreamDir.ofFlags(Record.getLong(obj, F_FLAGS));
}
/**
@@ -169,6 +174,30 @@ public class Record {
return getInt(obj, field, 0);
}
/**
* Gets a long value from the specified field in the ObjectNode.
*
* @param obj The ObjectNode to get the value from.
* @param field The name of the field.
* @param defaultValue The default value to return if the field is not found or is not a long.
* @return The long value from the field or the default value if the field is not found or is not a long.
*/
public static long getLong(final ObjectNode obj, final String field, final long defaultValue) {
final JsonNode node = obj.get(field);
return node != null && node.isNumber() ? node.asLong() : defaultValue;
}
/**
* Gets a long value from the specified field in the ObjectNode.
*
* @param obj The ObjectNode to get the value from.
* @param field The name of the field.
* @return The long value from the field or 0 if the field is not found or is not a long.
*/
private static long getLong(final ObjectNode obj, final String field) {
return getLong(obj, field, 0L);
}
/**
* Get a string value from the specified field in the ObjectNode.
*

View File

@@ -48,4 +48,22 @@ public enum StreamDir {
}
throw new IllegalArgumentException("Unknown StreamDir value '" + value + "'.");
}
/**
* Get the StreamDir enum based on the provided flags value.
*
* @param flags The flags.
* @return The corresponding StreamDir enum.
* @throws IllegalArgumentException if the provided value does not match any known StreamDir.
*/
public static StreamDir ofFlags(long flags) {
int v = 0;
if ((flags & 8192) == 8192) {
v += 1;
}
if ((flags & 16384) == 16384) {
v += 2;
}
return of(v);
}
}

View File

@@ -34,10 +34,10 @@ public class RecordTest {
final ObjectNode obj = mapper.createObjectNode();
final Record record = new Record(obj);
record.setString(Record.F_COMMON_SCHEMA_TYPE, SchemaType.RTP.getValue());
assertEquals(SchemaType.RTP.getValue(), record.getSchemaType());
assertEquals(SchemaType.RTP, record.getSchemaType());
obj.set(Record.F_COMMON_SCHEMA_TYPE, TextNode.valueOf(SchemaType.VOIP.getValue()));
assertEquals(SchemaType.VOIP.getValue(), record.getSchemaType());
assertEquals(SchemaType.VOIP, record.getSchemaType());
}
@Test