Merge branch 'hotfix/field-rename' into 'master'

[TSG-23852] feat: adapt to field renaming

See merge request galaxy/tsg_olap/sip-rtp-correlation!41

.gitlab-ci.yml

@@ -1,4 +1,4 @@
-image: 192.168.40.153:9080/common/maven:3.8.1-openjdk-11-slim-with-git
+image: 192.168.40.153:8082/common/maven:3.8.1-openjdk-11-slim
 
 variables:
   MAVEN_CLI_OPTS: "--batch-mode --errors --show-version"
@@ -8,16 +8,10 @@ stages:
   - test
   - build
 
-snapshot-version:
-  stage: check
-  script:
-    - mvn $MAVEN_CLI_OPTS enforcer:enforce@snapshot-version-check
-  rules:
-    - if: $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "develop" && $CI_PIPELINE_SOURCE == "merge_request_event"
-
-non-snapshot-version:
+release-version-check:
   stage: check
   script:
+    - echo "$MAVEN_SETTINGS_XML" > /usr/share/maven/conf/settings.xml
     - mvn $MAVEN_CLI_OPTS enforcer:enforce@release-version-check
     - |-
       if `mvn $MAVEN_CLI_OPTS dependency:get@release-deploy-check > /dev/null 2>&1`; then
@@ -27,21 +21,20 @@ non-snapshot-version:
         echo "The current version has not been deployed."
       fi
   rules:
-    - if: $CI_MERGE_REQUEST_TARGET_BRANCH_NAME =~ /(^master$|^release\/)/ && $CI_PIPELINE_SOURCE == "merge_request_event"
+    - if: $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "master" && $CI_PIPELINE_SOURCE == "merge_request_event"
 
 test:
   stage: test
   script:
+    - echo "$MAVEN_SETTINGS_XML" > /usr/share/maven/conf/settings.xml
     - mvn $MAVEN_CLI_OPTS clean test
   only:
     - merge_requests
 
-# Used for building snapshot versions on the develop branch.
 build:
   stage: build
   script:
     - echo "$MAVEN_SETTINGS_XML" > /usr/share/maven/conf/settings.xml
     - mvn clean site deploy -DskipTests
   only:
-    - master
-    - /^release\//
+    - master

pom.xml

@@ -7,7 +7,7 @@
 
     <groupId>com.geedgenetworks.application</groupId>
     <artifactId>sip-rtp-correlation</artifactId>
-    <version>2.0-SNAPSHOT</version>
+    <version>2.2.0</version>
 
     <name>Flink : SIP-RTP : Correlation</name>
 
@@ -19,7 +19,7 @@
         <scala.version>2.12.10</scala.version>
         <scala.binary.version>2.12</scala.binary.version>
         <flink.version>1.13.6</flink.version>
-        <easy.stream.version>1.3-SNAPSHOT</easy.stream.version>
+        <easy.stream.version>1.3-rc2</easy.stream.version>
         <slf4j.version>1.7.32</slf4j.version>
         <log4j.version>2.17.1</log4j.version>
         <junit.version>5.8.0</junit.version>
@@ -28,35 +28,21 @@
     <distributionManagement>
         <repository>
             <id>platform-releases</id>
-            <url>http://192.168.40.153:8099/content/repositories/platform-release</url>
+            <url>http://192.168.40.153:8081/content/repositories/platform-release</url>
             <uniqueVersion>true</uniqueVersion>
         </repository>
         <snapshotRepository>
             <id>platform-snapshots</id>
-            <url>http://192.168.40.153:8099/content/repositories/platform-snapshot</url>
+            <url>http://192.168.40.153:8081/content/repositories/platform-snapshot</url>
         </snapshotRepository>
         <site>
             <id>platform-site</id>
             <url>
-                dav:http://192.168.40.153:8099/content/sites/platform-site/platform/application/sip-rtp-correlate-${project.version}
+                dav:http://192.168.40.153:8081/content/sites/platform-site/platform/application/sip-rtp-correlate-${project.version}
             </url>
         </site>
     </distributionManagement>
 
-    <repositories>
-        <repository>
-            <id>central</id>
-            <url>http://192.168.40.153:8099/content/groups/public</url>
-        </repository>
-        <repository>
-            <id>snapshots</id>
-            <url>http://192.168.40.153:8099/content/groups/public</url>
-            <snapshots>
-                <enabled>true</enabled>
-            </snapshots>
-        </repository>
-    </repositories>
-
     <dependencies>
         <dependency>
             <groupId>org.slf4j</groupId>
@@ -76,8 +62,8 @@
         </dependency>
 
         <dependency>
-            <groupId>com.zdjizhi</groupId>
-            <artifactId>galaxy</artifactId>
+            <groupId>cn.hutool</groupId>
+            <artifactId>hutool-all</artifactId>
         </dependency>
         <dependency>
             <groupId>xyz.downgoon</groupId>
@@ -253,23 +239,9 @@
 
             <!--  Common  -->
             <dependency>
-                <groupId>com.zdjizhi</groupId>
-                <artifactId>galaxy</artifactId>
-                <version>1.1.3</version>
-                <exclusions>
-                    <exclusion>
-                        <groupId>org.slf4j</groupId>
-                        <artifactId>slf4j-log4j12</artifactId>
-                    </exclusion>
-                    <exclusion>
-                        <groupId>log4j</groupId>
-                        <artifactId>log4j</artifactId>
-                    </exclusion>
-                    <exclusion>
-                        <groupId>org.apache.commons</groupId>
-                        <artifactId>commons-lang3</artifactId>
-                    </exclusion>
-                </exclusions>
+                <groupId>cn.hutool</groupId>
+                <artifactId>hutool-all</artifactId>
+                <version>5.8.32</version>
             </dependency>
 
             <!-- Easy Stream-->
@@ -337,13 +309,13 @@
                 <groupId>org.apache.flink</groupId>
                 <artifactId>flink-runtime_${scala.binary.version}</artifactId>
                 <version>${flink.version}</version>
-                <scope>runtime</scope>
+                <scope>provided</scope>
             </dependency>
             <dependency>
                 <groupId>org.apache.flink</groupId>
                 <artifactId>flink-runtime-web_${scala.binary.version}</artifactId>
                 <version>${flink.version}</version>
-                <scope>runtime</scope>
+                <scope>provided</scope>
             </dependency>
 
             <dependency>
@@ -587,6 +559,27 @@
 
         <pluginManagement>
             <plugins>
+                <!-- CI plugins -->
+                <plugin>
+                    <groupId>org.apache.maven.plugins</groupId>
+                    <artifactId>maven-dependency-plugin</artifactId>
+                    <version>3.1.2</version>
+                    <executions>
+                        <execution>
+                            <id>release-deploy-check</id>
+                            <goals>
+                                <goal>get</goal>
+                            </goals>
+                            <configuration>
+                                <groupId>${project.groupId}</groupId>
+                                <artifactId>easy-stream-common</artifactId>
+                                <version>${project.version}</version>
+                                <remoteRepositories>${project.distributionManagement.repository.url}
+                                </remoteRepositories>
+                            </configuration>
+                        </execution>
+                    </executions>
+                </plugin>
                 <plugin>
                     <groupId>org.apache.maven.plugins</groupId>
                     <artifactId>maven-enforcer-plugin</artifactId>

src/main/java/com/geedgenetworks/flink/easy/application/voip/VoipUDFFactory.java

@@ -20,7 +20,9 @@ public class VoipUDFFactory implements UDFFactory {
                 put("HAS_EXTERNAL_IP_ADDRESS", new HasExternalIpAddress());
 
                 put("STREAM_DIR", new StreamDir());
+                put("STREAM_DIR_SET", new StreamDirSet());
                 put("FIND_NOT_BLANK", new FindNotBlank());
+                put("DISTINCT_CONCAT", new DistinctConcat());
                 put("SORT_ADDRESS", new SortAddress());
 
                 put("SNOWFLAKE_ID", new SnowflakeID());

src/main/java/com/geedgenetworks/flink/easy/application/voip/udf/DistinctConcat.java

@@ -0,0 +1,18 @@
+package com.geedgenetworks.flink.easy.application.voip.udf;
+
+import org.apache.commons.lang3.StringUtils;
+import org.apache.flink.table.annotation.DataTypeHint;
+import org.apache.flink.table.functions.ScalarFunction;
+
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+public class DistinctConcat extends ScalarFunction {
+
+    public @DataTypeHint("STRING") String eval(String s1, String s2) {
+        return Stream.of(s1, s2).filter(StringUtils::isNotBlank)
+                .map(StringUtils::trim)
+                .distinct()
+                .collect(Collectors.joining(","));
+    }
+}

src/main/java/com/geedgenetworks/flink/easy/application/voip/udf/HasIpAddress.java

@@ -1,17 +1,17 @@
 package com.geedgenetworks.flink.easy.application.voip.udf;
 
-import com.zdjizhi.utils.IPUtil;
 import org.apache.flink.table.annotation.DataTypeHint;
-import org.apache.flink.table.functions.ScalarFunction;
 
-public class HasIpAddress extends ScalarFunction {
+public class HasIpAddress extends IpAddressScalarFunction {
 
     public @DataTypeHint("BOOLEAN") Boolean eval(String... ipaddr) {
         if (null == ipaddr) {
             return false;
         }
         for (var ip : ipaddr) {
-            return ip != null && IPUtil.isIPAddress(ip);
+            if (ip != null && isIpAddress(ip)) {
+                return true;
+            }
         }
         return false;
     }

src/main/java/com/geedgenetworks/flink/easy/application/voip/udf/IpAddressScalarFunction.java

@@ -0,0 +1,25 @@
+package com.geedgenetworks.flink.easy.application.voip.udf;
+
+import cn.hutool.core.lang.Validator;
+import cn.hutool.core.net.NetUtil;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.flink.table.functions.ScalarFunction;
+
+public abstract class IpAddressScalarFunction extends ScalarFunction {
+
+    protected static boolean isIpAddress(String ipaddr) {
+        return isIpv4(ipaddr) || isIpv6(ipaddr);
+    }
+
+    protected static boolean isIpv4(String ipaddr) {
+        return Validator.isIpv4(StringUtils.trim(ipaddr));
+    }
+
+    protected static boolean isIpv6(String ipaddr) {
+        return Validator.isIpv6(StringUtils.trim(ipaddr));
+    }
+
+    protected static boolean isInternalIpAddress(String ipaddr) {
+        return NetUtil.isInnerIP(StringUtils.trim(ipaddr));
+    }
+}

src/main/java/com/geedgenetworks/flink/easy/application/voip/udf/IsExternalIpAddress.java

@@ -1,17 +1,13 @@
 package com.geedgenetworks.flink.easy.application.voip.udf;
 
-import com.zdjizhi.utils.IPUtil;
 import org.apache.flink.table.annotation.DataTypeHint;
-import org.apache.flink.table.functions.ScalarFunction;
 
-import static com.zdjizhi.utils.IPUtil.isIPAddress;
-
-public class IsExternalIpAddress extends ScalarFunction {
+public class IsExternalIpAddress extends IpAddressScalarFunction {
 
     public @DataTypeHint("BOOLEAN") Boolean eval(String ipaddr) {
-        if (ipaddr == null || !isIPAddress(ipaddr)) {
+        if (ipaddr == null || !isIpAddress(ipaddr)) {
             return false;
         }
-        return !IPUtil.internalIp(ipaddr);
+        return !isInternalIpAddress(ipaddr);
     }
 }

src/main/java/com/geedgenetworks/flink/easy/application/voip/udf/IsInternalIpAddress.java

@@ -1,17 +1,13 @@
 package com.geedgenetworks.flink.easy.application.voip.udf;
 
-import com.zdjizhi.utils.IPUtil;
 import org.apache.flink.table.annotation.DataTypeHint;
-import org.apache.flink.table.functions.ScalarFunction;
 
-import static com.zdjizhi.utils.IPUtil.isIPAddress;
-
-public class IsInternalIpAddress extends ScalarFunction {
+public class IsInternalIpAddress extends IpAddressScalarFunction {
 
     public @DataTypeHint("BOOLEAN") Boolean eval(String ipaddr) {
-        if (!isIPAddress(ipaddr)) {
+        if (!isIpAddress(ipaddr)) {
             return false;
         }
-        return IPUtil.internalIp(ipaddr);
+        return isInternalIpAddress(ipaddr);
     }
 }

src/main/java/com/geedgenetworks/flink/easy/application/voip/udf/IsIpAddress.java

@@ -1,15 +1,13 @@
 package com.geedgenetworks.flink.easy.application.voip.udf;
 
-import com.zdjizhi.utils.IPUtil;
 import org.apache.flink.table.annotation.DataTypeHint;
-import org.apache.flink.table.functions.ScalarFunction;
 
-public class IsIpAddress extends ScalarFunction {
+public class IsIpAddress extends IpAddressScalarFunction {
 
     public @DataTypeHint("BOOLEAN") Boolean eval(String ipaddr) {
         if (null == ipaddr) {
             return false;
         }
-        return IPUtil.isIPAddress(ipaddr);
+        return isIpAddress(ipaddr);
     }
 }

src/main/java/com/geedgenetworks/flink/easy/application/voip/udf/SortAddress.java

@@ -1,12 +1,14 @@
 package com.geedgenetworks.flink.easy.application.voip.udf;
 
-import com.google.common.collect.Lists;
-import com.zdjizhi.utils.IPUtil;
+import cn.hutool.core.net.NetUtil;
+import org.apache.commons.lang3.StringUtils;
 import org.apache.flink.api.java.tuple.Tuple2;
 import org.apache.flink.table.annotation.DataTypeHint;
-import org.apache.flink.table.functions.ScalarFunction;
 
-public class SortAddress extends ScalarFunction {
+import java.util.ArrayList;
+import java.util.List;
+
+public class SortAddress extends IpAddressScalarFunction {
 
     public @DataTypeHint("STRING")
     String eval(
@@ -16,11 +18,14 @@ public class SortAddress extends ScalarFunction {
 
     public static String of(
             Tuple2<String, Integer> a1, Tuple2<String, Integer> a2) {
-        var list = Lists.newArrayList(a1, a2);
+        var list = new ArrayList<>(List.of(a1, a2));
+        if (a1.f1 == null || a2.f1 == null || StringUtils.isAnyEmpty(a1.f0, a2.f0)
+                || !isIpAddress(a1.f0) || !isIpAddress(a2.f0)) {
+            return a1.f0 + ":" + a1.f1 + "," + a2.f0 + ":" + a2.f1;
+        }
         list.sort((a, b) -> {
             if (a.f1.equals(b.f1)) {
-                return Long.compare(
-                        IPUtil.getIpDesimal(a.f0), IPUtil.getIpDesimal(b.f0));
+                return compareAddress(a.f0, b.f0);
             } else {
                 return a.f1.compareTo(b.f1);
             }
@@ -28,4 +33,16 @@ public class SortAddress extends ScalarFunction {
         return String.format("%s:%s,%s:%s",
                 list.get(0).f0, list.get(0).f1, list.get(1).f0, list.get(1).f1);
     }
+
+    private static int compareAddress(String ip1, String ip2) {
+        try {
+            var v1 = isIpv4(ip1) ? NetUtil.ipv4ToLong(ip1) :
+                    NetUtil.ipv6ToBigInteger(ip1).longValue();
+            var v2 = isIpv4(ip2) ? NetUtil.ipv4ToLong(ip2) :
+                    NetUtil.ipv6ToBigInteger(ip2).longValue();
+            return Long.compare(v1, v2);
+        } catch (Exception e) {
+            return 0;
+        }
+    }
 }

src/main/java/com/geedgenetworks/flink/easy/application/voip/udf/StreamDir.java

@@ -18,9 +18,4 @@ public class StreamDir extends ScalarFunction {
         }
         return v;
     }
-
-    public static void main(String[] args) {
-        System.out.println(8192L + 16384L);
-        System.out.println(new StreamDir().eval(8192L + 16384L));
-    }
 }

src/main/java/com/geedgenetworks/flink/easy/application/voip/udf/StreamDirSet.java

@@ -0,0 +1,22 @@
+package com.geedgenetworks.flink.easy.application.voip.udf;
+
+import org.apache.flink.table.annotation.DataTypeHint;
+import org.apache.flink.table.functions.ScalarFunction;
+
+public class StreamDirSet extends ScalarFunction {
+
+    public @DataTypeHint("BIGINT") Long eval(Long flags) {
+        if (flags == null) {
+            return 8192 + 16384L;
+        }
+        long r = 0;
+        if ((flags & 8192) == 0) {
+            r += 8192;
+        }
+        if ((flags & 16384) == 0) {
+            r += 16384;
+        }
+        return r;
+    }
+
+}

src/site/markdown/deploy.md

@@ -1,13 +1,11 @@
 ## Deploy
 
 - 准备 JDK ${java.version} 的环境
-
 - 准备 Flink ${flink.version} 的环境
-
 - [下载](./download.html) 对应版本 UDF 依赖 Jar
-
 - [下载](./download.html) 对应版本 Job 配置 (一个 yml 文件)
-
 - 执行命令 `flink run -Dflink.rest.bind-port=8081 -c com.geedgenetworks.flink.easy.core.Runner path/to/sip-rtp-correlation-<version>.jar job.yml`
+- 您将在控制台看到启动日志，同时您可以在 `http://<you-host>:8081` 看到任务 UI。
 
-- 您将在控制台看到启动日志，同时您可以在 `http://<you-host>:8081` 看到任务 UI。
+> 注意：
+> SIP 和 RTP 融合作业强烈建议开启 Checkpoint 机制，否则将会由于丢失数据或重复数据导致业务数据关联错误。 

src/site/markdown/download.md

@@ -1,8 +1,102 @@
 ## Download
 
-### Easy Stream ${project.version}
+### ${project.version}
 
-| UDF Jar                                                                                                                                                                                          | Job             |
-|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| --------------- |
-| [JAR](${project.distributionManagement.repository.url}/com/geedgenetworks/application/sip-rtp-correlation/${project.version}/${project.artifactId}-${project.version}.jar) ( [MD5](${project.distributionManagement.repository.url}/com/geedgenetworks/application/sip-rtp-correlation/${project.version}/${project.artifactId}-${project.version}.jar.md5) [SHA1](${project.distributionManagement.repository.url}/com/geedgenetworks/application/sip-rtp-correlation/${project.version}/${project.artifactId}-${project.version}.jar.sha1) ) | [YML](${project.distributionManagement.repository.url}/com/geedgenetworks/application/sip-rtp-correlation/${project.version}/${project.artifactId}-${project.version}.tar.gz) ( [MD5](${project.distributionManagement.repository.url}/com/geedgenetworks/application/sip-rtp-correlation/${project.version}/${project.artifactId}-${project.version}.tar.gz.md5) [SHA1](${project.distributionManagement.repository.url}/com/geedgenetworks/application/sip-rtp-correlation/${project.version}/${project.artifactId}-${project.version}.tar.gz.sha1) ) |
+- [TSG-23852](https://jira.geedge.net/browse/TSG-23852) 适配 VoIP Records 以及 Session Records 中 Client/Server 字段重命名。
+
+| Easy Stream | UDF Jar                                                      | Job                                                          |
+|-------------| ------------------------------------------------------------ | ------------------------------------------------------------ |
+| ${easy.stream.version}         | [JAR](${project.distributionManagement.repository.url}/com/geedgenetworks/application/sip-rtp-correlation/${project.version}/${project.artifactId}-${project.version}.jar) ( [MD5](${project.distributionManagement.repository.url}/com/geedgenetworks/application/sip-rtp-correlation/${project.version}/${project.artifactId}-${project.version}.jar.md5) [SHA1](${project.distributionManagement.repository.url}/com/geedgenetworks/application/sip-rtp-correlation/${project.version}/${project.artifactId}-${project.version}.jar.sha1) ) | [YML](${project.distributionManagement.repository.url}/com/geedgenetworks/application/sip-rtp-correlation/${project.version}/${project.artifactId}-${project.version}.tar.gz) ( [MD5](${project.distributionManagement.repository.url}/com/geedgenetworks/application/sip-rtp-correlation/${project.version}/${project.artifactId}-${project.version}.tar.gz.md5) [SHA1](${project.distributionManagement.repository.url}/com/geedgenetworks/application/sip-rtp-correlation/${project.version}/${project.artifactId}-${project.version}.tar.gz.sha1) ) |
+
+### 2.1.1
+
+- 当 VoIP 融合成功时，RTP 将不再输出；同时如果 SIP 关联成功过 RTP 也将不再输出。
+- VpIP 中的字节数，包数使用 RTP 侧的数据指标，不再累加计算。
+- 将 SIP 五元组，RTP 四元组中所有带有空值的数据判定为异常数据不再参与关联过程。
+
+| Easy Stream | UDF Jar                                                      | Job                                                          |
+|-------------| ------------------------------------------------------------ | ------------------------------------------------------------ |
+| 1.3-rc2         | [JAR](http://192.168.40.153:8081/content/repositories/platform-release/com/geedgenetworks/application/sip-rtp-correlation/2.1.1/sip-rtp-correlation-2.1.1.jar) ( [MD5](http://192.168.40.153:8081/content/repositories/platform-release/com/geedgenetworks/application/sip-rtp-correlation/2.1.1/sip-rtp-correlation-2.1.1.jar.md5) [SHA1](http://192.168.40.153:8081/content/repositories/platform-release/com/geedgenetworks/application/sip-rtp-correlation/2.1.1/sip-rtp-correlation-2.1.1.jar.sha1) ) | [YML](http://192.168.40.153:8081/content/repositories/platform-release/com/geedgenetworks/application/sip-rtp-correlation/2.1.1/sip-rtp-correlation-2.1.1.tar.gz) ( [MD5](http://192.168.40.153:8081/content/repositories/platform-release/com/geedgenetworks/application/sip-rtp-correlation/2.1.1/sip-rtp-correlation-2.1.1.tar.gz.md5) [SHA1](http://192.168.40.153:8081/content/repositories/platform-release/com/geedgenetworks/application/sip-rtp-correlation/2.1.1/sip-rtp-correlation-2.1.1.tar.gz.sha1) ) |
+
+### 2.1
+
+- [TSG-23174](https://jira.geedge.net/browse/TSG-23174) 测试完成，升级为稳定版本。
+- 重复数据不再进行去重操作，未关联的全部输出。
+
+| Easy Stream | UDF Jar                                                      | Job                                                          |
+|-------------| ------------------------------------------------------------ | ------------------------------------------------------------ |
+| 1.3-rc2         | [JAR](http://192.168.40.153:8081/content/repositories/platform-release/com/geedgenetworks/application/sip-rtp-correlation/2.1/sip-rtp-correlation-2.1.jar) ( [MD5](http://192.168.40.153:8081/content/repositories/platform-release/com/geedgenetworks/application/sip-rtp-correlation/2.1/sip-rtp-correlation-2.1.jar.md5) [SHA1](http://192.168.40.153:8081/content/repositories/platform-release/com/geedgenetworks/application/sip-rtp-correlation/2.1/sip-rtp-correlation-2.1.jar.sha1) ) | [YML](http://192.168.40.153:8081/content/repositories/platform-release/com/geedgenetworks/application/sip-rtp-correlation/2.1/sip-rtp-correlation-2.1.tar.gz) ( [MD5](http://192.168.40.153:8081/content/repositories/platform-release/com/geedgenetworks/application/sip-rtp-correlation/2.1/sip-rtp-correlation-2.1.tar.gz.md5) [SHA1](http://192.168.40.153:8081/content/repositories/platform-release/com/geedgenetworks/application/sip-rtp-correlation/2.1/sip-rtp-correlation-2.1.tar.gz.sha1) ) |
+
+### 2.0-rc9
+
+- 修复 sip 双向关联模块的数据丢失问题。
+- [GAL-684](https://jira.geedge.net/browse/GAL-684) 调整融合后的字段映射，增加更加详细的监控指标。
+
+| Easy Stream | UDF Jar                                                      | Job                                                          |
+|-------------| ------------------------------------------------------------ | ------------------------------------------------------------ |
+| 1.3-rc1         | [JAR](${project.distributionManagement.repository.url}/com/geedgenetworks/application/sip-rtp-correlation/2.0-rc9/${project.artifactId}-2.0-rc9.jar) ( [MD5](${project.distributionManagement.repository.url}/com/geedgenetworks/application/sip-rtp-correlation/2.0-rc9/${project.artifactId}-2.0-rc9.jar.md5) [SHA1](${project.distributionManagement.repository.url}/com/geedgenetworks/application/sip-rtp-correlation/2.0-rc9/${project.artifactId}-2.0-rc9.jar.sha1) ) | [YML](${project.distributionManagement.repository.url}/com/geedgenetworks/application/sip-rtp-correlation/2.0-rc9/${project.artifactId}-2.0-rc9.tar.gz) ( [MD5](${project.distributionManagement.repository.url}/com/geedgenetworks/application/sip-rtp-correlation/2.0-rc9/${project.artifactId}-2.0-rc9.tar.gz.md5) [SHA1](${project.distributionManagement.repository.url}/com/geedgenetworks/application/sip-rtp-correlation/2.0-rc9/${project.artifactId}-2.0-rc9.tar.gz.sha1) ) |
+
+### 2.0-rc8
+
+- 修复 Extract Key 错误，由于 IpAddress 类型数据校验导致的空指针异常。
+
+| Easy Stream | UDF Jar                                                      | Job                                                          |
+|-------------| ------------------------------------------------------------ | ------------------------------------------------------------ |
+| 1.3-rc1         | [JAR](http://192.168.40.153:8081/content/repositories/platform-release/com/geedgenetworks/application/sip-rtp-correlation/2.0-rc8/sip-rtp-correlation-2.0-rc8.jar) ( [MD5](http://192.168.40.153:8081/content/repositories/platform-release/com/geedgenetworks/application/sip-rtp-correlation/2.0-rc8/sip-rtp-correlation-2.0-rc8.jar.md5) [SHA1](http://192.168.40.153:8081/content/repositories/platform-release/com/geedgenetworks/application/sip-rtp-correlation/2.0-rc8/sip-rtp-correlation-2.0-rc8.jar.sha1) ) | [YML](http://192.168.40.153:8081/content/repositories/platform-release/com/geedgenetworks/application/sip-rtp-correlation/2.0-rc8/sip-rtp-correlation-2.0-rc8.tar.gz) ( [MD5](http://192.168.40.153:8081/content/repositories/platform-release/com/geedgenetworks/application/sip-rtp-correlation/2.0-rc8/sip-rtp-correlation-2.0-rc8.tar.gz.md5) [SHA1](http://192.168.40.153:8081/content/repositories/platform-release/com/geedgenetworks/application/sip-rtp-correlation/2.0-rc8/sip-rtp-correlation-2.0-rc8.tar.gz.sha1) ) |
+
+
+### 2.0-rc7
+
+- VoIP Record 增加字段: SIP `sip_bye_reason` 相关字段。
+
+| Easy Stream | UDF Jar                                                      | Job                                                          |
+|-------------| ------------------------------------------------------------ | ------------------------------------------------------------ |
+| 1.3-rc1         | [JAR](http://192.168.40.153:8081/content/repositories/platform-release/com/geedgenetworks/application/sip-rtp-correlation/2.0-rc7/sip-rtp-correlation-2.0-rc7.jar) ( [MD5](http://192.168.40.153:8081/content/repositories/platform-release/com/geedgenetworks/application/sip-rtp-correlation/2.0-rc7/sip-rtp-correlation-2.0-rc7.jar.md5) [SHA1](http://192.168.40.153:8081/content/repositories/platform-release/com/geedgenetworks/application/sip-rtp-correlation/2.0-rc7/sip-rtp-correlation-2.0-rc7.jar.sha1) ) | [YML](http://192.168.40.153:8081/content/repositories/platform-release/com/geedgenetworks/application/sip-rtp-correlation/2.0-rc7/sip-rtp-correlation-2.0-rc7.tar.gz) ( [MD5](http://192.168.40.153:8081/content/repositories/platform-release/com/geedgenetworks/application/sip-rtp-correlation/2.0-rc7/sip-rtp-correlation-2.0-rc7.tar.gz.md5) [SHA1](http://192.168.40.153:8081/content/repositories/platform-release/com/geedgenetworks/application/sip-rtp-correlation/2.0-rc7/sip-rtp-correlation-2.0-rc7.tar.gz.sha1) ) |
+
+
+### 2.0-rc6
+
+- VoIP Record 增加字段: SIP， RTP `protocol` 相关字段。
+
+| Easy Stream | UDF Jar                                                      | Job                                                          |
+|-------------| ------------------------------------------------------------ | ------------------------------------------------------------ |
+| 1.3-rc1         | [JAR](http://192.168.40.153:8081/content/repositories/platform-release/com/geedgenetworks/application/sip-rtp-correlation/2.0-rc6/sip-rtp-correlation-2.0-rc6.jar) ( [MD5](http://192.168.40.153:8081/content/repositories/platform-release/com/geedgenetworks/application/sip-rtp-correlation/2.0-rc6/sip-rtp-correlation-2.0-rc6.jar.md5) [SHA1](http://192.168.40.153:8081/content/repositories/platform-release/com/geedgenetworks/application/sip-rtp-correlation/2.0-rc6/sip-rtp-correlation-2.0-rc6.jar.sha1) ) | [YML](http://192.168.40.153:8081/content/repositories/platform-release/com/geedgenetworks/application/sip-rtp-correlation/2.0-rc6/sip-rtp-correlation-2.0-rc6.tar.gz) ( [MD5](http://192.168.40.153:8081/content/repositories/platform-release/com/geedgenetworks/application/sip-rtp-correlation/2.0-rc6/sip-rtp-correlation-2.0-rc6.tar.gz.md5) [SHA1](http://192.168.40.153:8081/content/repositories/platform-release/com/geedgenetworks/application/sip-rtp-correlation/2.0-rc6/sip-rtp-correlation-2.0-rc6.tar.gz.sha1) ) |
+
+### 2.0-rc5
+
+- 修复 VoIP 字段错误，使用 `decoded_as` 表示 VoIP 日志。
+
+| Easy Stream | UDF Jar                                                      | Job                                                          |
+|-------------| ------------------------------------------------------------ | ------------------------------------------------------------ |
+| 1.3-rc1     | [JAR](http://192.168.40.153:8081/content/repositories/platform-release/com/geedgenetworks/application/sip-rtp-correlation/2.0-rc5/sip-rtp-correlation-2.0-rc5.jar) ( [MD5](http://192.168.40.153:8081/content/repositories/platform-release/com/geedgenetworks/application/sip-rtp-correlation/2.0-rc5/sip-rtp-correlation-2.0-rc5.jar.md5) [SHA1](http://192.168.40.153:8081/content/repositories/platform-release/com/geedgenetworks/application/sip-rtp-correlation/2.0-rc5/sip-rtp-correlation-2.0-rc5.jar.sha1) ) | [YML](http://192.168.40.153:8081/content/repositories/platform-release/com/geedgenetworks/application/sip-rtp-correlation/2.0-rc5/sip-rtp-correlation-2.0-rc5.tar.gz) ( [MD5](http://192.168.40.153:8081/content/repositories/platform-release/com/geedgenetworks/application/sip-rtp-correlation/2.0-rc5/sip-rtp-correlation-2.0-rc5.tar.gz.md5) [SHA1](http://192.168.40.153:8081/content/repositories/platform-release/com/geedgenetworks/application/sip-rtp-correlation/2.0-rc5/sip-rtp-correlation-2.0-rc5.tar.gz.sha1) ) |
+
+### 2.0-rc4
+
+- 移除对 `log_id`, `recv_time` 字段的强制非空校验, 该字段在后续步骤中处理。
+
+| Easy Stream | UDF Jar                                                      | Job                                                          |
+|-------------| ------------------------------------------------------------ | ------------------------------------------------------------ |
+| 1.3-rc1     | [JAR](http://192.168.40.153:8081/content/repositories/platform-release/com/geedgenetworks/application/sip-rtp-correlation/2.0-rc4/sip-rtp-correlation-2.0-rc4.jar) ( [MD5](http://192.168.40.153:8081/content/repositories/platform-release/com/geedgenetworks/application/sip-rtp-correlation/2.0-rc4/sip-rtp-correlation-2.0-rc4.jar.md5) [SHA1](http://192.168.40.153:8081/content/repositories/platform-release/com/geedgenetworks/application/sip-rtp-correlation/2.0-rc4/sip-rtp-correlation-2.0-rc4.jar.sha1) ) | [YML](http://192.168.40.153:8081/content/repositories/platform-release/com/geedgenetworks/application/sip-rtp-correlation/2.0-rc4/sip-rtp-correlation-2.0-rc4.tar.gz) ( [MD5](http://192.168.40.153:8081/content/repositories/platform-release/com/geedgenetworks/application/sip-rtp-correlation/2.0-rc4/sip-rtp-correlation-2.0-rc4.tar.gz.md5) [SHA1](http://192.168.40.153:8081/content/repositories/platform-release/com/geedgenetworks/application/sip-rtp-correlation/2.0-rc4/sip-rtp-correlation-2.0-rc4.tar.gz.sha1) ) |
+
+### 2.0-rc3
+
+- 修复由于 State 过期策略早于 Timer 触发而导致的未关联成功的 RTP 数据未正常输出的问题。
+
+| Easy Stream | UDF Jar                                                      | Job                                                          |
+|-------------| ------------------------------------------------------------ | ------------------------------------------------------------ |
+| 1.3-rc1     | [JAR](http://192.168.40.153:8081/content/repositories/platform-release/com/geedgenetworks/application/sip-rtp-correlation/2.0-rc3/sip-rtp-correlation-2.0-rc3.jar) ( [MD5](http://192.168.40.153:8081/content/repositories/platform-release/com/geedgenetworks/application/sip-rtp-correlation/2.0-rc3/sip-rtp-correlation-2.0-rc3.jar.md5) [SHA1](http://192.168.40.153:8081/content/repositories/platform-release/com/geedgenetworks/application/sip-rtp-correlation/2.0-rc3/sip-rtp-correlation-2.0-rc3.jar.sha1) ) | [YML](http://192.168.40.153:8081/content/repositories/platform-release/com/geedgenetworks/application/sip-rtp-correlation/2.0-rc3/sip-rtp-correlation-2.0-rc3.tar.gz) ( [MD5](http://192.168.40.153:8081/content/repositories/platform-release/com/geedgenetworks/application/sip-rtp-correlation/2.0-rc3/sip-rtp-correlation-2.0-rc3.tar.gz.md5) [SHA1](http://192.168.40.153:8081/content/repositories/platform-release/com/geedgenetworks/application/sip-rtp-correlation/2.0-rc3/sip-rtp-correlation-2.0-rc3.tar.gz.sha1) ) |
+
+### 2.0-rc2
+
+- 修复由于 Flink 1.13.6 和 Flink 1.13.1 之间某些 Runtime 不兼容的问题导致的系统无法启动。
+
+| Easy Stream | UDF Jar                                                      | Job                                                          |
+|-------------| ------------------------------------------------------------ | ------------------------------------------------------------ |
+| 1.3-rc1     | [JAR](http://192.168.40.153:8099/content/repositories/platform-release/com/geedgenetworks/application/sip-rtp-correlation/2.0-rc2/sip-rtp-correlation-2.0-rc2.jar) ( [MD5](http://192.168.40.153:8099/content/repositories/platform-release/com/geedgenetworks/application/sip-rtp-correlation/2.0-rc2/sip-rtp-correlation-2.0-rc2.jar.md5) [SHA1](http://192.168.40.153:8099/content/repositories/platform-release/com/geedgenetworks/application/sip-rtp-correlation/2.0-rc2/sip-rtp-correlation-2.0-rc2.jar.sha1) ) | [YML](http://192.168.40.153:8099/content/repositories/platform-release/com/geedgenetworks/application/sip-rtp-correlation/2.0-rc2/sip-rtp-correlation-2.0-rc2.tar.gz) ( [MD5](http://192.168.40.153:8099/content/repositories/platform-release/com/geedgenetworks/application/sip-rtp-correlation/2.0-rc2/sip-rtp-correlation-2.0-rc2.tar.gz.md5) [SHA1](http://192.168.40.153:8099/content/repositories/platform-release/com/geedgenetworks/application/sip-rtp-correlation/2.0-rc2/sip-rtp-correlation-2.0-rc2.tar.gz.sha1) ) |
+
+### 2.0-rc1
+
+| Easy Stream | UDF Jar                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                | Job                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                             |
+|-------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| 1.3-rc1     | [JAR](http://192.168.40.153:8099/content/repositories/platform-release/com/geedgenetworks/application/sip-rtp-correlation/2.0-rc1/sip-rtp-correlation-2.0-rc1.jar) ( [MD5](http://192.168.40.153:8099/content/repositories/platform-release/com/geedgenetworks/application/sip-rtp-correlation/2.0-rc1/sip-rtp-correlation-2.0-rc1.jar.md5) [SHA1](http://192.168.40.153:8099/content/repositories/platform-release/com/geedgenetworks/application/sip-rtp-correlation/2.0-rc1/sip-rtp-correlation-2.0-rc1.jar.sha1) ) | [YML](http://192.168.40.153:8099/content/repositories/platform-release/com/geedgenetworks/application/sip-rtp-correlation/2.0-rc1/sip-rtp-correlation-2.0-rc1.tar.gz) ( [MD5](http://192.168.40.153:8099/content/repositories/platform-release/com/geedgenetworks/application/sip-rtp-correlation/2.0-rc1/sip-rtp-correlation-2.0-rc1.tar.gz.md5) [SHA1](http://192.168.40.153:8099/content/repositories/platform-release/com/geedgenetworks/application/sip-rtp-correlation/2.0-rc1/sip-rtp-correlation-2.0-rc1.tar.gz.sha1) ) |
 

src/site/markdown/index.md

@@ -7,4 +7,4 @@ SIP RTP Correlation 项目可以用于实时监控和分析 VoIP 通话数据，
 <br/>
 
 
-You can download the latest release from [Job Yml](./jobs/job.yml). And you can changelog from [CHANGELOG.md](./changelogs.html).
+You can download the latest release from [Downloads](./download.html). And you can changelog from [Changelogs](./changelogs.html).

src/test/bin/flink-per-job.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+
+flink run -t yarn-per-job -Djobmanager.memory.process.size=1024m -Dtaskmanager.memory.process.size=4G -Dtaskmanager.numberOfTaskSlots=1 -Dtaskmanager.memory.framework.off-heap.size=256m -Dtaskmanager.memory.jvm-metaspace.size=256m -Dtaskmanager.memory.network.max=256m -Dyarn.application.name=voip-fusion -Drest.flamegraph.enabled=true -p 1 -d -c com.geedgenetworks.flink.easy.core.Runner /home/tsg/olap/flink/topology/sip-rtp-correlation/sip-rtp-correlation-2.1.jar /home/tsg/olap/flink/topology/sip-rtp-correlation/job.yml

src/test/resources/data/e2e-test-execution.jsonl

@@ -0,0 +1,4 @@
+{"__timestamp":946681200,"__inputid":"tsg_olap","session_id":10240001,"decoded_as":"RTP","ip_protocol":"udp","address_type":4,"client_ip":"192.168.64.8","server_ip":"192.168.39.62","client_port":25524,"server_port":4580,"t_vsys_id":0,"vsys_id":1024,"data_center":"tsg_olap","device_group":"tsg_olap","device_id":"0000000000000000","sled_ip":"127.0.0.1","app":"unknown","app_transition":"","client_geolocation":"unknown","server_geolocation":"unknown","decoded_path":"ETHERNET.IPv4.UDP.rtp","server_fqdn":"","start_timestamp_ms":1721639438014,"end_timestamp_ms":1721639438014,"tcp_rtt_ms":0,"tcp_client_isn":0,"tcp_server_isn":0,"tcp_handshake_latency_ms":0,"in_link_id":0,"out_link_id":0,"duration_ms":0,"sent_pkts":0,"sent_bytes":0,"received_pkts":0,"received_bytes":0,"flags":24576,"flags_identify_info":[1,1],"fqdn_category_list":[0],"client_os_desc":"Windows","server_os_desc":"Linux","rtp_pcap_path":"123e4567-e89b-12d3-a456-426614174005"}
+{"__timestamp":946681200,"__inputid":"tsg_olap","session_id":10240002,"decoded_as":"SIP","ip_protocol":"udp","address_type":4,"client_ip":"10.0.0.1","server_ip":"192.0.2.1","client_port":1000,"server_port":60000,"t_vsys_id":0,"vsys_id":1024,"data_center":"tsg_olap","device_group":"tsg_olap","device_id":"0000000000000000","sled_ip":"127.0.0.1","app":"unknown","app_transition":"","client_geolocation":"unknown","server_geolocation":"unknown","decoded_path":"ETHERNET.IPv4.UDP.sip","server_fqdn":"","start_timestamp_ms":1721639438014,"end_timestamp_ms":1721639438014,"tcp_rtt_ms":0,"tcp_client_isn":0,"tcp_server_isn":0,"tcp_handshake_latency_ms":0,"in_link_id":0,"out_link_id":0,"duration_ms":0,"sent_pkts":0,"sent_bytes":0,"received_pkts":0,"received_bytes":0,"flags":24584,"flags_identify_info":[1,1],"fqdn_category_list":[0],"client_os_desc":"Windows","server_os_desc":"Linux","sip_call_id":"NGMxZWY3Y2NmMzNlNGE3NzJhODgyZDAwM2YyMzQ4NGI.","sip_originator_description":"\"lina\"<sip:lina@192.0.2.1>;tag=1837055d","sip_responder_description":"\"1075\"<sip:1075@192.0.2.1>","sip_originator_sdp_connect_ip":"192.168.64.8","sip_originator_sdp_media_port":25524,"sip_originator_sdp_media_type":"application/sdp","sip_server":"OpenSIPS (2.4.11 (x86_64/linux))","sip_responder_sdp_connect_ip":"192.168.39.62","sip_responder_sdp_media_port":4580,"sip_responder_sdp_media_type":"application/sdp","sip_duration_s":590,"sip_bye":"responder","sip_cseq":"2 BYE","sip_via":"SIP/2.0/UDP 192.0.2.1:5060;branch=z9hG4bKbe7c.392190f1.0","sip_user_agent":"eyeBeam release 1011d stamp 40820","sip_is_request":0}
+{"__timestamp":946681200,"__inputid":"tsg_olap","session_id":10240003,"decoded_as":"RTP","ip_protocol":"udp","address_type":4,"client_ip":"192.168.164.18","server_ip":"192.168.39.162","client_port":65121,"server_port":4670,"t_vsys_id":0,"vsys_id":1024,"data_center":"tsg_olap","device_group":"tsg_olap","device_id":"0000000000000000","sled_ip":"127.0.0.1","app":"unknown","app_transition":"","client_geolocation":"unknown","server_geolocation":"unknown","decoded_path":"ETHERNET.IPv4.UDP.rtp","server_fqdn":"","start_timestamp_ms":1721639438014,"end_timestamp_ms":1721639438014,"tcp_rtt_ms":0,"tcp_client_isn":0,"tcp_server_isn":0,"tcp_handshake_latency_ms":0,"in_link_id":0,"out_link_id":0,"duration_ms":0,"sent_pkts":0,"sent_bytes":0,"received_pkts":0,"received_bytes":0,"flags":24584,"flags_identify_info":[1,1],"fqdn_category_list":[0],"client_os_desc":"Windows","server_os_desc":"Linux","rtp_pcap_path":"123e4567-e89b-12d3-a456-426614174001"}
+{"__timestamp":946681200,"__inputid":"tsg_olap","session_id":10240004,"decoded_as":"SIP","ip_protocol":"udp","address_type":4,"client_ip":"10.0.0.2","server_ip":"192.0.2.2","client_port":1000,"server_port":60000,"t_vsys_id":0,"vsys_id":1024,"data_center":"tsg_olap","device_group":"tsg_olap","device_id":"0000000000000000","sled_ip":"127.0.0.1","app":"unknown","app_transition":"","client_geolocation":"unknown","server_geolocation":"unknown","decoded_path":"ETHERNET.IPv4.UDP.sip","server_fqdn":"","start_timestamp_ms":1721639438014,"end_timestamp_ms":1721639438014,"tcp_rtt_ms":0,"tcp_client_isn":0,"tcp_server_isn":0,"tcp_handshake_latency_ms":0,"in_link_id":0,"out_link_id":0,"duration_ms":0,"sent_pkts":0,"sent_bytes":0,"received_pkts":0,"received_bytes":0,"flags":24584,"flags_identify_info":[1,1],"fqdn_category_list":[0],"client_os_desc":"Windows","server_os_desc":"Linux","sip_call_id":"CUMxZWY3Y2NmMzNlNGE3NzJhODgyZDAwM2YyMzQ4NGI.","sip_originator_description":"\"lina\"<sip:lina@192.0.2.1>;tag=1837055d","sip_responder_description":"\"1075\"<sip:1075@192.0.2.1>","sip_originator_sdp_connect_ip":"192.68.64.8","sip_originator_sdp_media_port":25524,"sip_originator_sdp_media_type":"application/sdp","sip_server":"OpenSIPS (2.4.11 (x86_64/linux))","sip_responder_sdp_connect_ip":"192.18.39.62","sip_responder_sdp_media_port":4580,"sip_responder_sdp_media_type":"application/sdp","sip_duration_s":590,"sip_bye":"responder","sip_cseq":"2 BYE","sip_via":"SIP/2.0/UDP 192.0.2.1:5060;branch=z9hG4bKbe7c.392190f1.0","sip_user_agent":"eyeBeam release 1011d stamp 40820","sip_is_request":0}

tools/maven/spotbugs-exclude.xml

@@ -6,6 +6,9 @@
     <Match>
         <Bug pattern="EI_EXPOSE_REP2"/>
     </Match>
+    <Match>
+        <Bug pattern="REC_CATCH_EXCEPTION"/>
+    </Match>
     <Match>
         <Bug pattern="SE_NO_SERIALVERSIONID"/>
     </Match>