gfwleak/galaxy-tsg-olap-flume-dynamic-complement
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

更新TSG-1845,适配TSG-2020

Browse Source
This commit is contained in:
qidaijie
2020-06-15 13:39:17 +08:00
parent a556d3ef66
commit 24ff96be4c
4 changed files with 305 additions and 71 deletions
Download Patch File Download Diff File Expand all files Collapse all files

135
FlumeDynamicInterceptor/dependency-reduced-pom.xml Normal file
View File

@@ -0,0 +1,135 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
<parent>
<artifactId>dynamic_complement</artifactId>
<groupId>com.zdjizhi</groupId>
<version>1.0</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>FlumeDynamicInterceptor</artifactId>
<build>
<resources>
<resource>
<directory>properties</directory>
<includes>
<include>**/*.properties</include>
<include>**/*.xml</include>
</includes>
</resource>
</resources>
<plugins>
<plugin>
<artifactId>maven-shade-plugin</artifactId>
<version>2.4.1</version>
<executions>
<execution>
<phase>package</phase>
<goals>
<goal>shade</goal>
</goals>
<configuration>
<transformers>
<transformer />
<transformer>
<mainClass>com.zdjizhi.flume.interceptor.FlumeDynamicApp</mainClass>
</transformer>
<transformer>
<resource>META-INF/spring.handlers</resource>
</transformer>
<transformer>
<resource>META-INF/spring.schemas</resource>
</transformer>
</transformers>
</configuration>
</execution>
</executions>
<configuration>
<createDependencyReducedPom>true</createDependencyReducedPom>
</configuration>
</plugin>
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>exec-maven-plugin</artifactId>
<version>1.2.1</version>
<executions>
<execution>
<goals>
<goal>exec</goal>
</goals>
</execution>
</executions>
<configuration>
<executable>java</executable>
<includeProjectDependencies>true</includeProjectDependencies>
<includePluginDependencies>false</includePluginDependencies>
<classpathScope>compile</classpathScope>
<mainClass>com.zdjizhi.flume.interceptor.FlumeDynamicApp</mainClass>
</configuration>
</plugin>
<plugin>
<artifactId>maven-compiler-plugin</artifactId>
<version>2.3.2</version>
<configuration>
<source>1.8</source>
<target>1.8</target>
</configuration>
</plugin>
</plugins>
</build>
<repositories>
<repository>
<id>nexus</id>
<name>Team Nexus Repository</name>
<url>http://192.168.40.125:8099/content/groups/public</url>
</repository>
<repository>
<id>ebi</id>
<name>www.ebi.ac.uk</name>
<url>http://www.ebi.ac.uk/intact/maven/nexus/content/groups/public/</url>
</repository>
</repositories>
<dependencies>
<dependency>
<groupId>org.apache.flume</groupId>
<artifactId>flume-ng-core</artifactId>
<version>1.9.0</version>
<scope>provided</scope>
<exclusions>
<exclusion>
<artifactId>flume-ng-sdk</artifactId>
<groupId>org.apache.flume</groupId>
</exclusion>
<exclusion>
<artifactId>flume-ng-configuration</artifactId>
<groupId>org.apache.flume</groupId>
</exclusion>
<exclusion>
<artifactId>flume-ng-auth</artifactId>
<groupId>org.apache.flume</groupId>
</exclusion>
<exclusion>
<artifactId>avro-ipc</artifactId>
<groupId>org.apache.avro</groupId>
</exclusion>
<exclusion>
<artifactId>jetty-jmx</artifactId>
<groupId>org.eclipse.jetty</groupId>
</exclusion>
<exclusion>
<artifactId>libthrift</artifactId>
<groupId>org.apache.thrift</groupId>
</exclusion>
<exclusion>
<artifactId>mina-core</artifactId>
<groupId>org.apache.mina</groupId>
</exclusion>
</exclusions>
</dependency>
</dependencies>
<properties>
<hbase.version>2.2.1</hbase.version>
<flume.version>1.9.0</flume.version>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
</properties>
</project>

7
FlumeDynamicInterceptor/src/main/java/com/zdjizhi/flume/interceptor/FlumeDynamicApp.java
View File

@@ -61,9 +61,10 @@ public class FlumeDynamicApp implements Interceptor {
/**
* v1.0.3-com.zdjizhi.galaxy
*/
.loadDataFile(ipDatPath + "all_ip_info_v4.mmdb")
.loadDataFilePrivateV4(ipDatPath + "ip_private.mmdb")
.loadDataFilePrivateV6(ipDatPath + "ip_private.mmdb")
.loadDataFileV4(ipDatPath + "ip_v4.mmdb")
.loadDataFileV6(ipDatPath + "ip_v6.mmdb")
.loadDataFilePrivateV4(ipDatPath + "ip_private_v4.mmdb")
.loadDataFilePrivateV6(ipDatPath + "ip_private_v6.mmdb")
.loadAsnDataFile(ipDatPath + "asn_v4.mmdb")
.loadAsnDataFileV6(ipDatPath + "asn_v6.mmdb")
.build();

2
FlumeDynamicInterceptor/src/main/java/com/zdjizhi/flume/interceptor/utils/hbase/HBaseUtils.java
View File

@@ -24,7 +24,7 @@ import java.util.TimerTask;
public class HBaseUtils {
private final static Logger logger = Logger.getLogger(HBaseUtils.class);
private static Map<String, String> subIdMap = new HashMap<>(333334);
private static Map<String, String> subIdMap = new HashMap<>(128);
private static Connection connection;
private static Long time;