gfwleak/galaxy-tsg-olap-dll-multipoint-aggregation
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

修正通联日志的任务bug

Browse Source
This commit is contained in:
李玺康
2020-05-13 10:31:32 +08:00
parent 5f285e3dd5
commit 022925d12c
8 changed files with 79 additions and 27 deletions
Download Patch File Download Diff File Expand all files Collapse all files

26
internal-flume/conf/connection/connection_f2k1.properties
View File

@@ -10,19 +10,19 @@ connection.sources.s2.positionFile = /home/test/1connection_position.json
#需要分组13台主机的数据 #需要分组13台主机的数据
connection.sources.s2.filegroups = f1 f2 f3 f4 f5 f6 f7 f8 f9 f10 f11 f12 f13 connection.sources.s2.filegroups = f1 f2 f3 f4 f5 f6 f7 f8 f9 f10 f11 f12 f13
#需要分组按照序号 [0-9] 目前最多支持10个进程，可以增加 #需要分组按照序号 [0-9] 目前最多支持10个进程，可以增加
connection.sources.s2.filegroups.f1 = /home/data/192.168.60.101/CONNECTION-RECORD-LOG\CONNECTION-RECORD-LOG/.*[0-1].dat connection.sources.s2.filegroups.f1 = /home/data/192.168.60.101/CONNECTION-RECORD-LOG/.*[0-1].dat
connection.sources.s2.filegroups.f2 = /home/data/192.168.60.102/CONNECTION-RECORD-LOG\CONNECTION-RECORD-LOG/.*[0-1].dat connection.sources.s2.filegroups.f2 = /home/data/192.168.60.102/CONNECTION-RECORD-LOG/.*[0-1].dat
connection.sources.s2.filegroups.f3 = /home/data/192.168.60.103/CONNECTION-RECORD-LOG\CONNECTION-RECORD-LOG/.*[0-1].dat connection.sources.s2.filegroups.f3 = /home/data/192.168.60.103/CONNECTION-RECORD-LOG/.*[0-1].dat
connection.sources.s2.filegroups.f4 = /home/data/192.168.60.104/CONNECTION-RECORD-LOG\CONNECTION-RECORD-LOG/.*[0-1].dat connection.sources.s2.filegroups.f4 = /home/data/192.168.60.104/CONNECTION-RECORD-LOG/.*[0-1].dat
connection.sources.s2.filegroups.f5 = /home/data/192.168.60.105/CONNECTION-RECORD-LOG\CONNECTION-RECORD-LOG/.*[0-1].dat connection.sources.s2.filegroups.f5 = /home/data/192.168.60.105/CONNECTION-RECORD-LOG/.*[0-1].dat
connection.sources.s2.filegroups.f6 = /home/data/192.168.60.106/CONNECTION-RECORD-LOG\CONNECTION-RECORD-LOG/.*[0-1].dat connection.sources.s2.filegroups.f6 = /home/data/192.168.60.106/CONNECTION-RECORD-LOG/.*[0-1].dat
connection.sources.s2.filegroups.f7 = /home/data/192.168.60.107/CONNECTION-RECORD-LOG\CONNECTION-RECORD-LOG/.*[0-1].dat connection.sources.s2.filegroups.f7 = /home/data/192.168.60.107/CONNECTION-RECORD-LOG/.*[0-1].dat
connection.sources.s2.filegroups.f8 = /home/data/192.168.60.108/CONNECTION-RECORD-LOG\CONNECTION-RECORD-LOG/.*[0-1].dat connection.sources.s2.filegroups.f8 = /home/data/192.168.60.108/CONNECTION-RECORD-LOG/.*[0-1].dat
connection.sources.s2.filegroups.f9 = /home/data/192.168.60.109/CONNECTION-RECORD-LOG\CONNECTION-RECORD-LOG/.*[0-1].dat connection.sources.s2.filegroups.f9 = /home/data/192.168.60.109/CONNECTION-RECORD-LOG/.*[0-1].dat
connection.sources.s2.filegroups.f10 = /home/data/192.168.60.110/CONNECTION-RECORD-LOG\CONNECTION-RECORD-LOG/.*[0-1].dat connection.sources.s2.filegroups.f10 = /home/data/192.168.60.110/CONNECTION-RECORD-LOG/.*[0-1].dat
connection.sources.s2.filegroups.f11 = /home/data/192.168.60.111/CONNECTION-RECORD-LOG\CONNECTION-RECORD-LOG/.*[0-1].dat connection.sources.s2.filegroups.f11 = /home/data/192.168.60.111/CONNECTION-RECORD-LOG/.*[0-1].dat
connection.sources.s2.filegroups.f12 = /home/data/192.168.60.112/CONNECTION-RECORD-LOG\CONNECTION-RECORD-LOG/.*[0-1].dat connection.sources.s2.filegroups.f12 = /home/data/192.168.60.112/CONNECTION-RECORD-LOG/.*[0-1].dat
connection.sources.s2.filegroups.f13 = /home/data/192.168.60.113/CONNECTION-RECORD-LOG\CONNECTION-RECORD-LOG/.*[0-1].dat connection.sources.s2.filegroups.f13 = /home/data/192.168.60.113/CONNECTION-RECORD-LOG/.*[0-1].dat
connection.sources.s2.maxBatchCount = 10000 connection.sources.s2.maxBatchCount = 10000

26
internal-flume/conf/connection/connection_f2k2.properties
View File

@@ -10,19 +10,19 @@ connection.sources.s2.positionFile = /home/test/2connection_position.json
#需要分组13台主机的数据 #需要分组13台主机的数据
connection.sources.s2.filegroups = f1 f2 f3 f4 f5 f6 f7 f8 f9 f10 f11 f12 f13 connection.sources.s2.filegroups = f1 f2 f3 f4 f5 f6 f7 f8 f9 f10 f11 f12 f13
#需要分组按照序号 [0-9] 目前最多支持10个进程，可以增加 #需要分组按照序号 [0-9] 目前最多支持10个进程，可以增加
connection.sources.s2.filegroups.f1 = /home/data/192.168.60.101/CONNECTION-RECORD-LOG/CONNECTION-RECORD-LOG/.*[2-3].dat connection.sources.s2.filegroups.f1 = /home/data/192.168.60.101/CONNECTION-RECORD-LOG/.*[2-3].dat
connection.sources.s2.filegroups.f2 = /home/data/192.168.60.102/CONNECTION-RECORD-LOG/CONNECTION-RECORD-LOG/.*[2-3].dat connection.sources.s2.filegroups.f2 = /home/data/192.168.60.102/CONNECTION-RECORD-LOG/.*[2-3].dat
connection.sources.s2.filegroups.f3 = /home/data/192.168.60.103/CONNECTION-RECORD-LOG/CONNECTION-RECORD-LOG/.*[2-3].dat connection.sources.s2.filegroups.f3 = /home/data/192.168.60.103/CONNECTION-RECORD-LOG/.*[2-3].dat
connection.sources.s2.filegroups.f4 = /home/data/192.168.60.104/CONNECTION-RECORD-LOG/CONNECTION-RECORD-LOG/.*[2-3].dat connection.sources.s2.filegroups.f4 = /home/data/192.168.60.104/CONNECTION-RECORD-LOG/.*[2-3].dat
connection.sources.s2.filegroups.f5 = /home/data/192.168.60.105/CONNECTION-RECORD-LOG/CONNECTION-RECORD-LOG/.*[2-3].dat connection.sources.s2.filegroups.f5 = /home/data/192.168.60.105/CONNECTION-RECORD-LOG/.*[2-3].dat
connection.sources.s2.filegroups.f6 = /home/data/192.168.60.106/CONNECTION-RECORD-LOG/CONNECTION-RECORD-LOG/.*[2-3].dat connection.sources.s2.filegroups.f6 = /home/data/192.168.60.106/CONNECTION-RECORD-LOG/.*[2-3].dat
connection.sources.s2.filegroups.f7 = /home/data/192.168.60.107/CONNECTION-RECORD-LOG/CONNECTION-RECORD-LOG/.*[2-3].dat connection.sources.s2.filegroups.f7 = /home/data/192.168.60.107/CONNECTION-RECORD-LOG/.*[2-3].dat
connection.sources.s2.filegroups.f8 = /home/data/192.168.60.108/CONNECTION-RECORD-LOG/CONNECTION-RECORD-LOG/.*[2-3].dat connection.sources.s2.filegroups.f8 = /home/data/192.168.60.108/CONNECTION-RECORD-LOG/.*[2-3].dat
connection.sources.s2.filegroups.f9 = /home/data/192.168.60.109/CONNECTION-RECORD-LOG/CONNECTION-RECORD-LOG/.*[2-3].dat connection.sources.s2.filegroups.f9 = /home/data/192.168.60.109/CONNECTION-RECORD-LOG/.*[2-3].dat
connection.sources.s2.filegroups.f10 = /home/data/192.168.60.110/CONNECTION-RECORD-LOG/CONNECTION-RECORD-LOG/.*[2-3].dat connection.sources.s2.filegroups.f10 = /home/data/192.168.60.110/CONNECTION-RECORD-LOG/.*[2-3].dat
connection.sources.s2.filegroups.f11 = /home/data/192.168.60.111/CONNECTION-RECORD-LOG/CONNECTION-RECORD-LOG/.*[2-3].dat connection.sources.s2.filegroups.f11 = /home/data/192.168.60.111/CONNECTION-RECORD-LOG/.*[2-3].dat
connection.sources.s2.filegroups.f12 = /home/data/192.168.60.112/CONNECTION-RECORD-LOG/CONNECTION-RECORD-LOG/.*[2-3].dat connection.sources.s2.filegroups.f12 = /home/data/192.168.60.112/CONNECTION-RECORD-LOG/.*[2-3].dat
connection.sources.s2.filegroups.f13 = /home/data/192.168.60.113/CONNECTION-RECORD-LOG/CONNECTION-RECORD-LOG/.*[2-3].dat connection.sources.s2.filegroups.f13 = /home/data/192.168.60.113/CONNECTION-RECORD-LOG/.*[2-3].dat
connection.sources.s2.maxBatchCount = 10000 connection.sources.s2.maxBatchCount = 10000

35
internal-flume/conf/connection/restart_log/restart_connection_f2k.log
View File

@@ -20,3 +20,38 @@
20200512184523 ---> the connection_f2k_APP restart ---> 3 20200512184523 ---> the connection_f2k_APP restart ---> 3
20200512184533 ---> the connection_f2k_APP restart ---> 4 20200512184533 ---> the connection_f2k_APP restart ---> 4
20200512184543 ---> the connection_f2k_APP restart ---> 5 20200512184543 ---> the connection_f2k_APP restart ---> 5
20200512191902 ---> the connection_f2k_APP restart ---> 1
20200512191912 ---> the connection_f2k_APP restart ---> 2
##############################################################
20200512192032 ---> the connection_f2k_APP restart ---> 1
20200512192043 ---> the connection_f2k_APP restart ---> 2
20200512192053 ---> the connection_f2k_APP restart ---> 3
20200512192103 ---> the connection_f2k_APP restart ---> 4
20200512192113 ---> the connection_f2k_APP restart ---> 5
20200512192323 ---> the connection_f2k_APP restart ---> 1
20200512192333 ---> the connection_f2k_APP restart ---> 2
20200512192343 ---> the connection_f2k_APP restart ---> 3
20200512192353 ---> the connection_f2k_APP restart ---> 4
20200512192403 ---> the connection_f2k_APP restart ---> 5
##############################################################
20200512193146 ---> the connection_f2k_APP restart ---> 1
20200512193156 ---> the connection_f2k_APP restart ---> 2
20200512193206 ---> the connection_f2k_APP restart ---> 3
20200512193216 ---> the connection_f2k_APP restart ---> 4
20200512193226 ---> the connection_f2k_APP restart ---> 5
20200512193406 ---> the connection_f2k_APP restart ---> 1
20200512193416 ---> the connection_f2k_APP restart ---> 2
20200512193426 ---> the connection_f2k_APP restart ---> 3
20200512193437 ---> the connection_f2k_APP restart ---> 4
##############################################################
20200512193804 ---> the connection_f2k_APP restart ---> 1
20200512193814 ---> the connection_f2k_APP restart ---> 2
20200512193824 ---> the connection_f2k_APP restart ---> 3
20200512193834 ---> the connection_f2k_APP restart ---> 4
20200512193844 ---> the connection_f2k_APP restart ---> 5
##############################################################
20200513102414 ---> the connection_f2k_APP restart ---> 1
20200513102424 ---> the connection_f2k_APP restart ---> 2
20200513102434 ---> the connection_f2k_APP restart ---> 3
20200513102444 ---> the connection_f2k_APP restart ---> 4
20200513102454 ---> the connection_f2k_APP restart ---> 5

2
internal-flume/conf/connection/start_flume.sh
View File

@@ -2,4 +2,4 @@
BASE_DIR=$(cd $(dirname $0); pwd) BASE_DIR=$(cd $(dirname $0); pwd)
nohup ${BASE_DIR}/dae_f2k.sh >/dev/null 2>&1 & nohup ${BASE_DIR}/dae_connection.sh >/dev/null 2>&1 &

BIN
internal-flume/dat/Kazakhstan.mmdb
View File

Binary file not shown.

BIN
internal-flume/dat/asn_v4.mmdb
View File

Binary file not shown.

BIN
internal-flume/dat/asn_v6.mmdb
View File

Binary file not shown.

17
internal-flume/docker-compose.yml Normal file
View File

@@ -0,0 +1,17 @@
version: '2'
services:
flume:
image: 192.168.40.153:9080/tsg/flume:1.9.0
restart: always
container_name: flume
volumes:
- "/home/tsg3.0-volumes/flume/conf:/home/apache-flume-1.9.0-bin/conf"
- "/home/tsg3.0-volumes/flume/flumeController:/home/apache-flume-1.9.0-bin/flumeController"
- "/home/tsg3.0-volumes/flume/json:/home/test"
- "/home/tsg3.0-volumes/flume/data:/home/data"
- "/home/tsg3.0-volumes/flume/flumeMonitor:/home/apache-flume-1.9.0-bin/flumeMonitor"
- "/etc/localtime:/etc/localtime:ro"
- "/etc/timezone:/etc/timezone:ro"
network_mode: "host"