diff --git a/internal-flume/conf/connection/connection_f2k1.properties b/internal-flume/conf/connection/connection_f2k1.properties index d05ccdc..c61222e 100644 --- a/internal-flume/conf/connection/connection_f2k1.properties +++ b/internal-flume/conf/connection/connection_f2k1.properties @@ -10,19 +10,19 @@ connection.sources.s2.positionFile = /home/test/1connection_position.json #需要分组13台主机的数据 connection.sources.s2.filegroups = f1 f2 f3 f4 f5 f6 f7 f8 f9 f10 f11 f12 f13 #需要分组按照序号 [0-9] 目前最多支持10个进程,可以增加 -connection.sources.s2.filegroups.f1 = /home/data/192.168.60.101/CONNECTION-RECORD-LOG\CONNECTION-RECORD-LOG/.*[0-1].dat -connection.sources.s2.filegroups.f2 = /home/data/192.168.60.102/CONNECTION-RECORD-LOG\CONNECTION-RECORD-LOG/.*[0-1].dat -connection.sources.s2.filegroups.f3 = /home/data/192.168.60.103/CONNECTION-RECORD-LOG\CONNECTION-RECORD-LOG/.*[0-1].dat -connection.sources.s2.filegroups.f4 = /home/data/192.168.60.104/CONNECTION-RECORD-LOG\CONNECTION-RECORD-LOG/.*[0-1].dat -connection.sources.s2.filegroups.f5 = /home/data/192.168.60.105/CONNECTION-RECORD-LOG\CONNECTION-RECORD-LOG/.*[0-1].dat -connection.sources.s2.filegroups.f6 = /home/data/192.168.60.106/CONNECTION-RECORD-LOG\CONNECTION-RECORD-LOG/.*[0-1].dat -connection.sources.s2.filegroups.f7 = /home/data/192.168.60.107/CONNECTION-RECORD-LOG\CONNECTION-RECORD-LOG/.*[0-1].dat -connection.sources.s2.filegroups.f8 = /home/data/192.168.60.108/CONNECTION-RECORD-LOG\CONNECTION-RECORD-LOG/.*[0-1].dat -connection.sources.s2.filegroups.f9 = /home/data/192.168.60.109/CONNECTION-RECORD-LOG\CONNECTION-RECORD-LOG/.*[0-1].dat -connection.sources.s2.filegroups.f10 = /home/data/192.168.60.110/CONNECTION-RECORD-LOG\CONNECTION-RECORD-LOG/.*[0-1].dat -connection.sources.s2.filegroups.f11 = /home/data/192.168.60.111/CONNECTION-RECORD-LOG\CONNECTION-RECORD-LOG/.*[0-1].dat -connection.sources.s2.filegroups.f12 = /home/data/192.168.60.112/CONNECTION-RECORD-LOG\CONNECTION-RECORD-LOG/.*[0-1].dat -connection.sources.s2.filegroups.f13 = /home/data/192.168.60.113/CONNECTION-RECORD-LOG\CONNECTION-RECORD-LOG/.*[0-1].dat +connection.sources.s2.filegroups.f1 = /home/data/192.168.60.101/CONNECTION-RECORD-LOG/.*[0-1].dat +connection.sources.s2.filegroups.f2 = /home/data/192.168.60.102/CONNECTION-RECORD-LOG/.*[0-1].dat +connection.sources.s2.filegroups.f3 = /home/data/192.168.60.103/CONNECTION-RECORD-LOG/.*[0-1].dat +connection.sources.s2.filegroups.f4 = /home/data/192.168.60.104/CONNECTION-RECORD-LOG/.*[0-1].dat +connection.sources.s2.filegroups.f5 = /home/data/192.168.60.105/CONNECTION-RECORD-LOG/.*[0-1].dat +connection.sources.s2.filegroups.f6 = /home/data/192.168.60.106/CONNECTION-RECORD-LOG/.*[0-1].dat +connection.sources.s2.filegroups.f7 = /home/data/192.168.60.107/CONNECTION-RECORD-LOG/.*[0-1].dat +connection.sources.s2.filegroups.f8 = /home/data/192.168.60.108/CONNECTION-RECORD-LOG/.*[0-1].dat +connection.sources.s2.filegroups.f9 = /home/data/192.168.60.109/CONNECTION-RECORD-LOG/.*[0-1].dat +connection.sources.s2.filegroups.f10 = /home/data/192.168.60.110/CONNECTION-RECORD-LOG/.*[0-1].dat +connection.sources.s2.filegroups.f11 = /home/data/192.168.60.111/CONNECTION-RECORD-LOG/.*[0-1].dat +connection.sources.s2.filegroups.f12 = /home/data/192.168.60.112/CONNECTION-RECORD-LOG/.*[0-1].dat +connection.sources.s2.filegroups.f13 = /home/data/192.168.60.113/CONNECTION-RECORD-LOG/.*[0-1].dat connection.sources.s2.maxBatchCount = 10000 diff --git a/internal-flume/conf/connection/connection_f2k2.properties b/internal-flume/conf/connection/connection_f2k2.properties index fd75769..03baa50 100644 --- a/internal-flume/conf/connection/connection_f2k2.properties +++ b/internal-flume/conf/connection/connection_f2k2.properties @@ -10,19 +10,19 @@ connection.sources.s2.positionFile = /home/test/2connection_position.json #需要分组13台主机的数据 connection.sources.s2.filegroups = f1 f2 f3 f4 f5 f6 f7 f8 f9 f10 f11 f12 f13 #需要分组按照序号 [0-9] 目前最多支持10个进程,可以增加 -connection.sources.s2.filegroups.f1 = /home/data/192.168.60.101/CONNECTION-RECORD-LOG/CONNECTION-RECORD-LOG/.*[2-3].dat -connection.sources.s2.filegroups.f2 = /home/data/192.168.60.102/CONNECTION-RECORD-LOG/CONNECTION-RECORD-LOG/.*[2-3].dat -connection.sources.s2.filegroups.f3 = /home/data/192.168.60.103/CONNECTION-RECORD-LOG/CONNECTION-RECORD-LOG/.*[2-3].dat -connection.sources.s2.filegroups.f4 = /home/data/192.168.60.104/CONNECTION-RECORD-LOG/CONNECTION-RECORD-LOG/.*[2-3].dat -connection.sources.s2.filegroups.f5 = /home/data/192.168.60.105/CONNECTION-RECORD-LOG/CONNECTION-RECORD-LOG/.*[2-3].dat -connection.sources.s2.filegroups.f6 = /home/data/192.168.60.106/CONNECTION-RECORD-LOG/CONNECTION-RECORD-LOG/.*[2-3].dat -connection.sources.s2.filegroups.f7 = /home/data/192.168.60.107/CONNECTION-RECORD-LOG/CONNECTION-RECORD-LOG/.*[2-3].dat -connection.sources.s2.filegroups.f8 = /home/data/192.168.60.108/CONNECTION-RECORD-LOG/CONNECTION-RECORD-LOG/.*[2-3].dat -connection.sources.s2.filegroups.f9 = /home/data/192.168.60.109/CONNECTION-RECORD-LOG/CONNECTION-RECORD-LOG/.*[2-3].dat -connection.sources.s2.filegroups.f10 = /home/data/192.168.60.110/CONNECTION-RECORD-LOG/CONNECTION-RECORD-LOG/.*[2-3].dat -connection.sources.s2.filegroups.f11 = /home/data/192.168.60.111/CONNECTION-RECORD-LOG/CONNECTION-RECORD-LOG/.*[2-3].dat -connection.sources.s2.filegroups.f12 = /home/data/192.168.60.112/CONNECTION-RECORD-LOG/CONNECTION-RECORD-LOG/.*[2-3].dat -connection.sources.s2.filegroups.f13 = /home/data/192.168.60.113/CONNECTION-RECORD-LOG/CONNECTION-RECORD-LOG/.*[2-3].dat +connection.sources.s2.filegroups.f1 = /home/data/192.168.60.101/CONNECTION-RECORD-LOG/.*[2-3].dat +connection.sources.s2.filegroups.f2 = /home/data/192.168.60.102/CONNECTION-RECORD-LOG/.*[2-3].dat +connection.sources.s2.filegroups.f3 = /home/data/192.168.60.103/CONNECTION-RECORD-LOG/.*[2-3].dat +connection.sources.s2.filegroups.f4 = /home/data/192.168.60.104/CONNECTION-RECORD-LOG/.*[2-3].dat +connection.sources.s2.filegroups.f5 = /home/data/192.168.60.105/CONNECTION-RECORD-LOG/.*[2-3].dat +connection.sources.s2.filegroups.f6 = /home/data/192.168.60.106/CONNECTION-RECORD-LOG/.*[2-3].dat +connection.sources.s2.filegroups.f7 = /home/data/192.168.60.107/CONNECTION-RECORD-LOG/.*[2-3].dat +connection.sources.s2.filegroups.f8 = /home/data/192.168.60.108/CONNECTION-RECORD-LOG/.*[2-3].dat +connection.sources.s2.filegroups.f9 = /home/data/192.168.60.109/CONNECTION-RECORD-LOG/.*[2-3].dat +connection.sources.s2.filegroups.f10 = /home/data/192.168.60.110/CONNECTION-RECORD-LOG/.*[2-3].dat +connection.sources.s2.filegroups.f11 = /home/data/192.168.60.111/CONNECTION-RECORD-LOG/.*[2-3].dat +connection.sources.s2.filegroups.f12 = /home/data/192.168.60.112/CONNECTION-RECORD-LOG/.*[2-3].dat +connection.sources.s2.filegroups.f13 = /home/data/192.168.60.113/CONNECTION-RECORD-LOG/.*[2-3].dat connection.sources.s2.maxBatchCount = 10000 diff --git a/internal-flume/conf/connection/restart_log/restart_connection_f2k.log b/internal-flume/conf/connection/restart_log/restart_connection_f2k.log index 0722d59..f5ec36f 100644 --- a/internal-flume/conf/connection/restart_log/restart_connection_f2k.log +++ b/internal-flume/conf/connection/restart_log/restart_connection_f2k.log @@ -20,3 +20,38 @@ 20200512184523 ---> the connection_f2k_APP restart ---> 3 20200512184533 ---> the connection_f2k_APP restart ---> 4 20200512184543 ---> the connection_f2k_APP restart ---> 5 +20200512191902 ---> the connection_f2k_APP restart ---> 1 +20200512191912 ---> the connection_f2k_APP restart ---> 2 +############################################################## +20200512192032 ---> the connection_f2k_APP restart ---> 1 +20200512192043 ---> the connection_f2k_APP restart ---> 2 +20200512192053 ---> the connection_f2k_APP restart ---> 3 +20200512192103 ---> the connection_f2k_APP restart ---> 4 +20200512192113 ---> the connection_f2k_APP restart ---> 5 +20200512192323 ---> the connection_f2k_APP restart ---> 1 +20200512192333 ---> the connection_f2k_APP restart ---> 2 +20200512192343 ---> the connection_f2k_APP restart ---> 3 +20200512192353 ---> the connection_f2k_APP restart ---> 4 +20200512192403 ---> the connection_f2k_APP restart ---> 5 +############################################################## +20200512193146 ---> the connection_f2k_APP restart ---> 1 +20200512193156 ---> the connection_f2k_APP restart ---> 2 +20200512193206 ---> the connection_f2k_APP restart ---> 3 +20200512193216 ---> the connection_f2k_APP restart ---> 4 +20200512193226 ---> the connection_f2k_APP restart ---> 5 +20200512193406 ---> the connection_f2k_APP restart ---> 1 +20200512193416 ---> the connection_f2k_APP restart ---> 2 +20200512193426 ---> the connection_f2k_APP restart ---> 3 +20200512193437 ---> the connection_f2k_APP restart ---> 4 +############################################################## +20200512193804 ---> the connection_f2k_APP restart ---> 1 +20200512193814 ---> the connection_f2k_APP restart ---> 2 +20200512193824 ---> the connection_f2k_APP restart ---> 3 +20200512193834 ---> the connection_f2k_APP restart ---> 4 +20200512193844 ---> the connection_f2k_APP restart ---> 5 +############################################################## +20200513102414 ---> the connection_f2k_APP restart ---> 1 +20200513102424 ---> the connection_f2k_APP restart ---> 2 +20200513102434 ---> the connection_f2k_APP restart ---> 3 +20200513102444 ---> the connection_f2k_APP restart ---> 4 +20200513102454 ---> the connection_f2k_APP restart ---> 5 diff --git a/internal-flume/conf/connection/start_flume.sh b/internal-flume/conf/connection/start_flume.sh index 619f9aa..e74045a 100755 --- a/internal-flume/conf/connection/start_flume.sh +++ b/internal-flume/conf/connection/start_flume.sh @@ -2,4 +2,4 @@ BASE_DIR=$(cd $(dirname $0); pwd) -nohup ${BASE_DIR}/dae_f2k.sh >/dev/null 2>&1 & +nohup ${BASE_DIR}/dae_connection.sh >/dev/null 2>&1 & diff --git a/internal-flume/dat/Kazakhstan.mmdb b/internal-flume/dat/Kazakhstan.mmdb deleted file mode 100644 index a5d5bd9..0000000 Binary files a/internal-flume/dat/Kazakhstan.mmdb and /dev/null differ diff --git a/internal-flume/dat/asn_v4.mmdb b/internal-flume/dat/asn_v4.mmdb deleted file mode 100644 index 4508ea2..0000000 Binary files a/internal-flume/dat/asn_v4.mmdb and /dev/null differ diff --git a/internal-flume/dat/asn_v6.mmdb b/internal-flume/dat/asn_v6.mmdb deleted file mode 100644 index a779312..0000000 Binary files a/internal-flume/dat/asn_v6.mmdb and /dev/null differ diff --git a/internal-flume/docker-compose.yml b/internal-flume/docker-compose.yml new file mode 100644 index 0000000..8c1f330 --- /dev/null +++ b/internal-flume/docker-compose.yml @@ -0,0 +1,17 @@ +version: '2' + +services: + flume: + image: 192.168.40.153:9080/tsg/flume:1.9.0 + restart: always + container_name: flume + volumes: + - "/home/tsg3.0-volumes/flume/conf:/home/apache-flume-1.9.0-bin/conf" + - "/home/tsg3.0-volumes/flume/flumeController:/home/apache-flume-1.9.0-bin/flumeController" + - "/home/tsg3.0-volumes/flume/json:/home/test" + - "/home/tsg3.0-volumes/flume/data:/home/data" + - "/home/tsg3.0-volumes/flume/flumeMonitor:/home/apache-flume-1.9.0-bin/flumeMonitor" + - "/etc/localtime:/etc/localtime:ro" + - "/etc/timezone:/etc/timezone:ro" + network_mode: "host" +