From f0206e5a38877393d620f316d962f213cbf97dfd Mon Sep 17 00:00:00 2001 From: zhanghongqing Date: Thu, 25 Nov 2021 15:32:37 +0800 Subject: [PATCH] 21.11 update --- .../TSG-21.09/Galaxy-job-service/admin/logback.xml | 2 +- .../TSG-21.09/Galaxy-job-service/executor/logback.xml | 2 +- .../TSG-21.09/Galaxy-qgw-service/log4j2.xml | 2 +- TSG发布版本更新记录/TSG-21.10/.gitkeep | 0 .../TSG-21.10/Clickhouse/.gitkeep | 0 TSG发布版本更新记录/TSG-21.10/docker/.gitkeep | 0 .../TSG-21.10/job/xxl_job_info | 3 + .../TSG-21.11/hbase/create_table | 5 +- .../TSG-21.11/hbase/hbase-site.xml | 2 +- .../TSG-21.11/nginx/nginx.conf | 88 +++---------------- .../TSG-21.11/qgw/galaxy-qgw-service.yml | 2 +- .../TSG-21.11/record-21.11.txt | 33 ++++--- 12 files changed, 41 insertions(+), 98 deletions(-) delete mode 100644 TSG发布版本更新记录/TSG-21.10/.gitkeep delete mode 100644 TSG发布版本更新记录/TSG-21.10/Clickhouse/.gitkeep delete mode 100644 TSG发布版本更新记录/TSG-21.10/docker/.gitkeep create mode 100644 TSG发布版本更新记录/TSG-21.10/job/xxl_job_info diff --git a/TSG发布版本更新记录/TSG-21.09/Galaxy-job-service/admin/logback.xml b/TSG发布版本更新记录/TSG-21.09/Galaxy-job-service/admin/logback.xml index aa070ad..14b9c4a 100644 --- a/TSG发布版本更新记录/TSG-21.09/Galaxy-job-service/admin/logback.xml +++ b/TSG发布版本更新记录/TSG-21.09/Galaxy-job-service/admin/logback.xml @@ -40,7 +40,7 @@ - + \ No newline at end of file diff --git a/TSG发布版本更新记录/TSG-21.09/Galaxy-job-service/executor/logback.xml b/TSG发布版本更新记录/TSG-21.09/Galaxy-job-service/executor/logback.xml index d55909e..36bfe6b 100644 --- a/TSG发布版本更新记录/TSG-21.09/Galaxy-job-service/executor/logback.xml +++ b/TSG发布版本更新记录/TSG-21.09/Galaxy-job-service/executor/logback.xml @@ -39,7 +39,7 @@ - + \ No newline at end of file diff --git a/TSG发布版本更新记录/TSG-21.09/Galaxy-qgw-service/log4j2.xml b/TSG发布版本更新记录/TSG-21.09/Galaxy-qgw-service/log4j2.xml index ab32848..6e4177b 100644 --- a/TSG发布版本更新记录/TSG-21.09/Galaxy-qgw-service/log4j2.xml +++ b/TSG发布版本更新记录/TSG-21.09/Galaxy-qgw-service/log4j2.xml @@ -49,7 +49,7 @@ - + diff --git a/TSG发布版本更新记录/TSG-21.10/.gitkeep b/TSG发布版本更新记录/TSG-21.10/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/TSG发布版本更新记录/TSG-21.10/Clickhouse/.gitkeep b/TSG发布版本更新记录/TSG-21.10/Clickhouse/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/TSG发布版本更新记录/TSG-21.10/docker/.gitkeep b/TSG发布版本更新记录/TSG-21.10/docker/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/TSG发布版本更新记录/TSG-21.10/job/xxl_job_info b/TSG发布版本更新记录/TSG-21.10/job/xxl_job_info new file mode 100644 index 0000000..d996c96 --- /dev/null +++ b/TSG发布版本更新记录/TSG-21.10/job/xxl_job_info @@ -0,0 +1,3 @@ +-- 修改下面SQL中的url地址为网关地址后，xxl_job数据库执行以下SQL + +INSERT INTO `xxl_job_info` VALUES ('90', '3', '* 10 * * * ?', 'DNS Amplification', '2021-09-22 17:31:10', '2021-11-02 11:46:37', 'galaxy', '', 'FIRST', 'httpToKafkaJobHandler', '{\r\n \"url\":\"http://192.168.44.67:9999/?query=SELECT 1500 as param_s2c_byte_diff, 20 as param_amplification_factor, min(common_start_time) as start_time, max(common_end_time) as end_time, \'DNS Amplification\' as attack_type, \'Major\' as severity, concat(\'> Response \', toString(param_s2c_byte_diff), \' Bytes and Amplification Factor \', toString(param_amplification_factor), \'X\') as conditions, common_server_ip as destination_ip, any(arrayElement(splitByString(\',\',common_server_location), -1)) as destination_country, groupUniqArray(common_client_ip) as source_ip_list, groupUniqArray(arrayElement(splitByString(\',\', common_client_location), -1)) as source_country_list, count(*) as sessions, ROUND(sessions / 600) as session_rate, sum(common_c2s_byte_diff + common_s2c_byte_diff) as bytes, ROUND(bytes * 8 / 600) as bit_rate, sum(common_c2s_pkt_diff + common_s2c_pkt_diff) as packets, ROUND(packets / 600) as packet_rate, sum(common_c2s_byte_diff) as request_bytes, sum(common_s2c_byte_diff) as response_bytes, sum(common_c2s_pkt_diff) request_packets, sum(common_s2c_pkt_diff) as response_packets, round((response_bytes / if(request_bytes > 0, request_bytes, 1)), 2) as byte_ratio from transaction_record where common_recv_time >= toStartOfTenMinutes(now()- INTERVAL 10 minute) and common_recv_time < toStartOfTenMinutes(now()) and common_server_port = 53 and common_schema_type = \'DNS\' and common_s2c_byte_diff > param_s2c_byte_diff and common_c2s_byte_diff > 0 and round((common_s2c_byte_diff / if(common_c2s_byte_diff > 0, common_c2s_byte_diff, 1)), 2) > param_amplification_factor and common_c2s_pkt_diff = 1 and common_s2c_pkt_diff = 1 group by common_server_ip having sessions > 1 order by sessions desc limit 100\",\r\n \"method\":\"get\",\r\n \"requestBody\":{},\r\n \"resultKey\": \"data\",\r\n \"topic\": \"DOS-EVENT\"\r\n}', 'SERIAL_EXECUTION', '0', '0', 'BEAN', '', 'GLUE代码初始化', '2021-09-22 17:31:10', '', '1', '1637802659000', '1637806200000'); diff --git a/TSG发布版本更新记录/TSG-21.11/hbase/create_table b/TSG发布版本更新记录/TSG-21.11/hbase/create_table index 31dbee9..9d4e6a0 100644 --- a/TSG发布版本更新记录/TSG-21.11/hbase/create_table +++ b/TSG发布版本更新记录/TSG-21.11/hbase/create_table @@ -1,9 +1,12 @@ + +-- 1 hbase 创建表 进入 hbase shell 执行以下 + create_namespace 'tsg_galaxy' create 'tsg_galaxy:relation_account_framedip', {NAME => 'radius', VERSIONS => 1} create 'tsg_galaxy:relation_framedip_account', {NAME => 'radius', VERSIONS => 1} --- Phoenix创建视图 +-- 2 Phoenix创建视图 进入 phoenix-hbase-2.2-5.1.2-bin/bin，执行 ./sqlline.py zookeeper1,zookeeper2,zookeeper3 ，然后执行以下命令 CREATE view "tsg"."report_result"( ROWKEY VARCHAR PRIMARY KEY, "detail"."excute_sql" VARCHAR, "detail"."read_rows" UNSIGNED_LONG, "detail"."result_id" UNSIGNED_INT, "response"."result" VARCHAR); CREATE view "tsg_galaxy"."relation_account_framedip" ( ROWKEY VARCHAR PRIMARY KEY, "radius"."framed_ip" VARCHAR, "radius"."account" VARCHAR, "radius"."first_found_time" UNSIGNED_LONG, "radius"."last_update_time" UNSIGNED_LONG); diff --git a/TSG发布版本更新记录/TSG-21.11/hbase/hbase-site.xml b/TSG发布版本更新记录/TSG-21.11/hbase/hbase-site.xml index a3ed1f1..c43d59c 100644 --- a/TSG发布版本更新记录/TSG-21.11/hbase/hbase-site.xml +++ b/TSG发布版本更新记录/TSG-21.11/hbase/hbase-site.xml @@ -1,4 +1,4 @@ - + diff --git a/TSG发布版本更新记录/TSG-21.11/nginx/nginx.conf b/TSG发布版本更新记录/TSG-21.11/nginx/nginx.conf index b878e29..f6dcf40 100644 --- a/TSG发布版本更新记录/TSG-21.11/nginx/nginx.conf +++ b/TSG发布版本更新记录/TSG-21.11/nginx/nginx.conf @@ -10,71 +10,21 @@ http { default_type application/octet-stream; sendfile on; keepalive_timeout 65; - keepalive_requests 10000; - vhost_traffic_status_zone; + vhost_traffic_status_zone; vhost_traffic_status_filter_by_host on; - client_max_body_size 1024M; - client_body_buffer_size 128k; + client_max_body_size 1024M; #(设置客户端请求体最大值) + client_body_buffer_size 128k; #(配置请求体缓存区大小,) access_log off; # 关闭access日志 - - - upstream qgwService { - server 192.168.44.10:8183; - server 192.168.44.13:8183; - } - - upstream jobAdmin { - server 192.168.44.10:8184; - server 192.168.44.13:8184; - } upstream hos { - server 192.168.44.10:8186; - server 192.168.44.13:8186; + server {{ machine_host }}:8186; keepalive 1000; keepalive_timeout 65; keepalive_requests 100000; } geo $islocalip { default 0; - #192.168.40.0/24 1; #添加本地不需要加密的ip，支持掩码形式，格式为xxx.xxx.xxx.xxx 1; - } - upstream nacos { - server 192.168.44.11:8848; - server 192.168.44.14:8848; - server 192.168.44.15:8848; - } - - server { - listen 9999; - server_name localhost; - location / { - proxy_pass http://qgwService; #请求转发到查询引擎集群 - proxy_http_version 1.1; #指定使用http1.1版本 - proxy_read_timeout 21600; #等待后端服务响应的最大时长 - gzip on; #开启压缩 - gzip_comp_level 6; #压缩级别 - gzip_min_length 1k; #启用gzip压缩的最小文件,小于设置值的文件将不会压缩 - gzip_types application/json; #压缩文件类型 - gzip_vary on; #是否传输gzip压缩标志 - } - } - - server { - listen 8181; - server_name localhost; - location / { - proxy_pass http://jobAdmin; - } - } - server { - listen 9913; - server_name localhost; - location /status { - vhost_traffic_status_display; - vhost_traffic_status_display_format html; - } - + {{ allowed_host }}.0/24 1; #添加本地不需要加密的ip，支持掩码形式，格式为xxx.xxx.xxx.xxx 1; } #hos非加密 server { @@ -83,10 +33,10 @@ http { client_max_body_size 1024m; location / { - proxy_http_version 1.1; + proxy_http_version 1.1; proxy_set_header Connection ""; proxy_set_header Host $http_host; - set $ssl 1; #是否开启重定向加密，开启是1，关闭是0，默认关闭 + set $ssl 0; #是否开启重定向加密，开启是1，关闭是0，默认关闭 if ($islocalip = 0){ set $ssl "${ssl}1"; } @@ -106,29 +56,11 @@ http { server_name localhost; proxy_set_header Host $host:9098; client_max_body_size 1024m; - ssl_certificate /usr/local/nginx/conf/server.crt; - ssl_certificate_key /usr/local/nginx/conf/server.key; + ssl_certificate /usr/local/nginx/conf/self-sign.crt; + ssl_certificate_key /usr/local/nginx/conf/self-sign.key; location / { proxy_pass http://hos; } } - server { - listen 8848; - server_name localhost; - location / { - proxy_pass http://nacos; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header REMOTE-HOST $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - } - } - server { - listen 8849 ssl; - server_name localhost; - ssl_certificate /usr/local/nginx/conf/server.crt; - ssl_certificate_key /usr/local/nginx/conf/server.key; - location / { - proxy_pass http://nacos; - } - } } + diff --git a/TSG发布版本更新记录/TSG-21.11/qgw/galaxy-qgw-service.yml b/TSG发布版本更新记录/TSG-21.11/qgw/galaxy-qgw-service.yml index bb99a3a..ce8a76a 100644 --- a/TSG发布版本更新记录/TSG-21.11/qgw/galaxy-qgw-service.yml +++ b/TSG发布版本更新记录/TSG-21.11/qgw/galaxy-qgw-service.yml @@ -1,4 +1,4 @@ -# qgw 修改以下配置 +# qgw 在之前的配置基础上修改以下配置 spring: datasource: diff --git a/TSG发布版本更新记录/TSG-21.11/record-21.11.txt b/TSG发布版本更新记录/TSG-21.11/record-21.11.txt index 434e006..1356f46 100644 --- a/TSG发布版本更新记录/TSG-21.11/record-21.11.txt +++ b/TSG发布版本更新记录/TSG-21.11/record-21.11.txt @@ -1,29 +1,34 @@ 21.11更新记录 hbase -1.Phoenix集成，hbase新增表tsg_galaxy:relation_account_framedip，tsg_galaxy:relation_framedip_account -2.Phoenix创建视图tsg"."report_result，tsg_galaxy"."relation_account_framedip +1.新增hbase表 +2.Phoenix集成到hbase +2.新增Phoenix视图 -ck -1.新增字段 +clickhouse +1.新增字段，进入ck客户端执行SQL文件里的SQL -exporter -1. zk ,ck ,kafka,node-exporter +监控 +1.新增 zookeeper-exporter ,clickhouse-exporter ,kafka-exporter,node-exporter 预处理 - -1.更新jar包及配置 -2.新增了subid任务 +1.更新补全jar包及配置 +2.新增frameip任务 +3.更新dos任务 网关 -更新镜像，schema +1.更新镜像349 +2.更新schema +3.更新配置文件 调度任务 -1.更新了任务，sql执行器url添加/，更新了压缩任务 +1.更新druid压缩任务，执行sql，表xxl_job_group中 address_list字段url最后添加/（如有则忽略） +2.更新镜像galaxy-job-executor:v1.3.211102 flink -1. 更新了checkpoint配置 - +1.更新flink-conf.yml配置 nginx -1. 更新了hos配置 \ No newline at end of file +1. 更新hos配置 + +执行完成后重启所有gohangout \ No newline at end of file