21.11 update

TSG发布版本更新记录/TSG-21.09/Galaxy-job-service/admin/logback.xml

@@ -40,7 +40,7 @@
 
     <root level="${LOG_LEVEL}">
         <appender-ref ref="ALL"/>
-        <appender-ref ref="STDOUT"/>
+<!--        <appender-ref ref="STDOUT"/>-->
     </root>
 
 </configuration>

TSG发布版本更新记录/TSG-21.09/Galaxy-job-service/executor/logback.xml

@@ -39,7 +39,7 @@
 
         <root level="${LOG_LEVEL}">
             <appender-ref ref="ALL"/>
-            <appender-ref ref="STDOUT"/>
+<!--            <appender-ref ref="STDOUT"/>-->
         </root>
 
     </configuration>

TSG发布版本更新记录/TSG-21.09/Galaxy-qgw-service/log4j2.xml

@@ -49,7 +49,7 @@
     </appenders>
     <loggers>
         <root level="${LOG_LEVEL}">
-            <appender-ref ref="consoleSystemOutAppender"/>
+<!--            <appender-ref ref="consoleSystemOutAppender"/>-->
             <appender-ref ref="rollingFileAllAppender"/>
         </root>
     </loggers>

TSG发布版本更新记录/TSG-21.10/job/xxl_job_info

@@ -0,0 +1,3 @@
+-- 修改下面SQL中的url地址为网关地址后，xxl_job数据库执行以下SQL
+
+INSERT INTO `xxl_job_info` VALUES ('90', '3', '* 10 * * * ?', 'DNS Amplification', '2021-09-22 17:31:10', '2021-11-02 11:46:37', 'galaxy', '', 'FIRST', 'httpToKafkaJobHandler', '{\r\n    \"url\":\"http://192.168.44.67:9999/?query=SELECT 1500 as param_s2c_byte_diff, 20 as param_amplification_factor, min(common_start_time) as start_time, max(common_end_time) as end_time, \'DNS Amplification\' as attack_type, \'Major\' as severity, concat(\'> Response \', toString(param_s2c_byte_diff), \' Bytes and Amplification Factor \', toString(param_amplification_factor), \'X\') as conditions, common_server_ip as destination_ip, any(arrayElement(splitByString(\',\',common_server_location), -1)) as destination_country, groupUniqArray(common_client_ip) as source_ip_list, groupUniqArray(arrayElement(splitByString(\',\', common_client_location), -1)) as source_country_list, count(*) as sessions, ROUND(sessions / 600) as session_rate, sum(common_c2s_byte_diff + common_s2c_byte_diff) as bytes, ROUND(bytes * 8 / 600) as bit_rate, sum(common_c2s_pkt_diff + common_s2c_pkt_diff) as packets, ROUND(packets / 600) as packet_rate, sum(common_c2s_byte_diff) as request_bytes, sum(common_s2c_byte_diff) as response_bytes, sum(common_c2s_pkt_diff) request_packets, sum(common_s2c_pkt_diff) as response_packets, round((response_bytes / if(request_bytes > 0, request_bytes, 1)), 2) as byte_ratio from transaction_record where common_recv_time >= toStartOfTenMinutes(now()- INTERVAL 10 minute) and common_recv_time < toStartOfTenMinutes(now()) and common_server_port = 53 and common_schema_type = \'DNS\' and common_s2c_byte_diff > param_s2c_byte_diff and common_c2s_byte_diff > 0 and round((common_s2c_byte_diff / if(common_c2s_byte_diff > 0, common_c2s_byte_diff, 1)), 2) > param_amplification_factor and common_c2s_pkt_diff = 1 and common_s2c_pkt_diff = 1 group by common_server_ip having sessions > 1 order by sessions desc limit 100\",\r\n    \"method\":\"get\",\r\n    \"requestBody\":{},\r\n    \"resultKey\": \"data\",\r\n    \"topic\": \"DOS-EVENT\"\r\n}', 'SERIAL_EXECUTION', '0', '0', 'BEAN', '', 'GLUE代码初始化', '2021-09-22 17:31:10', '', '1', '1637802659000', '1637806200000');

TSG发布版本更新记录/TSG-21.11/hbase/create_table

@@ -1,9 +1,12 @@
+
+-- 1 hbase 创建表 进入 hbase shell 执行以下
+
 create_namespace 'tsg_galaxy'
 
 create 'tsg_galaxy:relation_account_framedip', {NAME => 'radius', VERSIONS => 1}
 create 'tsg_galaxy:relation_framedip_account', {NAME => 'radius', VERSIONS => 1}
 
--- Phoenix创建视图
+-- 2  Phoenix创建视图 进入 phoenix-hbase-2.2-5.1.2-bin/bin，执行 ./sqlline.py zookeeper1,zookeeper2,zookeeper3 ，然后执行以下命令
 CREATE view "tsg"."report_result"( ROWKEY VARCHAR PRIMARY KEY, "detail"."excute_sql" VARCHAR, "detail"."read_rows" UNSIGNED_LONG, "detail"."result_id" UNSIGNED_INT, "response"."result" VARCHAR);
 
 CREATE view "tsg_galaxy"."relation_account_framedip" ( ROWKEY VARCHAR PRIMARY KEY, "radius"."framed_ip" VARCHAR, "radius"."account" VARCHAR, "radius"."first_found_time" UNSIGNED_LONG, "radius"."last_update_time" UNSIGNED_LONG);

TSG发布版本更新记录/TSG-21.11/hbase/hbase-site.xml

@@ -1,4 +1,4 @@
-<!-- hbase hbase-site.xml 新增以下配置,集成Phoenix -->
+<!-- hbase hbase-site.xml 新增以下配置,集成Phoenix ，修改后复制一份到phoenix /bin目录下-->
 
 <configuration>
   <property>

TSG发布版本更新记录/TSG-21.11/nginx/nginx.conf

@@ -10,71 +10,21 @@ http {
     default_type  application/octet-stream;
     sendfile  on;
     keepalive_timeout  65;
-    keepalive_requests 10000;
+    vhost_traffic_status_zone;
-    vhost_traffic_status_zone; 
     vhost_traffic_status_filter_by_host on;
-    client_max_body_size 1024M;
+    client_max_body_size 1024M;  #(设置客户端请求体最大值)
-    client_body_buffer_size 128k;
+    client_body_buffer_size 128k;  #(配置请求体缓存区大小,)
     access_log off; # 关闭access日志
-    
-     
-    upstream qgwService {
-        server 192.168.44.10:8183;
-        server 192.168.44.13:8183;
-    }
-  
-    upstream jobAdmin {
-        server 192.168.44.10:8184;
-        server 192.168.44.13:8184;
-    }
 
     upstream hos {
-        server 192.168.44.10:8186;
+        server {{ machine_host }}:8186;
-        server 192.168.44.13:8186;
         keepalive 1000;
         keepalive_timeout  65;
         keepalive_requests 100000;
     }
     geo $islocalip {
         default 0;
-        #192.168.40.0/24 1; #添加本地不需要加密的ip，支持掩码形式，格式为xxx.xxx.xxx.xxx 1;
+        {{ allowed_host }}.0/24 1; #添加本地不需要加密的ip，支持掩码形式，格式为xxx.xxx.xxx.xxx 1;
-    }
-    upstream nacos {
-        server 192.168.44.11:8848;
-        server 192.168.44.14:8848;
-        server 192.168.44.15:8848;
-    }
-
-    server {
-        listen  9999;
-        server_name  localhost;
-        location / {
-            proxy_pass http://qgwService; #请求转发到查询引擎集群
-            proxy_http_version 1.1; #指定使用http1.1版本
-            proxy_read_timeout 21600; #等待后端服务响应的最大时长
-            gzip on; #开启压缩
-            gzip_comp_level 6; #压缩级别
-            gzip_min_length 1k; #启用gzip压缩的最小文件,小于设置值的文件将不会压缩
-            gzip_types application/json; #压缩文件类型
-            gzip_vary on; #是否传输gzip压缩标志
-        }
-    }
-  
-    server {
-        listen  8181;
-        server_name  localhost;
-        location / {
-            proxy_pass http://jobAdmin;
-        }
-    }
-    server {
-        listen  9913;
-        server_name  localhost;
-        location /status {
-            vhost_traffic_status_display;
-            vhost_traffic_status_display_format html;
-        }
- 
     }
     #hos非加密
     server {
@@ -83,10 +33,10 @@ http {
         client_max_body_size 1024m;
 
         location / {
-	    proxy_http_version 1.1;
+            proxy_http_version 1.1;
             proxy_set_header Connection "";
             proxy_set_header Host $http_host;
-            set $ssl 1; #是否开启重定向加密，开启是1，关闭是0，默认关闭
+            set $ssl 0; #是否开启重定向加密，开启是1，关闭是0，默认关闭
             if ($islocalip = 0){
                 set $ssl "${ssl}1";
             }
@@ -106,29 +56,11 @@ http {
         server_name localhost;
         proxy_set_header Host $host:9098;
         client_max_body_size 1024m;
-        ssl_certificate /usr/local/nginx/conf/server.crt;
+        ssl_certificate /usr/local/nginx/conf/self-sign.crt;
-        ssl_certificate_key /usr/local/nginx/conf/server.key;
+        ssl_certificate_key /usr/local/nginx/conf/self-sign.key;
         location / {
             proxy_pass http://hos;
         }
     }
-    server {
-       listen  8848;
-       server_name  localhost;
-       location / {
-           proxy_pass http://nacos;
-           proxy_set_header X-Real-IP $remote_addr;
-           proxy_set_header REMOTE-HOST $remote_addr;
-           proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
-       }
-    }
-     server {
-        listen 8849 ssl;
-        server_name localhost;
-        ssl_certificate /usr/local/nginx/conf/server.crt;
-        ssl_certificate_key /usr/local/nginx/conf/server.key;
-        location / {
-            proxy_pass http://nacos;
-        }
-    }
 }
+

TSG发布版本更新记录/TSG-21.11/qgw/galaxy-qgw-service.yml

@@ -1,4 +1,4 @@
-# qgw 修改以下配置
+# qgw 在之前的配置基础上修改以下配置
 
 spring:
   datasource:

TSG发布版本更新记录/TSG-21.11/record-21.11.txt

@@ -1,29 +1,34 @@
 21.11更新记录
 
 hbase
-1.Phoenix集成，hbase新增表tsg_galaxy:relation_account_framedip，tsg_galaxy:relation_framedip_account
+1.新增hbase表
-2.Phoenix创建视图tsg"."report_result，tsg_galaxy"."relation_account_framedip
+2.Phoenix集成到hbase
+2.新增Phoenix视图
 
-ck
+clickhouse
-1.新增字段
+1.新增字段，进入ck客户端执行SQL文件里的SQL
 
-exporter
+监控
-1. zk ,ck ,kafka,node-exporter
+1.新增 zookeeper-exporter ,clickhouse-exporter ,kafka-exporter,node-exporter
 
 预处理
-
+1.更新补全jar包及配置
-1.更新jar包及配置
+2.新增frameip任务
-2.新增了subid任务
+3.更新dos任务
 
 网关
-更新镜像，schema
+1.更新镜像349
+2.更新schema
+3.更新配置文件
 
 调度任务
-1.更新了任务，sql执行器url添加/，更新了压缩任务
+1.更新druid压缩任务，执行sql，表xxl_job_group中 address_list字段url最后添加/（如有则忽略）
+2.更新镜像galaxy-job-executor:v1.3.211102
 
 flink
-1. 更新了checkpoint配置
+1.更新flink-conf.yml配置
-
 
 nginx
-1. 更新了hos配置
+1. 更新hos配置
+
+执行完成后重启所有gohangout