21.11 update
This commit is contained in:
zhanghongqing
@@ -40,7 +40,7 @@
|
||||
|
||||
<root level="${LOG_LEVEL}">
|
||||
<appender-ref ref="ALL"/>
|
||||
<appender-ref ref="STDOUT"/>
|
||||
<!-- <appender-ref ref="STDOUT"/>-->
|
||||
</root>
|
||||
|
||||
</configuration>
|
||||
@@ -39,7 +39,7 @@
|
||||
|
||||
<root level="${LOG_LEVEL}">
|
||||
<appender-ref ref="ALL"/>
|
||||
<appender-ref ref="STDOUT"/>
|
||||
<!-- <appender-ref ref="STDOUT"/>-->
|
||||
</root>
|
||||
|
||||
</configuration>
|
||||
@@ -49,7 +49,7 @@
|
||||
</appenders>
|
||||
<loggers>
|
||||
<root level="${LOG_LEVEL}">
|
||||
<appender-ref ref="consoleSystemOutAppender"/>
|
||||
<!-- <appender-ref ref="consoleSystemOutAppender"/>-->
|
||||
<appender-ref ref="rollingFileAllAppender"/>
|
||||
</root>
|
||||
</loggers>
|
||||
|
||||
3
TSG发布版本更新记录/TSG-21.10/job/xxl_job_info
Normal file
3
TSG发布版本更新记录/TSG-21.10/job/xxl_job_info
Normal file
@@ -0,0 +1,3 @@
|
||||
-- 修改下面SQL中的url地址为网关地址后，xxl_job数据库执行以下SQL
|
||||
|
||||
INSERT INTO `xxl_job_info` VALUES ('90', '3', '* 10 * * * ?', 'DNS Amplification', '2021-09-22 17:31:10', '2021-11-02 11:46:37', 'galaxy', '', 'FIRST', 'httpToKafkaJobHandler', '{\r\n \"url\":\"http://192.168.44.67:9999/?query=SELECT 1500 as param_s2c_byte_diff, 20 as param_amplification_factor, min(common_start_time) as start_time, max(common_end_time) as end_time, \'DNS Amplification\' as attack_type, \'Major\' as severity, concat(\'> Response \', toString(param_s2c_byte_diff), \' Bytes and Amplification Factor \', toString(param_amplification_factor), \'X\') as conditions, common_server_ip as destination_ip, any(arrayElement(splitByString(\',\',common_server_location), -1)) as destination_country, groupUniqArray(common_client_ip) as source_ip_list, groupUniqArray(arrayElement(splitByString(\',\', common_client_location), -1)) as source_country_list, count(*) as sessions, ROUND(sessions / 600) as session_rate, sum(common_c2s_byte_diff + common_s2c_byte_diff) as bytes, ROUND(bytes * 8 / 600) as bit_rate, sum(common_c2s_pkt_diff + common_s2c_pkt_diff) as packets, ROUND(packets / 600) as packet_rate, sum(common_c2s_byte_diff) as request_bytes, sum(common_s2c_byte_diff) as response_bytes, sum(common_c2s_pkt_diff) request_packets, sum(common_s2c_pkt_diff) as response_packets, round((response_bytes / if(request_bytes > 0, request_bytes, 1)), 2) as byte_ratio from transaction_record where common_recv_time >= toStartOfTenMinutes(now()- INTERVAL 10 minute) and common_recv_time < toStartOfTenMinutes(now()) and common_server_port = 53 and common_schema_type = \'DNS\' and common_s2c_byte_diff > param_s2c_byte_diff and common_c2s_byte_diff > 0 and round((common_s2c_byte_diff / if(common_c2s_byte_diff > 0, common_c2s_byte_diff, 1)), 2) > param_amplification_factor and common_c2s_pkt_diff = 1 and common_s2c_pkt_diff = 1 group by common_server_ip having sessions > 1 order by sessions desc limit 100\",\r\n \"method\":\"get\",\r\n \"requestBody\":{},\r\n \"resultKey\": \"data\",\r\n \"topic\": \"DOS-EVENT\"\r\n}', 'SERIAL_EXECUTION', '0', '0', 'BEAN', '', 'GLUE代码初始化', '2021-09-22 17:31:10', '', '1', '1637802659000', '1637806200000');
|
||||
@@ -1,9 +1,12 @@
|
||||
|
||||
-- 1 hbase 创建表 进入 hbase shell 执行以下
|
||||
|
||||
create_namespace 'tsg_galaxy'
|
||||
|
||||
create 'tsg_galaxy:relation_account_framedip', {NAME => 'radius', VERSIONS => 1}
|
||||
create 'tsg_galaxy:relation_framedip_account', {NAME => 'radius', VERSIONS => 1}
|
||||
|
||||
-- Phoenix创建视图
|
||||
-- 2 Phoenix创建视图 进入 phoenix-hbase-2.2-5.1.2-bin/bin，执行 ./sqlline.py zookeeper1,zookeeper2,zookeeper3 ，然后执行以下命令
|
||||
CREATE view "tsg"."report_result"( ROWKEY VARCHAR PRIMARY KEY, "detail"."excute_sql" VARCHAR, "detail"."read_rows" UNSIGNED_LONG, "detail"."result_id" UNSIGNED_INT, "response"."result" VARCHAR);
|
||||
|
||||
CREATE view "tsg_galaxy"."relation_account_framedip" ( ROWKEY VARCHAR PRIMARY KEY, "radius"."framed_ip" VARCHAR, "radius"."account" VARCHAR, "radius"."first_found_time" UNSIGNED_LONG, "radius"."last_update_time" UNSIGNED_LONG);
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
<!-- hbase hbase-site.xml 新增以下配置,集成Phoenix -->
|
||||
<!-- hbase hbase-site.xml 新增以下配置,集成Phoenix ，修改后复制一份到phoenix /bin目录下-->
|
||||
|
||||
<configuration>
|
||||
<property>
|
||||
|
||||
@@ -10,71 +10,21 @@ http {
|
||||
default_type application/octet-stream;
|
||||
sendfile on;
|
||||
keepalive_timeout 65;
|
||||
keepalive_requests 10000;
|
||||
vhost_traffic_status_zone;
|
||||
vhost_traffic_status_zone;
|
||||
vhost_traffic_status_filter_by_host on;
|
||||
client_max_body_size 1024M;
|
||||
client_body_buffer_size 128k;
|
||||
client_max_body_size 1024M; #(设置客户端请求体最大值)
|
||||
client_body_buffer_size 128k; #(配置请求体缓存区大小,)
|
||||
access_log off; # 关闭access日志
|
||||
|
||||
|
||||
upstream qgwService {
|
||||
server 192.168.44.10:8183;
|
||||
server 192.168.44.13:8183;
|
||||
}
|
||||
|
||||
upstream jobAdmin {
|
||||
server 192.168.44.10:8184;
|
||||
server 192.168.44.13:8184;
|
||||
}
|
||||
|
||||
upstream hos {
|
||||
server 192.168.44.10:8186;
|
||||
server 192.168.44.13:8186;
|
||||
server {{ machine_host }}:8186;
|
||||
keepalive 1000;
|
||||
keepalive_timeout 65;
|
||||
keepalive_requests 100000;
|
||||
}
|
||||
geo $islocalip {
|
||||
default 0;
|
||||
#192.168.40.0/24 1; #添加本地不需要加密的ip，支持掩码形式，格式为xxx.xxx.xxx.xxx 1;
|
||||
}
|
||||
upstream nacos {
|
||||
server 192.168.44.11:8848;
|
||||
server 192.168.44.14:8848;
|
||||
server 192.168.44.15:8848;
|
||||
}
|
||||
|
||||
server {
|
||||
listen 9999;
|
||||
server_name localhost;
|
||||
location / {
|
||||
proxy_pass http://qgwService; #请求转发到查询引擎集群
|
||||
proxy_http_version 1.1; #指定使用http1.1版本
|
||||
proxy_read_timeout 21600; #等待后端服务响应的最大时长
|
||||
gzip on; #开启压缩
|
||||
gzip_comp_level 6; #压缩级别
|
||||
gzip_min_length 1k; #启用gzip压缩的最小文件,小于设置值的文件将不会压缩
|
||||
gzip_types application/json; #压缩文件类型
|
||||
gzip_vary on; #是否传输gzip压缩标志
|
||||
}
|
||||
}
|
||||
|
||||
server {
|
||||
listen 8181;
|
||||
server_name localhost;
|
||||
location / {
|
||||
proxy_pass http://jobAdmin;
|
||||
}
|
||||
}
|
||||
server {
|
||||
listen 9913;
|
||||
server_name localhost;
|
||||
location /status {
|
||||
vhost_traffic_status_display;
|
||||
vhost_traffic_status_display_format html;
|
||||
}
|
||||
|
||||
{{ allowed_host }}.0/24 1; #添加本地不需要加密的ip，支持掩码形式，格式为xxx.xxx.xxx.xxx 1;
|
||||
}
|
||||
#hos非加密
|
||||
server {
|
||||
@@ -83,10 +33,10 @@ http {
|
||||
client_max_body_size 1024m;
|
||||
|
||||
location / {
|
||||
proxy_http_version 1.1;
|
||||
proxy_http_version 1.1;
|
||||
proxy_set_header Connection "";
|
||||
proxy_set_header Host $http_host;
|
||||
set $ssl 1; #是否开启重定向加密，开启是1，关闭是0，默认关闭
|
||||
set $ssl 0; #是否开启重定向加密，开启是1，关闭是0，默认关闭
|
||||
if ($islocalip = 0){
|
||||
set $ssl "${ssl}1";
|
||||
}
|
||||
@@ -106,29 +56,11 @@ http {
|
||||
server_name localhost;
|
||||
proxy_set_header Host $host:9098;
|
||||
client_max_body_size 1024m;
|
||||
ssl_certificate /usr/local/nginx/conf/server.crt;
|
||||
ssl_certificate_key /usr/local/nginx/conf/server.key;
|
||||
ssl_certificate /usr/local/nginx/conf/self-sign.crt;
|
||||
ssl_certificate_key /usr/local/nginx/conf/self-sign.key;
|
||||
location / {
|
||||
proxy_pass http://hos;
|
||||
}
|
||||
}
|
||||
server {
|
||||
listen 8848;
|
||||
server_name localhost;
|
||||
location / {
|
||||
proxy_pass http://nacos;
|
||||
proxy_set_header X-Real-IP $remote_addr;
|
||||
proxy_set_header REMOTE-HOST $remote_addr;
|
||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||
}
|
||||
}
|
||||
server {
|
||||
listen 8849 ssl;
|
||||
server_name localhost;
|
||||
ssl_certificate /usr/local/nginx/conf/server.crt;
|
||||
ssl_certificate_key /usr/local/nginx/conf/server.key;
|
||||
location / {
|
||||
proxy_pass http://nacos;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
# qgw 修改以下配置
|
||||
# qgw 在之前的配置基础上修改以下配置
|
||||
|
||||
spring:
|
||||
datasource:
|
||||
|
||||
@@ -1,29 +1,34 @@
|
||||
21.11更新记录
|
||||
|
||||
hbase
|
||||
1.Phoenix集成，hbase新增表tsg_galaxy:relation_account_framedip，tsg_galaxy:relation_framedip_account
|
||||
2.Phoenix创建视图tsg"."report_result，tsg_galaxy"."relation_account_framedip
|
||||
1.新增hbase表
|
||||
2.Phoenix集成到hbase
|
||||
2.新增Phoenix视图
|
||||
|
||||
ck
|
||||
1.新增字段
|
||||
clickhouse
|
||||
1.新增字段，进入ck客户端执行SQL文件里的SQL
|
||||
|
||||
exporter
|
||||
1. zk ,ck ,kafka,node-exporter
|
||||
监控
|
||||
1.新增 zookeeper-exporter ,clickhouse-exporter ,kafka-exporter,node-exporter
|
||||
|
||||
预处理
|
||||
|
||||
1.更新jar包及配置
|
||||
2.新增了subid任务
|
||||
1.更新补全jar包及配置
|
||||
2.新增frameip任务
|
||||
3.更新dos任务
|
||||
|
||||
网关
|
||||
更新镜像，schema
|
||||
1.更新镜像349
|
||||
2.更新schema
|
||||
3.更新配置文件
|
||||
|
||||
调度任务
|
||||
1.更新了任务，sql执行器url添加/，更新了压缩任务
|
||||
1.更新druid压缩任务，执行sql，表xxl_job_group中 address_list字段url最后添加/（如有则忽略）
|
||||
2.更新镜像galaxy-job-executor:v1.3.211102
|
||||
|
||||
flink
|
||||
1. 更新了checkpoint配置
|
||||
|
||||
1.更新flink-conf.yml配置
|
||||
|
||||
nginx
|
||||
1. 更新了hos配置
|
||||
1. 更新hos配置
|
||||
|
||||
执行完成后重启所有gohangout
|
||||
Reference in New Issue
Block a user