gfwleak/galaxy-tsg-olap-log-stream-doublewrite
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat:adapt to percent log structure

Browse Source
This commit is contained in:
wangchengcheng
2024-01-18 19:31:05 +08:00
parent 68b4805c4f
commit 4cda43724d
17 changed files with 468 additions and 153 deletions
Download Patch File Download Diff File Expand all files Collapse all files

35
properties/action_definition.properties Normal file
View File

@@ -0,0 +1,35 @@
none=0
Monitor=1
monitor=1
Intercept=2
intercept=2
NoIntercept=3
nointercept=3
ActiveDefence=4
activedefence=4
WANNAT=8
wannat=8
Reject=16
reject=16
Deny=16
deny=16
Shaping=32
shaping=32
Manipulate=48
manipulate=48
ServiceChaining=64
servicechaining=64
Allow=96
allow=96
Bypass=96
bypass=96
Shunt=128
shunt=128
Statistics=129
statistics=129
redirect=48
replace=48
hijack=48
insert=48
edit_element=48
run_script=48

68
properties/percent_proxy_event.properties Normal file
View File

@@ -0,0 +1,68 @@
#session-record
tcp_c2s_ip_fragments=common_c2s_ipfrag_num
tcp_s2c_ip_fragments=common_s2c_ipfrag_num
tcp_c2s_o3_pkts=common_c2s_tcp_unorder_num
tcp_s2c_o3_pkts=common_s2c_tcp_unorder_num
http_response_latency_ms=http_response_latency_ms
http_session_duration_ms=http_session_duration_ms
security_rule_list=security_rule_list
monitor_rule_list=monitor_rule_list
tcp_handshake_latency_ms=common_establish_latency_ms
mail_protocol_type=mail_protocol_type
mail_account=mail_account
mail_password=mail_passwd
mail_from_cmd=mail_from_cmd
mail_to_cmd=mail_to_cmd
mail_from=mail_from
mail_to=mail_to
mail_cc=mail_cc
mail_bcc=mail_bcc
mail_subject=mail_subject
mail_subject_charset=mail_subject_charset
mail_attachment_name=mail_attachment_name
mail_attachment_name_charset=mail_attachment_name_charset
mail_eml_file=mail_eml_file
dns_message_id=dns_message_id
dns_qr=dns_qr
dns_opcode=dns_opcode
dns_aa=dns_aa
dns_tc=dns_tc
dns_rd=dns_rd
dns_ra=dns_ra
dns_rcode=dns_rcode
dns_qdcount=dns_qdcount
dns_ancount=dns_ancount
dns_nscount=dns_nscount
dns_arcount=dns_arcount
dns_qname=dns_qname
dns_qtype=dns_qtype
dns_qclass=dns_qclass
dns_cname=dns_cname
dns_sub=dns_sub
dns_rr=dns_rr
ssl_version=ssl_version
ssl_sni=ssl_sni
ssl_san=ssl_san
ssl_cn=ssl_cn
ssl_handshake_latency_ms=ssl_con_latency_ms
ssl_ja3_hash=ssl_ja3_hash
ssl_cert_issuer=ssl_cert_issuer
ssl_cert_subject=ssl_cert_subject
quic_version=quic_version
quic_sni=quic_sni
quic_user_agent=quic_user_agent
ftp_account=ftp_account
ftp_url=ftp_url
ftp_link_type=ftp_link_type
http_proxy_flag=http_proxy_flag
http_sequence=http_sequence
tcp_client_isn=common_tcp_client_isn
tcp_server_isn=common_tcp_server_isn
sent_pkts=common_c2s_pkt_num
received_pkts=common_s2c_pkt_num
app=common_app_label
out_link_id=common_egress_link_id
in_link_id=common_ingress_link_id
duration_ms=common_con_duration_ms
http_request_line=http_request_line
http_response_line=http_response_line

4
properties/percent_security_event.properties Normal file
View File

@@ -0,0 +1,4 @@
tcp_c2s_ip_fragments=common_c2s_ipfrag_num
tcp_s2c_ip_fragments=common_s2c_ipfrag_num
tcp_c2s_o3_pkts=common_c2s_tcp_unorder_num
tcp_s2c_o3_pkts=common_s2c_tcp_unorder_num

42
properties/percent_session_record.properties Normal file
View File

@@ -0,0 +1,42 @@
#security-event
http_request_body=http_request_body
http_response_body=http_response_body
http_response_latency_ms=http_response_latency_ms
http_session_duration_ms=http_session_duration_ms
security_rule_list=security_rule_list
monitor_rule_list=monitor_rule_list
tcp_handshake_latency_ms=common_establish_latency_ms
#proxy-event
http_action_file_size=http_action_file_size
doh_url=doh_url
doh_host=doh_host
doh_cookie=doh_cookie
doh_referer=doh_referer
doh_user_agent=doh_user_agent
doh_version=doh_version
doh_message_id=doh_message_id
doh_qr=doh_qr
doh_opcode=doh_opcode
doh_aa=doh_aa
doh_tc=doh_tc
doh_rd=doh_rd
doh_ra=doh_ra
doh_rcode=doh_rcode
doh_qdcount=doh_qdcount
doh_ancount=doh_ancount
doh_nscount=doh_nscount
doh_arcount=doh_arcount
doh_qname=doh_qname
doh_qtype=doh_qtype
doh_qclass=doh_qclass
doh_cname=doh_cname
doh_sub=doh_sub
doh_rr=doh_rr
proxy_rule_list=proxy_rule_list

68
properties/proxy_event_mapping_table.properties
View File

@@ -1,68 +0,0 @@
recv_time=common_recv_time
log_id=common_log_id
start_timestamp_ms=common_start_timestamp_ms
end_timestamp_ms=common_end_timestamp_ms
processing_time=common_processing_time
device_id=common_device_id
data_center=common_data_center
sled_ip=common_sled_ip
device_tag=common_device_tag
client_ip=common_client_ip
client_port=common_client_port
client_asn=common_client_asn
subscriber_id=common_subscriber_id
imei=common_imei
imsi=common_imsi
phone_number=common_phone_number
server_ip=common_server_ip
server_port=common_server_port
server_asn=common_server_asn
address_type=common_address_type
http_url=http_url
http_host=http_host
http_request_line=http_request_line
http_response_line=http_response_line
http_request_body=http_request_body
http_response_body=http_response_body
http_cookie=http_cookie
http_referer=http_referer
http_user_agent=http_user_agent
http_request_content_length=http_request_content_length
http_request_content_type=http_request_content_type
http_response_content_length=http_response_content_length
http_response_content_type=http_response_content_type
http_set_cookie=http_set_cookie
http_version=http_version
http_action_file_size=http_action_file_size
doh_url=doh_url
doh_host=doh_host
doh_cookie=doh_cookie
doh_referer=doh_referer
doh_user_agent=doh_user_agent
doh_version=doh_version
doh_message_id=doh_message_id
doh_qr=doh_qr
doh_opcode=doh_opcode
doh_aa=doh_aa
doh_tc=doh_tc
doh_rd=doh_rd
doh_ra=doh_ra
doh_rcode=doh_rcode
doh_qdcount=doh_qdcount
doh_ancount=doh_ancount
doh_nscount=doh_nscount
doh_arcount=doh_arcount
doh_qname=doh_qname
doh_qtype=doh_qtype
doh_qclass=doh_qclass
doh_cname=doh_cname
doh_sub=doh_sub
doh_rr=doh_rr
client_geolocation=common_client_location
server_geolocation=common_server_location
ip_protocol=common_l4_protocol
sent_bytes=common_c2s_byte_num
received_bytes=common_s2c_byte_num
decoded_as=common_schema_type
proxy_rule_list=proxy_rule_list
session_id=common_stream_trace_id

19
properties/service_flow_config.properties
View File

@@ -1,7 +1,7 @@
#--------------------------------地址配置------------------------------#
#管理kafka地址
#source.kafka.servers=192.168.44.12:9094
source.kafka.servers=192.168.44.11:9094,192.168.44.14:9094,192.168.44.15:9094
source.kafka.servers=192.168.44.12:9094
#source.kafka.servers=192.168.44.11:9094,192.168.44.14:9094,192.168.44.15:9094
#百分点输出kafka地址
percent.sink.kafka.servers=192.168.44.12:9094
#文件源数据topic输出kafka地址
@@ -24,14 +24,16 @@ nacos.server=192.168.44.12:8848
nacos.schema.namespace=P19
#schema data id名称
nacos.schema.data.id=proxy_event.json
nacos.schema.data.id=session_record.json
#--------------------------------Kafka消费/生产配置------------------------------#
#kafka 接收数据topic
source.kafka.topic=SESSION-RECORD
sink.percent.kafka.topic=PERCENT-RECORD
sink.percent.kafka.topic.session=PERCENT-SESSION-RECORD
sink.percent.kafka.topic.security=PERCENT-SECURITY-RECORD
sink.percent.kafka.topic.proxy=PERCENT-POLICY-RECORD
sink.file.data.kafka.topic=test-file-data
#读取topic,存储该spout id的消费offset信息，可通过该拓扑命名;具体存储offset的位置，确定下次读取不重复的数据；
@@ -47,17 +49,13 @@ transform.parallelism=1
deal.file.parallelism=1
sink.file.data.parallelism=1
sink.percent.parallelism=1
#数据中心，取值范围(0-31)
data.center.id.num=0
#hbase 更新时间，如填写0则不更新缓存
hbase.tick.tuple.freq.secs=180
#--------------------------------默认值配置------------------------------#
#生产者压缩模式 none or snappy
producer.kafka.compression.type=snappy
#------------------------------------OOS配置------------------------------------#
#oos地址
oos.servers=10.3.45.100:8057
@@ -65,15 +63,14 @@ oos.servers=10.3.45.100:8057
prometheus.pushgateway.address=192.168.44.12:9091
pushgateway.statistics.time=300
deal.file.statistics.time=60
#------------------------------------knowledge配置------------------------------------#
knowledge.execution.minutes=600
knowledge.base.uri=http://192.168.44.67:9999
knowledge.base.uri=http://192.168.44.12:9999
knowledge.base.path=/v1/knowledge_base
ip.user.defined.kd.id=004390bc-3135-4a6f-a492-3662ecb9e289
ip.builtin.kd.id=64af7077-eb9b-4b8f-80cf-2ceebc89bea9
asn.builtin.kd.id=f9f6bc91-2142-4673-8249-e097c00fe1ea
hos.url=http://192.168.44.67:9098/hos/traffic_file_bucket/
hos.url=http://192.168.44.12:9098/hos/traffic_file_bucket/

81
properties/security_event_mapping_table.properties → properties/session_record_mapping_table.properties
View File

@@ -19,10 +19,22 @@ server_ip=common_server_ip
server_port=common_server_port
server_asn=common_server_asn
address_type=common_address_type
out_link_id=common_egress_link_id
in_link_id=common_ingress_link_id
client_geolocation=common_client_location
server_geolocation=common_server_location
app=common_app_label
ip_protocol=common_l4_protocol
sent_pkts=common_c2s_pkt_num
received_pkts=common_s2c_pkt_num
sent_bytes=common_c2s_byte_num
received_bytes=common_s2c_byte_num
tcp_client_isn=common_tcp_client_isn
tcp_server_isn=common_tcp_server_isn
decoded_as=common_schema_type
session_id=common_stream_trace_id
http_url=http_url
http_host=http_host
http_request_body=http_request_body
http_response_body=http_response_body
http_proxy_flag=http_proxy_flag
http_sequence=http_sequence
http_cookie=http_cookie
@@ -34,8 +46,6 @@ http_response_content_length=http_response_content_length
http_response_content_type=http_response_content_type
http_set_cookie=http_set_cookie
http_version=http_version
http_response_latency_ms=http_response_latency_ms
http_session_duration_ms=http_session_duration_ms
mail_protocol_type=mail_protocol_type
mail_account=mail_account
mail_password=mail_passwd
@@ -68,26 +78,6 @@ dns_qclass=dns_qclass
dns_cname=dns_cname
dns_sub=dns_sub
dns_rr=dns_rr
quic_version=quic_version
quic_sni=quic_sni
quic_user_agent=quic_user_agent
ftp_account=ftp_account
ftp_url=ftp_url
ftp_link_type=ftp_link_type
out_link_id=common_egress_link_id
in_link_id=common_ingress_link_id
client_geolocation=common_client_location
server_geolocation=common_server_location
app=common_app_label
ip_protocol=common_l4_protocol
sent_pkts=common_c2s_pkt_num
received_pkts=common_s2c_pkt_num
sent_bytes=common_c2s_byte_num
received_bytes=common_s2c_byte_num
tcp_client_isn=common_tcp_client_isn
tcp_server_isn=common_tcp_server_isn
decoded_as=common_schema_type
session_id=common_stream_trace_id
ssl_version=ssl_version
ssl_sni=ssl_sni
ssl_san=ssl_san
@@ -96,6 +86,49 @@ ssl_handshake_latency_ms=ssl_con_latency_ms
ssl_ja3_hash=ssl_ja3_hash
ssl_cert_issuer=ssl_cert_issuer
ssl_cert_subject=ssl_cert_subject
quic_version=quic_version
quic_sni=quic_sni
quic_user_agent=quic_user_agent
ftp_account=ftp_account
ftp_url=ftp_url
ftp_link_type=ftp_link_type
#security-event
http_request_body=http_request_body
http_response_body=http_response_body
http_response_latency_ms=http_response_latency_ms
http_session_duration_ms=http_session_duration_ms
security_rule_list=security_rule_list
monitor_rule_list=monitor_rule_list
tcp_handshake_latency_ms=common_establish_latency_ms
#proxy-event
http_action_file_size=http_action_file_size
doh_url=doh_url
doh_host=doh_host
doh_cookie=doh_cookie
doh_referer=doh_referer
doh_user_agent=doh_user_agent
doh_version=doh_version
doh_message_id=doh_message_id
doh_qr=doh_qr
doh_opcode=doh_opcode
doh_aa=doh_aa
doh_tc=doh_tc
doh_rd=doh_rd
doh_ra=doh_ra
doh_rcode=doh_rcode
doh_qdcount=doh_qdcount
doh_ancount=doh_ancount
doh_nscount=doh_nscount
doh_arcount=doh_arcount
doh_qname=doh_qname
doh_qtype=doh_qtype
doh_qclass=doh_qclass
doh_cname=doh_cname
doh_sub=doh_sub
doh_rr=doh_rr
proxy_rule_list=proxy_rule_list
#session-record
tcp_c2s_ip_fragments=common_c2s_ipfrag_num
tcp_s2c_ip_fragments=common_s2c_ipfrag_num
tcp_c2s_o3_pkts=common_c2s_tcp_unorder_num
tcp_s2c_o3_pkts=common_s2c_tcp_unorder_num