20.11.rc3 rebase version 20.11

roles/firewall/tasks/main.yml

@@ -11,21 +11,22 @@
     skip_broken: yes
   vars:
     fw_packages:
-      - /tmp/ansible_deploy/capture_packet_plug-3.0.4.42574b7-2.el7.x86_64.rpm
+      - /tmp/ansible_deploy/capture_packet_plug-3.0.6.a2db4a4-2.el7.x86_64.rpm
       - /tmp/ansible_deploy/conn_telemetry-1.0.2.8d6da43-2.el7.x86_64.rpm
       - /tmp/ansible_deploy/dns-2.0.9.b639626-2.el7.x86_64.rpm
       - /tmp/ansible_deploy/ftp-1.0.8.13d5fda-2.el7.x86_64.rpm
       - /tmp/ansible_deploy/fw_dns_plug-3.0.2.dab58fa-2.el7.x86_64.rpm
       - /tmp/ansible_deploy/fw_ftp_plug-3.0.1.0a78573-2.el7.x86_64.rpm
-      - /tmp/ansible_deploy/fw_http_plug-3.0.1.0c7e082-2.el7.x86_64.rpm
-      - /tmp/ansible_deploy/fw_mail_plug-3.0.1.02465eb-2.el7.x86_64.rpm
-      - /tmp/ansible_deploy/fw_quic_plug-3.0.1.b790ee1-2.el7.x86_64.rpm
-      - /tmp/ansible_deploy/fw_ssl_plug-3.0.4.a0b19ee-2.el7.x86_64.rpm
+      - /tmp/ansible_deploy/fw_http_plug-3.0.4.484b54d-2.el7.x86_64.rpm
+      - /tmp/ansible_deploy/fw_mail_plug-3.0.2.7401550-2.el7.x86_64.rpm
+      - /tmp/ansible_deploy/fw_quic_plug-3.0.4.947ef77-2.el7.x86_64.rpm
+      - /tmp/ansible_deploy/fw_ssl_plug-3.0.6.a121701-2.el7.x86_64.rpm
       - /tmp/ansible_deploy/http-2.0.5.c61ad9a-2.el7.x86_64.rpm
       - /tmp/ansible_deploy/mail-1.0.9.c1d3bde-2.el7.x86_64.rpm
-      - /tmp/ansible_deploy/quic-1.1.10.c2b90a0-2.el7.x86_64.rpm
-      - /tmp/ansible_deploy/ssl-1.0.9.69f3742-2.el7.x86_64.rpm
-      - /tmp/ansible_deploy/tsg_conn_sketch-2.0.6.abb4f4d-2.el7.x86_64.rpm
+      - /tmp/ansible_deploy/quic-1.1.17.8c22b4d-2.el7.x86_64.rpm
+      - /tmp/ansible_deploy/ssl-1.0.12.16b8fb5-2.el7.x86_64.rpm
+      - /tmp/ansible_deploy/tsg_conn_sketch-2.0.12.0ad5a3b-2.el7.x86_64.rpm
+      - /tmp/ansible_deploy/app_control_plug-1.0.9.97846eb-2.el7.x86_64.rpm
 
 - name: "Template the tsgconf/main.conf"
   template:
@@ -40,14 +41,20 @@
     dest: /home/mesasoft/sapp_run/tsgconf/maat.conf
   tags: template
 
-- name: "Template the tsgconf/tsg_log_field.conf"
-  template:
-    src: "{{ role_path }}/templates/tsg_log_field.conf.j2"
-    dest: /home/mesasoft/sapp_run/tsgconf/tsg_log_field.conf
-  tags: template
-
 - name: "Template the conf/capture_packet_plug.conf.j2"
   template:
     src: "{{ role_path }}/templates/capture_packet_plug.conf.j2"
     dest: /home/mesasoft/sapp_run/conf/capture_packet_plug.conf
   tags: template
+
+- name: "Template the tsgconf/app_l7_proto_id.conf"
+  template:
+    src: "{{ role_path }}/templates/app_l7_proto_id.conf.j2"
+    dest: /home/mesasoft/sapp_run/tsgconf/app_l7_proto_id.conf
+ 
+- name: "Template the /home/mesasoft/sapp_run/plug/business/tsg_conn_sketch/tsg_conn_sketch.inf"
+  template:
+    src: "{{ role_path }}/templates/tsg_conn_sketch.inf.j2"
+    dest: /home/mesasoft/sapp_run/plug/business/tsg_conn_sketch/tsg_conn_sketch.inf
+  tags: template
+

roles/firewall/templates/app_l7_proto_id.conf.j2

@@ -0,0 +1,51 @@
+#TYPE：1:UCHAR,2:USHORT,3:USTRING,4:ULOG,5:USTRING,6:FILE,7:UBASE64,8:PACKET
+#TYPE	FIELD		VALUE
+STRING	UNCATEGORIZED	100
+STRING	UNCATEGORIZED	101
+STRING	UNKNOWN_OTHER	102
+STRING	DNS		103
+STRING	FTP		104
+STRING	FTPS		105
+STRING	HTTP		106
+STRING	HTTPS		107
+STRING	ICMP		108
+STRING	IKE		109
+STRING	MAIL		110
+STRING	IMAPS		111
+STRING	IPSEC		112
+STRING	XMPP		113
+STRING	L2TP		114
+STRING	NTP		115
+STRING	POP3S		117
+STRING	PPTP		118
+STRING	QUIC		119
+STRING	SIP		120
+STRING	SMB		121
+STRING	SMTPS		123
+STRING	SPDY		124
+STRING	SSH		125
+STRING	SSL		126
+STRING	SOCKS		127
+STRING	TELNET		128
+STRING	DHCP		129
+STRING	RADIUS		130
+STRING	OPENVPN		131
+STRING	STUN		132
+STRING	TEREDO		133
+STRING	DTLS		134
+STRING	DoH		135
+STRING	ISAKMP		136
+STRING	MDNS		137
+STRING	NETBIOS		138
+STRING	NETFLOW		139
+STRING	RDP		140
+STRING	RTCP		141
+STRING	RTP		142
+STRING	SLP		143
+STRING	SNMP		144
+STRING	SSDP		145
+STRING	TFTP		146
+STRING	BJNP		147
+STRING	LDAP		148
+STRING	RTMP		149
+STRING	RTSP		150

roles/firewall/templates/capture_packet_plug.conf.j2

@@ -19,7 +19,7 @@ ACCEPT_TAGS={"tags":[{"tag":"data_center","value":"{{ data_center }}"}]}
 
 [LOG]
 NIC_NAME={{ nic_mgr.name }}
-BROKER_LIST={{ log_kafkabrokers.address }}
+BROKER_LIST={{ log_kafkabrokers.address | join(",") }}
 FIELD_FILE=conf/capture_packet_log_field.conf
 
 [SYSTEM]

roles/firewall/templates/main.conf.j2

@@ -20,6 +20,10 @@ LOG_LEVEL={{ fw_dns_log_level }}
 LOG_PATH="./tsglog/fw_quic_plug/fw_quic_plug"
 LOG_LEVEL={{ fw_quic_log_level }}
 
+[CONTROL_PLUG]
+LOG_PATH="./tsglog/app_control_plug/app_control_plug"
+LOG_LEVEL={{ app_control_log_level }}
+
 [MAAT]
 PROFILE="./tsgconf/maat.conf"
 SUBSCRIBER_ID_TABLE="TSG_OBJ_SUBSCRIBER_ID"
@@ -32,7 +36,7 @@ NIC_NAME="{{ nic_mgr.name }}"
 MAX_SERVICE=1
 LOG_LEVEL={{ tsg_log_level }}
 LOG_PATH="./tsglog/tsglog"
-BROKER_LIST="{{ log_kafkabrokers.address }}"
+BROKER_LIST="{{ log_kafkabrokers.address | join(",") }}"
 COMMON_FIELD_FILE="tsgconf/tsg_log_field.conf"
 
 [STATISTIC]
@@ -55,3 +59,6 @@ LOG_LEVEL={{ tsg_master_log_level }}
 LOG_PATH="./tsglog/tsg_master"
 POLICY_PRIORITY_LABEL="POLICY_PRIORITY"
 DEVICE_ID_COMMAND="hostname | awk -F'-' '{print $3}'| awk -F'adc' '{print $2}'"
+
+[TSG_CONN_SKETCH]
+log_service=2

roles/firewall/templates/tsg_conn_sketch.inf.j2

@@ -0,0 +1,35 @@
+[PLUGINFO]
+PLUGNAME=TSG_CONN_SKETCH
+SO_PATH=./plug/business/tsg_conn_sketch/tsg_conn_sketch.so
+INIT_FUNC=tsg_conn_record_init
+DESTROY_FUNC=tsg_conn_record_destroy
+
+
+[TCP]
+FUNC_FLAG=ALL
+FUNC_NAME=tsg_record_tcp_entry
+
+[TCP_ALL]
+FUNC_FLAG=ALL
+FUNC_NAME=tsg_record_tcpall_entry
+
+[UDP]
+FUNC_FLAG=ALL
+FUNC_NAME=tsg_record_udp_entry
+
+[HTTP]
+FUNC_FLAG=ALL
+FUNC_NAME=tsg_record_http_entry
+
+[SSL]
+FUNC_FLAG=SSL_CLIENT_HELLO,SSL_SERVER_HELLO,SSL_APPLICATION_DATA,SSL_CERTIFICATE_DETAIL
+FUNC_NAME=tsg_record_ssl_entry
+
+#[DNS]
+#FUNC_FLAG=ALL
+#FUNC_NAME=tsg_record_dns_entry
+
+[MAIL]
+FUNC_FLAG=ALL
+FUNC_NAME=tsg_record_mail_entry
+

roles/firewall/templates/tsg_log_field.conf.j2

@@ -1,52 +0,0 @@
-#TYPE：1:UCHAR,2:USHORT,3:ULONG,4:ULOG,5:USTRING,6:FILE,7:UBASE64,8:PACKET
-#TYPE	TOPIC	SERVICE
-TOPIC	SECURITY-EVENT-LOG	0
-TOPIC	CONNECTION-RECORD-LOG	1
-TOPIC	CONNECTION-SKETCH	2
-
-#TYPE		FIELD		VALUE
-LONG   		common_policy_id	1
-LONG   		common_service		2
-LONG   		common_action		3
-LONG   		common_start_time	4
-LONG   		common_end_time		5
-STRING		common_l4_protocol	6
-LONG   		common_address_type	7
-STRING		common_server_ip	8
-STRING		common_client_ip	9
-LONG   		common_server_port	10
-LONG   		common_client_port	11
-LONG   		common_stream_dir	12
-STRING		common_address_list	13
-LONG   		common_entrance_id	14	
-LONG   		common_device_id	15	
-LONG   		common_link_id		16	
-STRING		common_isp		17	
-LONG   		common_encapsulation	18	
-LONG   		common_direction	19	
-STRING		common_sled_ip		20	
-STRING		common_user_tags	21
-STRING		common_user_region	22
-STRING		common_app_label	23
-LONG		common_app_id		24
-LONG		common_protocol_id	25
-LONG		common_c2s_pkt_num	26
-LONG		common_s2c_pkt_num	27
-LONG		common_c2s_byte_num	28
-LONG		common_s2c_byte_num	29	
-LONG		common_con_duration_ms	30
-LONG		common_has_dup_traffic	31
-STRING		common_stream_error	32
-STRING		common_stream_trace_id	33
-STRING		common_schema_type	34
-STRING		http_host		35
-STRING		ssl_sni			36
-LONG		common_establish_latency_ms	37
-STRING		common_sub_action		38
-STRING		common_client_asn	39
-STRING		common_server_asn	40
-STRING		common_client_location	41
-STRING		common_server_location	42
-STRING		quic_sni			43
-STRING		ssl_ja3_fingerprint		44
-STRING		common_data_center	45