20.11.rc3 rebase version 20.11

roles/tfe/files/memory.conf

@@ -1,2 +1,3 @@
 [Service]
-MemoryMax=100G
+MemoryLimit=100G
+ExecStartPost=/bin/bash -c "echo 100G > /sys/fs/cgroup/memory/system.slice/tfe.service/memory.memsw.limit_in_bytes"

roles/tfe/files/tfe.service

@@ -8,7 +8,7 @@ After=tfe-env.service
 Type=notify
 ExecStart=/opt/tsg/tfe/bin/tfe
 WorkingDirectory=/opt/tsg/tfe/
-TimeoutSec=7200s
+TimeoutSec=900s
 RestartSec=10s
 Restart=always
 LimitNOFILE=524288

roles/tfe/files/tsg_diagnose_ca.pem

@@ -0,0 +1,49 @@
+-----BEGIN CERTIFICATE-----
+MIIGWzCCBEOgAwIBAgIJAMimxpHS+4hRMA0GCSqGSIb3DQEBCwUAMHcxCzAJBgNV
+BAYTAlVTMRMwEQYDVQQIDApDYWxpZm9ybmlhMRYwFAYDVQQHDA1TYW4gRnJhbmNp
+c2NvMQ8wDQYDVQQKDAZCYWRTU0wxKjAoBgNVBAMMIUJhZFNTTCBSb290IENlcnRp
+ZmljYXRlIEF1dGhvcml0eTAeFw0yMDEwMjYwODQ3NDZaFw00MDEwMjEwODQ3NDZa
+MHcxCzAJBgNVBAYTAlVTMRMwEQYDVQQIDApDYWxpZm9ybmlhMRYwFAYDVQQHDA1T
+YW4gRnJhbmNpc2NvMQ8wDQYDVQQKDAZCYWRTU0wxKjAoBgNVBAMMIUJhZFNTTCBS
+b290IENlcnRpZmljYXRlIEF1dGhvcml0eTCCAiIwDQYJKoZIhvcNAQEBBQADggIP
+ADCCAgoCggIBAKnefEvaekYAdlfFtpnaPaKYgl+X3FOXUEiYLHuX9YZjuhjVAf/I
+19iW7+k6mln3jSxD05YZQk/jUVTTVjYgQftHzlZiJG086AGhG86QwDIPb9nQIGy8
+3DscFFQGGOoYPdV9E+s1cFDTIFGqqqlJ5T5jpjnAL/3WR2LxrgzPVkBjcOTJnkU6
+Gv2jqwQYGSz8+A6FYsGLqO6Pv7uKY1OPELNcTGnSwD1uctsMHn/Xqx4nMaBoMuSc
+TZQEneSagGDgF1dVqEFhVEPo4VXiVthhS82xA3xK69UKfKLFkjjy+icH8LllKUFo
+Psu+w/9V3OZ4xfzjEdpoRwRUmOesS5wlEkd3rLKEWXG/A8Uul5iCZ2Dez9nE6wi7
+w7JD7R1InPoD+7KXtT2JWS+9sj+Vre7XIjSEQuBRGiTOXnDcuYjFOkvCqS7OToUc
+fOJAlKHCndqBnzLoLJHU2ozrqgz8SU0Iv1CPW6YXLtRFFX3K9WUvX7XNTonh+oWS
+6IGifWnVcYh2N5peUuNVT4heD4QfIDpCvjwUAp2IWr1GnEjvjhPaHialRotHhfCi
+t3T0F58IhFQ6+CLQwE57Yd+7zGbc7osqTe1hbiK2wcciTuajmGZyfev8atFey+Y5
+N/7jD3U0a6u4Z+DyGcc08Pj94cM5AJ7SA45LKwt6xhmGLzhemmdGLJLNAgMBAAGj
+gekwgeYwHQYDVR0OBBYEFMGs0F0ycvMIQgM6oTyOBrxzjCPKMIGpBgNVHSMEgaEw
+gZ6AFMGs0F0ycvMIQgM6oTyOBrxzjCPKoXukeTB3MQswCQYDVQQGEwJVUzETMBEG
+A1UECAwKQ2FsaWZvcm5pYTEWMBQGA1UEBwwNU2FuIEZyYW5jaXNjbzEPMA0GA1UE
+CgwGQmFkU1NMMSowKAYDVQQDDCFCYWRTU0wgUm9vdCBDZXJ0aWZpY2F0ZSBBdXRo
+b3JpdHmCCQDIpsaR0vuIUTAMBgNVHRMEBTADAQH/MAsGA1UdDwQEAwIBBjANBgkq
+hkiG9w0BAQsFAAOCAgEAeZzR9GKvTRiKfRqCzjhylk+7IbymWjxNTc2LQ3+O6lww
+kw6Z2ybzvR3i/IZ7Hw+DBo1MXku9qHW/1uKR2BssoLHU1p1iHCBrZ1nw9MXxqXa3
+PhgxUZZu39NdXFc12fY/SYP8XQkNVzQCNouOvb75hj087ZDHvGztHIaB3VNUs1p+
+qMvGm8RVUGfDDqynUBZ814N32eCu+13N+dGL7yxASzD6Y3/myhVjixUuoUG3zFTW
+NnIWspbC8MxhP/3QUMYi4KJM4KDiJQxPhGkMBwlhgAz/QPEJApKq0Cl0Reez7Gyd
+KdnrqvCKhf8K53Su8L1GeRvzzKb7Hi+kMWIZVJPGz2DHgOymP5RCsIuWG6cDgx5E
+3LfZYEPG63ezj+qMZmkdEMnD9SVBi85dOTOJ+OJgxxX2OahUKPUdDP89ZmHdOjR9
+CqUxnA+eqRNz1TajnjRFXir3/20SoBtrHBck3bxpmZwsF7A6Sg5RdlvQjK2Oy6g0
+9LrkPUgu9O/sBfz8uyG/HlQD7EuUNo0NQHqznnde3T+w5wY2vL3XUAl39qcpNPF6
+auCS8+aygYYmCUooZVzKlXGU3VUPGwcfmLE4gnPLT0+pnHtBS8tKLOzXAJjYQ3s+
+QpP3aO4lJvoZ6Oes/JRxNPW8dmaLxTKPqsaPEWWuoSYr0higPTBXQNg+++PYRY4=
+-----END CERTIFICATE-----
+-----BEGIN CERTIFICATE-----
+MIIB8jCCAVugAwIBAgIJAP3GpXchIMWHMA0GCSqGSIb3DQEBCwUAMBExDzANBgNV
+BAsMBkdFRURHRTAgFw0yMDAzMDkxNjEyNTlaGA8yMDUwMDMwMjE2MTI1OVowETEP
+MA0GA1UECwwGR0VFREdFMIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQCraZpH
+Fca2Iu+9E9HzKbEi2Akdk4RrUJxkQjB2Tr7fGxwPDXqdGvSoXDdgnSA0I0bbNqMs
+drgiCWimjnGiWfY0sssKg7plNTQ4i7Zz7P9Isyf6TuxvB09CzdhH2FQ3lLRTb8pv
+BA0E28CCYiZhtX1/3RlDSvxaRKOM3yEt0q+FRQIDAQABo1AwTjAdBgNVHQ4EFgQU
+NqrpSlpCuMBJlCLZEE/D5ZpBy8swHwYDVR0jBBgwFoAUNqrpSlpCuMBJlCLZEE/D
+5ZpBy8swDAYDVR0TBAUwAwEB/zANBgkqhkiG9w0BAQsFAAOBgQBsybFxUAjzhJ5H
+VbSLhyillxtAJ3vEKtLrMVnAgRUEwamyu1JQGndF9kh8RapSmHhmuZM9iTc+NsNb
+DKGKmEOY0vQMw83xE7EGYj4Nhww9UMyGglmTLbd3yB+uJA97beNVduU2mifDHGmN
+4buMiPl3AozGRl9p5UCzZM5XxMMw1A==
+-----END CERTIFICATE-----

roles/tfe/tasks/main.yml

@@ -14,9 +14,14 @@
   yum:
     name:
       - /tmp/ansible_deploy/tfe-kmod-v1.0.5.20200408-1dkms.noarch.rpm
-      - /tmp/ansible_deploy/tfe-4.3.16.b1c3ba7-1.el7.x86_64.rpm
+      - /tmp/ansible_deploy/tfe-4.3.28.ce28c42-1.el7.x86_64.rpm
     state: present
 
+- name: "tfe:copy cert file to device"
+  copy:
+    src: '{{ role_path }}/files/tsg_diagnose_ca.pem'
+    dest: /opt/tsg/tfe/resource/tfe/
+
 - name: "template tfe-env config"
   template:
     src: "{{ role_path }}/templates/tfe-env-config.j2"

roles/tfe/templates/future.conf.j2

@@ -1,7 +1,7 @@
 [STAT]
 no_stats=0
-statsd_server=127.0.0.1
-statsd_port=58100
+statsd_server=192.168.100.1
+statsd_port=8100
 histogram_bins=0.50,0.80,0.9,0.95
 statsd_cycle=5
 # FS_OUTPUT_STATSD=1, FS_OUTPUT_INFLUX_LINE=2

roles/tfe/templates/pangu_pxy.conf.j2

@@ -34,7 +34,7 @@ cache_store_object_way=0
 redis_cache_object_size=1024000
 #Configs of WiredLB for Minios load balancer.
 #WIREDLB_OVERRIDE=1
-wiredlb_health_port=42310
+#wiredlb_health_port=42310
 #If CACHE_STORE_OBJECT_WAY is not 0, we will use redis to store meta and object.
 redis_cluster_ip_list=192.168.10.62-63;
 redis_cluster_port_range=6379
@@ -43,7 +43,7 @@ redis_cluster_port_range=6379
 wiredlb_override=1
 wiredlb_topic=MinioFileLog
 wiredlb_datacenter=k18consul-tse
-wiredlb_health_port=52102
+wiredlb_health_port=8560
 wiredlb_group=FileLog
 
 log_fsstat_appname=tango_log_file
@@ -103,7 +103,3 @@ log_fsstat_dst_ip=10.4.20.201
 log_fsstat_dst_port=8125
 
 
-[traffic_mirror]
-table_info=resource/pangu/table_info_traffic_mirror.conf
-stat_file=log/traffic_mirror.status
-

roles/tfe/templates/tfe.conf.j2

@@ -63,7 +63,7 @@ service_cache_slots=4194304
 service_cache_expire_seconds=300
 service_cache_fail_as_pinning_cnt=4
 service_cache_fail_as_proto_err_cnt=5
-service_cache_succ_as_app_not_pinning_cnt=0
+#service_cache_succ_as_app_not_pinning_cnt=0
 service_cache_fail_time_window=30
 
 # cert
@@ -84,9 +84,11 @@ key_log_file=log/sslkeylog.log
 # mid cert cache
 mc_cache_enable=1
 mc_cache_eth={{ nic_mgr.name }}
-mc_cache_broker_list={{ log_kafkabrokers.address }}
+mc_cache_broker_list={{ log_kafkabrokers.address | join(",") }}
 mc_cache_topic=PXY-EXCH-INTERMEDIA-CERT
 
+ssl_ja3_table=PXY_SSL_FINGERPRINT
+
 [key_keeper]
 #Mode: debug - generate cert with ca_path, normal - generate cert with cert store
 #0 on cache 1 off cache
@@ -132,12 +134,14 @@ tcp_ttl_upstream=75
 tcp_ttl_downstream=70
 
 [stat]
-statsd_server=127.0.0.1
-statsd_port=58100
+statsd_server=192.168.100.1
+statsd_port=8100
 statsd_cycle=5
 # 1:FS_OUTPUT_STATSD; 2:FS_OUTPUT_INFLUX_LINE
 statsd_format=2
 histogram_bins=0.5,0.8,0.9,0.95
+statsd_set_prometheus_port=9001
+statsd_set_prometheus_url_path=/metrics
 
 [traffic_mirror]
 {% if tsg_running_type != 2 %}
@@ -151,11 +155,13 @@ device={{ nic_traffic_mirror.name }}
 # 0:TRAFFIC_MIRROR_ETHDEV_AF_PACKET; 1:TRAFFIC_MIRROR_ETHDEV_MARSIO
 type=1
 {% endif %}
+table_info=resource/pangu/table_info_traffic_mirror.conf
+stat_file=log/traffic_mirror.status
 
 [kafka]
 enable=1
 NIC_NAME={{ nic_mgr.name }}
-kafka_brokerlist={{ log_kafkabrokers.address }}
+kafka_brokerlist={{ log_kafkabrokers.address | join(",") }}
 kafka_topic=PROXY-EVENT-LOG
 device_id_filepath=/opt/tsg/etc/tsg_sn.json
 
@@ -170,13 +176,6 @@ stat_file=log/pangu_scan.fs2
 effect_interval_s=1
 deferred_load_on=0
 
-# Pangu uses accept_tags to support the effective range of the device.
-# Traffic mirroring does not need to support the effective range of the device,
-# but pangu and traffic mirroring use the same maat configuration file.
-# Therefore, there is no need to set accept_tags in tfe.conf,
-# just set accept_tags in the tfe_resource_init() code
-# accept_tags={"tags":[{"tag":"device_id","value":"device_1"}]}
-
 # json mode conf iterm
 json_cfg_file=resource/pangu/pangu_http.json