feature:TSG-21884: Sync proxy support fs4 modify in OS.

ansible/roles/traffic-engine/files/helm/conf/telegraf_proxy.conf

@@ -1,67 +0,0 @@
-[global_tags]
-   device_id = "${device_id}"
-{{- range .Values.device.tags -}}
-{{- range $key,$val := . }}
-   {{ $key }} = {{ $val | quote }}
-{{- end }}
-{{- end }}
-[agent]
-   interval = "1s"
-   round_interval = true
-   metric_batch_size = 1000
-   metric_buffer_limit = 100000
-   collection_jitter = "0s"
-   flush_interval = "1s"
-   flush_jitter = "0s"
-   precision = ""
-   debug = false
-   quiet = false
-   logfile = ""
-   hostname = ""
-   omit_hostname = true
-
-[[outputs.kafka]]
-   sasl_username = "{{ .Values.external_resources.olap.kafka_brokers.sasl_username }}"
-   sasl_password = "{{ .Values.external_resources.olap.kafka_brokers.sasl_password }}"
-   brokers = [ "{{- include "traffic-engine.config.addresses.converter"  (list .Values.external_resources.olap.kafka_brokers.addresses "\",\"") }}" ]
-   topic = "POLICY-RULE-METRIC"
-   client_id = "POLICY-RULE-METRIC"
-   data_format = "json"
-   json_timestamp_units = "1ms"
-   json_transformation = '''
-      $merge([$, { "timestamp_ms": timestamp}]) ~> |$|{}, ['timestamp']|
-      '''
-
-[[inputs.socket_listener]]
-   service_address = "udp://:8900"
-   data_format = "influx"
-   read_buffer_size = "32MiB"
-[[processors.rename]]
-   [[processors.rename.replace]]
-     field = "hit_count_sum"
-     dest = "hit_count"
-
-   [[processors.rename.replace]]
-     field = "in_bytes_sum"
-     dest = "in_bytes"
-
-   [[processors.rename.replace]]
-     field = "out_bytes_sum"
-     dest = "out_bytes"
-
-   [[processors.rename.replace]]
-     field = "in_pkts_sum"
-     dest = "in_pkts"
-
-   [[processors.rename.replace]]
-     field = "out_pkts_sum"
-     dest = "out_pkts"
-
-[[aggregators.basicstats]]
-  period = "1s"
-  delay = "1s"
-  grace = "1s"
-  drop_original = true
-  stats = ["sum"]
-  fieldpass = ["hit_count", "in_bytes", "out_bytes", "in_pkts", "out_pkts"]
-  namepass = ["proxy_rule_hits"]   # only "pass" swap metrics through the aggregator.

ansible/roles/traffic-engine/files/helm/conf/tfe.conf

@@ -23,6 +23,20 @@ cpu_affinity_mask={{- include "traffic-engine.tfe.cpu-affinity" . }}
 # LEAST_CONN = 0; ROUND_ROBIN = 1
 load_balance=1
 
+[public]
+vsys_id={{ .Values.vsys_id }}
+{{- range .Values.device.tags -}}
+{{- range $key,$val := . }}
+{{- if eq $key "data_center" }}
+data_center={{ $val }}
+{{- end }}
+{{- if eq $key "device_group" }}
+device_group={{ $val }}
+{{- end }}
+{{- end }}
+{{- end }}
+device_id=DEVICE_ID_PLACE_HOLDER_MARK
+
 # for enable kni v3
 [nfq]
 queue_id=1
@@ -97,9 +111,6 @@ trusted_cert_dir=resource/tfe/trusted_storage
 log_master_key=0
 key_log_file=log/sslkeylog.log
 
-# mid cert cache
-mc_cache_enable=1
-
 [key_keeper]
 #Mode: debug - generate cert with ca_path, normal - generate cert with cert store
 #0 on cache 1 off cache
@@ -176,15 +187,13 @@ stat_file=log/traffic_mirror.status
 default_vlan_id=0
 
 [kafka]
-enable=1
-kafka_brokerlist={{- include "traffic-engine.config.addresses.converter" (list .Values.external_resources.olap.kafka_brokers.addresses  ",") }}
-logger_send_topic=PROXY-EVENT
-file_bucket_topic=TRAFFIC-FILE-STREAM-RECORD
-mc_cache_topic=PXY-EXCH-INTERMEDIA-CERT
+brokerlist={{- include "traffic-engine.config.addresses.converter" (list .Values.external_resources.olap.kafka_brokers.addresses  ",") }}
 sasl_username={{ .Values.external_resources.olap.kafka_brokers.sasl_username }}
 sasl_passwd={{ .Values.external_resources.olap.kafka_brokers.sasl_password }}
-device_id_filepath=/opt/tsg/etc/tsg_sn.json
-vsystem_id={{ .Values.vsys_id }}
+rule_hits_topic=POLICY-RULE-METRIC
+proxy_event_topic=PROXY-EVENT
+file_stream_topic=TRAFFIC-FILE-STREAM-RECORD
+exch_cert_topic=PXY-EXCH-INTERMEDIA-CERT
 
 [maat]
 # 0:json 1:redis 2:iris
@@ -208,13 +217,13 @@ maat_redis_db_index={{ .Values.vsys_id }}
 # iris mode conf iterm
 full_cfg_dir=pangu_policy/full/index/
 inc_cfg_dir=pangu_policy/inc/index/
-accept_tag_key=data_center
 
 [proxy_hits]
-cycle=1000
-telegraf_port=8900
-telegraf_ip=127.0.0.1
+cycle=0
 app_name="proxy_rule_hits"
+output_fs_interval_ms=500
+output_kafka_interval_ms=1000
+outpath="metrics/porxy_fieldstat.json"
 
 # for enable kni v4
 [packet_io]

ansible/roles/traffic-engine/files/helm/templates/configmap-telegraf.yaml

@@ -1,7 +0,0 @@
-apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: telegraf-{{ .Release.Name }}
-  namespace: default
-data:
-  telegraf_proxy.conf: {{ tpl (.Files.Get "conf/telegraf_proxy.conf") . | quote }}

ansible/roles/traffic-engine/files/helm/templates/configmap-tfe.yaml

@@ -1,10 +1,10 @@
+{{- if and (eq .Values.proxy.enable .Values.define_enable_val_yes) (ge (len .Values.tfe_affinity) 1) }}
 apiVersion: v1
 kind: ConfigMap
 metadata:
   name: tfe-{{ .Release.Name }}
   namespace: default
 data:
-{{- if and (eq .Values.proxy.enable .Values.define_enable_val_yes) (ge (len .Values.tfe_affinity) 1) }}
   tfe.conf: {{ tpl (.Files.Get "conf/tfe.conf") . | quote }}
-{{- end }}
   tsg_device_tag.json: {{ tpl (.Files.Get "conf/tsg_device_tag.json") . | quote }}
+{{- end }}

ansible/roles/traffic-engine/files/helm/templates/deployment-proxy.yaml

@@ -166,29 +166,6 @@ spec:
         volumeMounts:
         {{- include "public.sync-host-timezone.volume-mount" . | nindent 8 }}  
 
-      - name: telegraf-proxy
-        image: "registry.gdnt-cloud.website/tsg-telegraf:{{ .Chart.AppVersion }}"
-        imagePullPolicy: Never
-        command:
-          - "bash"
-          - "-ec"
-          - |
-            source /etc/profile.d/device_id.sh
-            /usr/bin/telegraf -config /etc/telegraf/telegraf_proxy.conf -config-directory /etc/telegraf/telegraf_statistic.d
-        securityContext:
-          privileged: true
-        volumeMounts:
-        - name: telegraf-configs-volume
-          mountPath: "/etc/telegraf/telegraf_proxy.conf"
-          subPath: "telegraf_proxy.conf"
-        - name: shared-configs-volume
-          mountPath: "/opt/tsg/etc/tsg_sn.json"
-          subPath: "tsg_sn.json"
-        - name: shared-configs-volume
-          mountPath: "/etc/profile.d/device_id.sh"
-          subPath: "device_id.sh"
-        {{- include "public.sync-host-timezone.volume-mount" . | nindent 8 }}
-
       - name: prometheus-endpoint
         image: "quay.io/rebuy/exporter-merger:v0.2.0"
         imagePullPolicy: Never
@@ -256,7 +233,7 @@ spec:
         imagePullPolicy: Never
         command:
           - "bash"
-          - "-ec"
+          - "-ecx"
           - |
             mount -o remount,rw /sys
             # disable rpfilter
@@ -350,6 +327,7 @@ spec:
             cp -r /opt/tsg/configs/* /opt/tsg/shared-configs/
             {{ include "public.prepare-access-API" . | nindent 12 }}
             {{- include "public.serialize-node-annotations" . | nindent 12 }}
+            sed -Ei -c "s|DEVICE_ID_PLACE_HOLDER_MARK|${DEVICE_SN?}|g" /opt/tsg/shared-configs/proxy/tfe.conf
 
         securityContext:
           privileged: true
@@ -382,9 +360,6 @@ spec:
       - name: tfe-configs-volume
         configMap:
           name: tfe-{{ .Release.Name }}
-      - name: telegraf-configs-volume
-        configMap:
-          name: telegraf-{{ .Release.Name }}
       - name: shared-configs-volume
         emptyDir: {}
       - name: proxy-log