diff --git a/ansible/roles/traffic-engine/files/helm/conf/telegraf_proxy.conf b/ansible/roles/traffic-engine/files/helm/conf/telegraf_proxy.conf deleted file mode 100644 index 431c54a4..00000000 --- a/ansible/roles/traffic-engine/files/helm/conf/telegraf_proxy.conf +++ /dev/null @@ -1,67 +0,0 @@ -[global_tags] - device_id = "${device_id}" -{{- range .Values.device.tags -}} -{{- range $key,$val := . }} - {{ $key }} = {{ $val | quote }} -{{- end }} -{{- end }} -[agent] - interval = "1s" - round_interval = true - metric_batch_size = 1000 - metric_buffer_limit = 100000 - collection_jitter = "0s" - flush_interval = "1s" - flush_jitter = "0s" - precision = "" - debug = false - quiet = false - logfile = "" - hostname = "" - omit_hostname = true - -[[outputs.kafka]] - sasl_username = "{{ .Values.external_resources.olap.kafka_brokers.sasl_username }}" - sasl_password = "{{ .Values.external_resources.olap.kafka_brokers.sasl_password }}" - brokers = [ "{{- include "traffic-engine.config.addresses.converter" (list .Values.external_resources.olap.kafka_brokers.addresses "\",\"") }}" ] - topic = "POLICY-RULE-METRIC" - client_id = "POLICY-RULE-METRIC" - data_format = "json" - json_timestamp_units = "1ms" - json_transformation = ''' - $merge([$, { "timestamp_ms": timestamp}]) ~> |$|{}, ['timestamp']| - ''' - -[[inputs.socket_listener]] - service_address = "udp://:8900" - data_format = "influx" - read_buffer_size = "32MiB" -[[processors.rename]] - [[processors.rename.replace]] - field = "hit_count_sum" - dest = "hit_count" - - [[processors.rename.replace]] - field = "in_bytes_sum" - dest = "in_bytes" - - [[processors.rename.replace]] - field = "out_bytes_sum" - dest = "out_bytes" - - [[processors.rename.replace]] - field = "in_pkts_sum" - dest = "in_pkts" - - [[processors.rename.replace]] - field = "out_pkts_sum" - dest = "out_pkts" - -[[aggregators.basicstats]] - period = "1s" - delay = "1s" - grace = "1s" - drop_original = true - stats = ["sum"] - fieldpass = ["hit_count", "in_bytes", "out_bytes", "in_pkts", "out_pkts"] - namepass = ["proxy_rule_hits"] # only "pass" swap metrics through the aggregator. diff --git a/ansible/roles/traffic-engine/files/helm/conf/tfe.conf b/ansible/roles/traffic-engine/files/helm/conf/tfe.conf index d52a4ec4..399295cb 100644 --- a/ansible/roles/traffic-engine/files/helm/conf/tfe.conf +++ b/ansible/roles/traffic-engine/files/helm/conf/tfe.conf @@ -23,6 +23,20 @@ cpu_affinity_mask={{- include "traffic-engine.tfe.cpu-affinity" . }} # LEAST_CONN = 0; ROUND_ROBIN = 1 load_balance=1 +[public] +vsys_id={{ .Values.vsys_id }} +{{- range .Values.device.tags -}} +{{- range $key,$val := . }} +{{- if eq $key "data_center" }} +data_center={{ $val }} +{{- end }} +{{- if eq $key "device_group" }} +device_group={{ $val }} +{{- end }} +{{- end }} +{{- end }} +device_id=DEVICE_ID_PLACE_HOLDER_MARK + # for enable kni v3 [nfq] queue_id=1 @@ -97,9 +111,6 @@ trusted_cert_dir=resource/tfe/trusted_storage log_master_key=0 key_log_file=log/sslkeylog.log -# mid cert cache -mc_cache_enable=1 - [key_keeper] #Mode: debug - generate cert with ca_path, normal - generate cert with cert store #0 on cache 1 off cache @@ -176,15 +187,13 @@ stat_file=log/traffic_mirror.status default_vlan_id=0 [kafka] -enable=1 -kafka_brokerlist={{- include "traffic-engine.config.addresses.converter" (list .Values.external_resources.olap.kafka_brokers.addresses ",") }} -logger_send_topic=PROXY-EVENT -file_bucket_topic=TRAFFIC-FILE-STREAM-RECORD -mc_cache_topic=PXY-EXCH-INTERMEDIA-CERT +brokerlist={{- include "traffic-engine.config.addresses.converter" (list .Values.external_resources.olap.kafka_brokers.addresses ",") }} sasl_username={{ .Values.external_resources.olap.kafka_brokers.sasl_username }} sasl_passwd={{ .Values.external_resources.olap.kafka_brokers.sasl_password }} -device_id_filepath=/opt/tsg/etc/tsg_sn.json -vsystem_id={{ .Values.vsys_id }} +rule_hits_topic=POLICY-RULE-METRIC +proxy_event_topic=PROXY-EVENT +file_stream_topic=TRAFFIC-FILE-STREAM-RECORD +exch_cert_topic=PXY-EXCH-INTERMEDIA-CERT [maat] # 0:json 1:redis 2:iris @@ -208,13 +217,13 @@ maat_redis_db_index={{ .Values.vsys_id }} # iris mode conf iterm full_cfg_dir=pangu_policy/full/index/ inc_cfg_dir=pangu_policy/inc/index/ -accept_tag_key=data_center [proxy_hits] -cycle=1000 -telegraf_port=8900 -telegraf_ip=127.0.0.1 +cycle=0 app_name="proxy_rule_hits" +output_fs_interval_ms=500 +output_kafka_interval_ms=1000 +outpath="metrics/porxy_fieldstat.json" # for enable kni v4 [packet_io] diff --git a/ansible/roles/traffic-engine/files/helm/templates/configmap-telegraf.yaml b/ansible/roles/traffic-engine/files/helm/templates/configmap-telegraf.yaml deleted file mode 100644 index fb81b701..00000000 --- a/ansible/roles/traffic-engine/files/helm/templates/configmap-telegraf.yaml +++ /dev/null @@ -1,7 +0,0 @@ -apiVersion: v1 -kind: ConfigMap -metadata: - name: telegraf-{{ .Release.Name }} - namespace: default -data: - telegraf_proxy.conf: {{ tpl (.Files.Get "conf/telegraf_proxy.conf") . | quote }} \ No newline at end of file diff --git a/ansible/roles/traffic-engine/files/helm/templates/configmap-tfe.yaml b/ansible/roles/traffic-engine/files/helm/templates/configmap-tfe.yaml index 95740bef..bcd3690b 100644 --- a/ansible/roles/traffic-engine/files/helm/templates/configmap-tfe.yaml +++ b/ansible/roles/traffic-engine/files/helm/templates/configmap-tfe.yaml @@ -1,10 +1,10 @@ +{{- if and (eq .Values.proxy.enable .Values.define_enable_val_yes) (ge (len .Values.tfe_affinity) 1) }} apiVersion: v1 kind: ConfigMap metadata: name: tfe-{{ .Release.Name }} namespace: default data: -{{- if and (eq .Values.proxy.enable .Values.define_enable_val_yes) (ge (len .Values.tfe_affinity) 1) }} tfe.conf: {{ tpl (.Files.Get "conf/tfe.conf") . | quote }} -{{- end }} tsg_device_tag.json: {{ tpl (.Files.Get "conf/tsg_device_tag.json") . | quote }} +{{- end }} diff --git a/ansible/roles/traffic-engine/files/helm/templates/deployment-proxy.yaml b/ansible/roles/traffic-engine/files/helm/templates/deployment-proxy.yaml index e21d33a5..146ec312 100644 --- a/ansible/roles/traffic-engine/files/helm/templates/deployment-proxy.yaml +++ b/ansible/roles/traffic-engine/files/helm/templates/deployment-proxy.yaml @@ -166,29 +166,6 @@ spec: volumeMounts: {{- include "public.sync-host-timezone.volume-mount" . | nindent 8 }} - - name: telegraf-proxy - image: "registry.gdnt-cloud.website/tsg-telegraf:{{ .Chart.AppVersion }}" - imagePullPolicy: Never - command: - - "bash" - - "-ec" - - | - source /etc/profile.d/device_id.sh - /usr/bin/telegraf -config /etc/telegraf/telegraf_proxy.conf -config-directory /etc/telegraf/telegraf_statistic.d - securityContext: - privileged: true - volumeMounts: - - name: telegraf-configs-volume - mountPath: "/etc/telegraf/telegraf_proxy.conf" - subPath: "telegraf_proxy.conf" - - name: shared-configs-volume - mountPath: "/opt/tsg/etc/tsg_sn.json" - subPath: "tsg_sn.json" - - name: shared-configs-volume - mountPath: "/etc/profile.d/device_id.sh" - subPath: "device_id.sh" - {{- include "public.sync-host-timezone.volume-mount" . | nindent 8 }} - - name: prometheus-endpoint image: "quay.io/rebuy/exporter-merger:v0.2.0" imagePullPolicy: Never @@ -256,7 +233,7 @@ spec: imagePullPolicy: Never command: - "bash" - - "-ec" + - "-ecx" - | mount -o remount,rw /sys # disable rpfilter @@ -350,6 +327,7 @@ spec: cp -r /opt/tsg/configs/* /opt/tsg/shared-configs/ {{ include "public.prepare-access-API" . | nindent 12 }} {{- include "public.serialize-node-annotations" . | nindent 12 }} + sed -Ei -c "s|DEVICE_ID_PLACE_HOLDER_MARK|${DEVICE_SN?}|g" /opt/tsg/shared-configs/proxy/tfe.conf securityContext: privileged: true @@ -382,9 +360,6 @@ spec: - name: tfe-configs-volume configMap: name: tfe-{{ .Release.Name }} - - name: telegraf-configs-volume - configMap: - name: telegraf-{{ .Release.Name }} - name: shared-configs-volume emptyDir: {} - name: proxy-log