From fb22ee583c207e316daa43a91101add3731e5bd9 Mon Sep 17 00:00:00 2001 From: fumingwei Date: Wed, 8 Feb 2023 19:41:17 +0800 Subject: [PATCH] =?UTF-8?q?feature:TSG-13624:OS=E6=94=AF=E6=8C=81shaping?= =?UTF-8?q?=20master=20=E5=AF=B9=E5=BA=94=E7=9A=84telegraf=E9=85=8D?= =?UTF-8?q?=E7=BD=AE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../files/telegraf_shaping.service | 16 +++ .../roles/telegraf_statistic/tasks/main.yml | 27 ++++- .../templates/telegraf_shaping.conf.j2.j2 | 110 +++++++++++++++++ .../files/helm/conf/telegraf_shaping.conf | 114 ++++++++++++++++++ .../files/helm/templates/telegraf.yaml | 1 + .../files/helm/templates/traffic-engine.yaml | 22 ++++ 6 files changed, 288 insertions(+), 2 deletions(-) create mode 100644 ansible/roles/telegraf_statistic/files/telegraf_shaping.service create mode 100644 ansible/roles/telegraf_statistic/templates/telegraf_shaping.conf.j2.j2 create mode 100644 ansible/roles/traffic-engine/files/helm/conf/telegraf_shaping.conf diff --git a/ansible/roles/telegraf_statistic/files/telegraf_shaping.service b/ansible/roles/telegraf_statistic/files/telegraf_shaping.service new file mode 100644 index 00000000..c89cc5e5 --- /dev/null +++ b/ansible/roles/telegraf_statistic/files/telegraf_shaping.service @@ -0,0 +1,16 @@ +[Unit] +Description=Shaping information +Documentation=https://github.com/influxdata/telegraf +After=network.target + +[Service] +EnvironmentFile=-/etc/default/telegraf +User=telegraf +ExecStart=/usr/bin/telegraf -config /etc/telegraf/telegraf_shaping.conf -config-directory /etc/telegraf/telegraf_statistic.d $TELEGRAF_OPTS +ExecReload=/bin/kill -HUP $MAINPID +Restart=on-failure +RestartForceExitStatus=SIGPIPE +KillMode=control-group + +[Install] +WantedBy=multi-user.target diff --git a/ansible/roles/telegraf_statistic/tasks/main.yml b/ansible/roles/telegraf_statistic/tasks/main.yml index 8005cdfd..163f8a78 100644 --- a/ansible/roles/telegraf_statistic/tasks/main.yml +++ b/ansible/roles/telegraf_statistic/tasks/main.yml @@ -17,25 +17,42 @@ tags: template when: runtime_env != 'TSG-X-P0906' +- name: "Templates telegraf.conf" + template: + src: "{{role_path}}/templates/telegraf_shaping.conf.j2.j2" + dest: /opt/tsg/tsg-os-provision/templates/telegraf_shaping.conf.j2 + tags: template + when: runtime_env != 'TSG-X-P0906' + - name: "copy telegraf_statistic.service to destination server" copy: src: "{{ role_path }}/files/telegraf_statistic.service" dest: /usr/lib/systemd/system mode: 0644 -- name: "Create /usr/lib/systemd/system/telegraf_statistic.service.d/ directory if it does not exist" +- name: "copy telegraf_shaping.service to destination server" + copy: + src: "{{ role_path }}/files/telegraf_shaping.service" + dest: /usr/lib/systemd/system + mode: 0644 + +- name: "Create /usr/lib/systemd/system/telegraf_statistic.service.d/ and /usr/lib/systemd/system/telegraf_shaping.service.d directory if it does not exist" file: path: "{{ item }}" state: directory mode: '0755' with_items: - /usr/lib/systemd/system/telegraf_statistic.service.d + - /usr/lib/systemd/system/telegraf_shaping.service.d - name: "copy slice file to telegraf_statistic.service.d" copy: src: "{{ role_path }}/templates/service_override_slice.conf.j2" - dest: /usr/lib/systemd/system/telegraf_statistic.service.d/service_override_slice.conf + dest: "{{ item }}" mode: 0644 + with_items: + - /usr/lib/systemd/system/telegraf_statistic.service.d/service_override_slice.conf + - /usr/lib/systemd/system/telegraf_shaping.service.d/service_override_slice.conf when: runtime_env != 'TSG-X-P0906' - name: "replace action: replace service WantedBy from multi-user.target to workload.target" @@ -45,6 +62,7 @@ replace: 'WantedBy=workload.target' with_items: - /usr/lib/systemd/system/telegraf_statistic.service + - /usr/lib/systemd/system/telegraf_shaping.service when: runtime_env != 'TSG-X-P0906' ##################### telegraf ##################### @@ -57,4 +75,9 @@ - name: "Start telegraf_statistic" systemd: name: telegraf_statistic.service + enabled: yes + +- name: "Start telegraf_shaping" + systemd: + name: telegraf_shaping.service enabled: yes \ No newline at end of file diff --git a/ansible/roles/telegraf_statistic/templates/telegraf_shaping.conf.j2.j2 b/ansible/roles/telegraf_statistic/templates/telegraf_shaping.conf.j2.j2 new file mode 100644 index 00000000..a027f0d5 --- /dev/null +++ b/ansible/roles/telegraf_statistic/templates/telegraf_shaping.conf.j2.j2 @@ -0,0 +1,110 @@ +# Telegraf Configuration +[global_tags] + device_id = "${device_id}" + vsys_id = "{% raw %}{{ vsys_id }}{% endraw %}" +[agent] + interval = "1s" + round_interval = true + metric_batch_size = 1000 + metric_buffer_limit = 10000 + collection_jitter = "0s" + flush_interval = "1s" + flush_jitter = "0s" + precision = "" + debug = false + quiet = false + logfile = "" + hostname = "" + omit_hostname = true + + +[[inputs.socket_listener]] + service_address = "udp://:8200" + data_format = "influx" + +#[[processors.converter]] +# [processors.converter.tags] +# measurement = ["topic"] + +[[processors.rename]] + [[processors.rename.replace]] + field = "active_sessions_sum" + dest = "active_sessions" + + [[processors.rename.replace]] + field = "in_drop_pkts_sum" + dest = "in_drop_pkts" + + [[processors.rename.replace]] + field = "in_max_latency_us_max" + dest = "in_max_latency_us" + + [[processors.rename.replace]] + field = "in_queue_len_sum" + dest = "in_queue_len" + + [[processors.rename.replace]] + field = "in_rx_bytes_sum" + dest = "in_rx_bytes" + + [[processors.rename.replace]] + field = "in_rx_pkts_sum" + dest = "in_rx_pkts" + + [[processors.rename.replace]] + field = "in_tx_bytes_sum" + dest = "in_tx_bytes" + + [[processors.rename.replace]] + field = "in_tx_pkts_sum" + dest = "in_tx_pkts" + + [[processors.rename.replace]] + field = "out_drop_pkts_sum" + dest = "out_drop_pkts" + + [[processors.rename.replace]] + field = "out_max_latency_us_max" + dest = "out_max_latency_us" + + [[processors.rename.replace]] + field = "out_queue_len_sum" + dest = "out_queue_len" + + [[processors.rename.replace]] + field = "out_rx_bytes_sum" + dest = "out_rx_bytes" + + [[processors.rename.replace]] + field = "out_rx_pkts_sum" + dest = "out_rx_pkts" + + [[processors.rename.replace]] + field = "out_tx_bytes_sum" + dest = "out_tx_bytes" + + [[processors.rename.replace]] + field = "out_tx_pkts_sum" + dest = "out_tx_pkts" + + [[processors.rename.replace]] + field = "queueing_sessions_sum" + dest = "queueing_sessions" + +[[aggregators.basicstats]] + period = "1s" + drop_original = true + stats = ["sum", "max"] + + +[[outputs.kafka]] + sasl_username = "admin" + sasl_password = "galaxy2019" +{% raw %} brokers = [ "{{ olap.kafka_broker.address_list | join("\",\"") }}" ] +{% endraw %} + topic = "TRAFFIC-SHAPING-METRICS" + fielddrop = ["*pkts*max", "*bytes*max", "*session*max", "*queue*max", "*latency*sum"] + data_format = "json" + json_transformation = ''' + $merge([{"timestamp": timestamp}, tags, fields]) + ''' diff --git a/ansible/roles/traffic-engine/files/helm/conf/telegraf_shaping.conf b/ansible/roles/traffic-engine/files/helm/conf/telegraf_shaping.conf new file mode 100644 index 00000000..57d74692 --- /dev/null +++ b/ansible/roles/traffic-engine/files/helm/conf/telegraf_shaping.conf @@ -0,0 +1,114 @@ +# Telegraf Configuration +[global_tags] + device_id = "${device_id}" +{{- range .Values.device.tags -}} +{{- range $key,$val := . }} + {{ $key }} = {{ $val | quote }} +{{- end }} +{{- end }} + vsys_id = {{ .Values.vsys_id | quote }} +[agent] + interval = "1s" + round_interval = true + metric_batch_size = 1000 + metric_buffer_limit = 10000 + collection_jitter = "0s" + flush_interval = "1s" + flush_jitter = "0s" + precision = "" + debug = false + quiet = false + logfile = "" + hostname = "" + omit_hostname = true + + +[[inputs.socket_listener]] + service_address = "udp://:8200" + data_format = "influx" + +#[[processors.converter]] +# [processors.converter.tags] +# measurement = ["topic"] + +[[processors.rename]] + [[processors.rename.replace]] + field = "active_sessions_sum" + dest = "active_sessions" + + [[processors.rename.replace]] + field = "in_drop_pkts_sum" + dest = "in_drop_pkts" + + [[processors.rename.replace]] + field = "in_max_latency_us_max" + dest = "in_max_latency_us" + + [[processors.rename.replace]] + field = "in_queue_len_sum" + dest = "in_queue_len" + + [[processors.rename.replace]] + field = "in_rx_bytes_sum" + dest = "in_rx_bytes" + + [[processors.rename.replace]] + field = "in_rx_pkts_sum" + dest = "in_rx_pkts" + + [[processors.rename.replace]] + field = "in_tx_bytes_sum" + dest = "in_tx_bytes" + + [[processors.rename.replace]] + field = "in_tx_pkts_sum" + dest = "in_tx_pkts" + + [[processors.rename.replace]] + field = "out_drop_pkts_sum" + dest = "out_drop_pkts" + + [[processors.rename.replace]] + field = "out_max_latency_us_max" + dest = "out_max_latency_us" + + [[processors.rename.replace]] + field = "out_queue_len_sum" + dest = "out_queue_len" + + [[processors.rename.replace]] + field = "out_rx_bytes_sum" + dest = "out_rx_bytes" + + [[processors.rename.replace]] + field = "out_rx_pkts_sum" + dest = "out_rx_pkts" + + [[processors.rename.replace]] + field = "out_tx_bytes_sum" + dest = "out_tx_bytes" + + [[processors.rename.replace]] + field = "out_tx_pkts_sum" + dest = "out_tx_pkts" + + [[processors.rename.replace]] + field = "queueing_sessions_sum" + dest = "queueing_sessions" + +[[aggregators.basicstats]] + period = "1s" + drop_original = true + stats = ["sum", "max"] + + +[[outputs.kafka]] + sasl_username = "{{ .Values.external_resources.olap.kafka_brokers.sasl_username }}" + sasl_password = "{{ .Values.external_resources.olap.kafka_brokers.sasl_password }}" + brokers = [ "{{- include "traffic-engine.config.olap-address" (list . "\",\"") }}" ] + topic = "TRAFFIC-SHAPING-METRICS" + fielddrop = ["*pkts*max", "*bytes*max", "*session*max", "*queue*max", "*latency*sum"] + data_format = "json" + json_transformation = ''' + $merge([{"timestamp": timestamp}, tags, fields]) + ''' diff --git a/ansible/roles/traffic-engine/files/helm/templates/telegraf.yaml b/ansible/roles/traffic-engine/files/helm/templates/telegraf.yaml index ffdc86b1..50f04dfe 100644 --- a/ansible/roles/traffic-engine/files/helm/templates/telegraf.yaml +++ b/ansible/roles/traffic-engine/files/helm/templates/telegraf.yaml @@ -5,3 +5,4 @@ metadata: namespace: default data: telegraf_statistic.conf: {{ tpl (.Files.Get "conf/telegraf_statistic.conf") . | quote }} + telegraf_shaping.conf: {{ tpl (.Files.Get "conf/telegraf_shaping.conf") . | quote }} \ No newline at end of file diff --git a/ansible/roles/traffic-engine/files/helm/templates/traffic-engine.yaml b/ansible/roles/traffic-engine/files/helm/templates/traffic-engine.yaml index 1e39e147..8d3e3ef7 100644 --- a/ansible/roles/traffic-engine/files/helm/templates/traffic-engine.yaml +++ b/ansible/roles/traffic-engine/files/helm/templates/traffic-engine.yaml @@ -201,6 +201,28 @@ spec: mountPath: /etc/localtime readOnly: true +{{- if eq .Values.shaping.enable .Values.define_enable_val_yes }} + - name: telegraf-shaping + image: "{{ .Values.image.telegraf.repository }}:{{ .Values.image.telegraf.tag | default .Chart.AppVersion }}" + imagePullPolicy: {{ .Values.image.telegraf.pullPolicy }} + command: ["/usr/bin/telegraf", "-config", "/etc/telegraf/telegraf_shaping.conf", "-config-directory", "/etc/telegraf/telegraf_statistic.d"] + securityContext: + privileged: true + volumeMounts: + - name: telegraf + mountPath: "/etc/telegraf/telegraf_shaping.conf" + subPath: "telegraf_shaping.conf" + - name: config-volume + mountPath: "/opt/tsg/etc/tsg_sn.json" + subPath: "opt/tsg/etc/tsg_sn.json" + - name: config-volume + mountPath: "/etc/default/telegraf" + subPath: "etc/default/telegraf" + - name: localtime-node + mountPath: /etc/localtime + readOnly: true +{{- end }} + - name: merge-exporter image: "{{ .Values.mergeExporter.image.repository }}:{{ .Values.mergeExporter.image.tag }}" imagePullPolicy: {{ .Values.mergeExporter.image.pullPolicy }}