From 82ca73cec948ebcf8f5c4da6f9cc20da7392bab6 Mon Sep 17 00:00:00 2001 From: linxin Date: Tue, 30 Apr 2024 18:12:58 +0800 Subject: [PATCH] =?UTF-8?q?=E2=9C=A8=20feat:dp-trace=E7=BB=84=E4=BB=B6?= =?UTF-8?q?=E9=80=82=E9=85=8Dclixon=E9=85=8D=E7=BD=AE=E4=B8=8B=E5=8F=91?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../container-tools-install/tasks/main.yml | 6 ++ .../files/helm/templates/_config.tpl | 10 +-- .../files/helm/templates/clusterrole.yaml | 10 +++ .../helm/templates/clusterrolebinding.yaml | 14 ++++ .../files/helm/templates/deployment.yaml | 64 +++++++++++++++---- .../files/helm/templates/dp-trace.yaml | 31 +++++++-- .../files/helm/templates/serviceaccount.yaml | 6 ++ .../dp-trace-telemetry/files/helm/values.yaml | 10 +-- tools/mk-base-image | 2 +- 9 files changed, 120 insertions(+), 33 deletions(-) create mode 100644 ansible/roles/dp-trace-telemetry/files/helm/templates/clusterrole.yaml create mode 100644 ansible/roles/dp-trace-telemetry/files/helm/templates/clusterrolebinding.yaml create mode 100644 ansible/roles/dp-trace-telemetry/files/helm/templates/serviceaccount.yaml diff --git a/ansible/roles/container-tools-install/tasks/main.yml b/ansible/roles/container-tools-install/tasks/main.yml index aae08a47..fb6c2396 100644 --- a/ansible/roles/container-tools-install/tasks/main.yml +++ b/ansible/roles/container-tools-install/tasks/main.yml @@ -160,4 +160,10 @@ yum: name: crudini conf_file: "{{ rpm_repo_config_path }}" + state: present + +- name: "install inotify-tools" + yum: + name: inotify-tools + conf_file: "{{ rpm_repo_config_path }}" state: present \ No newline at end of file diff --git a/ansible/roles/dp-trace-telemetry/files/helm/templates/_config.tpl b/ansible/roles/dp-trace-telemetry/files/helm/templates/_config.tpl index 370a21f5..0a2f97bb 100644 --- a/ansible/roles/dp-trace-telemetry/files/helm/templates/_config.tpl +++ b/ansible/roles/dp-trace-telemetry/files/helm/templates/_config.tpl @@ -1,5 +1,5 @@ {{- define "dp_trace_telemetry.start" -}} -{{- if eq .Values.dp_trace_telemetry.debug.enable_interactive_startup .Values.define_enable_val_yes -}} +{{- if eq .Values.datapath_trace.debug.enable_interactive_startup .Values.define_enable_val_yes -}} while true; do sleep 10;done {{- else -}} exec /opt/tsg/dp_trace_telemetry/bin/dp_trace_telemetry -c /opt/tsg/dp_trace_telemetry/etc/dp_trace.conf -d /opt/tsg/dp_trace_telemetry/etc/dp_trace_dy.conf 2>&1|tee >(systemd-cat -t $DEPLOYMENT_NAME) @@ -7,10 +7,10 @@ {{- end -}} {{- define "dp_trace_telemetry.volume.prestart" -}} -{{- if eq .Values.dp_trace_telemetry.debug.enable_prestart_script .Values.define_enable_val_yes }} +{{- if eq .Values.datapath_trace.debug.enable_prestart_script .Values.define_enable_val_yes }} - name: dp_trace_telemetry-prestart hostPath: -{{- if .Values.dp_trace_telemetry.debug.prestart_script }} +{{- if .Values.datapath_trace.debug.prestart_script }} path: {{ .Values.dp_trace_telemetry.debug.prestart_script }} {{- else }} path: /etc/tsg-os/{{ .Release.Name }}/dp_trace_telemetry_prestart_script.sh @@ -20,7 +20,7 @@ {{- end -}} {{- define "dp_trace_telemetry.mount.prestart" -}} -{{- if eq .Values.dp_trace_telemetry.debug.enable_prestart_script .Values.define_enable_val_yes }} +{{- if eq .Values.datapath_trace.debug.enable_prestart_script .Values.define_enable_val_yes }} - name: prestart-dir mountPath: /tmp/prestart - name: dp_trace_telemetry-prestart @@ -29,7 +29,7 @@ {{- end -}} {{- define "dp_trace_telemetry.prestart" -}} -{{- if eq .Values.dp_trace_telemetry.debug.enable_prestart_script .Values.define_enable_val_yes }} +{{- if eq .Values.datapath_trace.debug.enable_prestart_script .Values.define_enable_val_yes }} echo WARNING: PRESTART.sh is enable, the commands in PRESTART.sh is: cat /opt/tsg/scripts/prestart.sh diff --git a/ansible/roles/dp-trace-telemetry/files/helm/templates/clusterrole.yaml b/ansible/roles/dp-trace-telemetry/files/helm/templates/clusterrole.yaml new file mode 100644 index 00000000..cd454a73 --- /dev/null +++ b/ansible/roles/dp-trace-telemetry/files/helm/templates/clusterrole.yaml @@ -0,0 +1,10 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + serviceFunction: {{ .Release.Name }} + name: {{ .Release.Name }} +rules: + - apiGroups: [""] + resources: ["services", "nodes"] + verbs: ["get", "list", "watch"] diff --git a/ansible/roles/dp-trace-telemetry/files/helm/templates/clusterrolebinding.yaml b/ansible/roles/dp-trace-telemetry/files/helm/templates/clusterrolebinding.yaml new file mode 100644 index 00000000..79f0538c --- /dev/null +++ b/ansible/roles/dp-trace-telemetry/files/helm/templates/clusterrolebinding.yaml @@ -0,0 +1,14 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + serviceFunction: {{ .Release.Name }} + name: {{ .Release.Name }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ .Release.Name }} +subjects: + - kind: ServiceAccount + name: {{ .Release.Name }} + namespace: {{ .Release.Namespace }} diff --git a/ansible/roles/dp-trace-telemetry/files/helm/templates/deployment.yaml b/ansible/roles/dp-trace-telemetry/files/helm/templates/deployment.yaml index e9d654ef..b6682ffc 100644 --- a/ansible/roles/dp-trace-telemetry/files/helm/templates/deployment.yaml +++ b/ansible/roles/dp-trace-telemetry/files/helm/templates/deployment.yaml @@ -26,6 +26,7 @@ spec: prometheus.io/port: "9005" prometheus.io/scrape: "true" spec: + serviceAccountName: {{ .Release.Name }} shareProcessNamespace: true tolerations: - key: node-role.kubernetes.io/control-plane @@ -42,6 +43,14 @@ spec: - "bash" - "-ec" - | + dnf -y install autoconf automake libtool + wget -c https://github.com/inotify-tools/inotify-tools/archive/refs/tags/3.21.9.6.tar.gz + tar -zvxf 3.21.9.6.tar.gz -C /usr/local/src/ + cd /usr/local/src/inotify-tools-3.21.9.6/ + ./autogen.sh && ./configure --prefix=/usr/local/inotify-tools && make && make install + echo 'export PATH=$PATH:/usr/local/inotify-tools/bin/' >> ~/.bashrc + source ~/.bashrc + WATCH_DIR="/opt/tsg/dp_trace_telemetry/etc/dynamic" while inotifywait -r -e modify,create "$WATCH_DIR"; do echo "send HUB signal to dp_trace" @@ -60,9 +69,21 @@ spec: - "-ec" - | ldconfig - {{ template "dp_trace_telemetry.prestart" . }} cp /opt/tsg/dp_trace_telemetry/etc/dp_trace_configmap.conf /opt/tsg/dp_trace_telemetry/etc/dp_trace.conf - cp /opt/tsg/dp_trace_telemetry/etc/dynamic/config /opt/tsg/dp_trace_telemetry/etc/dp_trace_dy.conf + {{- if eq .Values.datapath_trace.cm.connectivity "local_cache" }} + export APISERVER=https://kubernetes.default.svc + export SERVICEACCOUNT=/var/run/secrets/kubernetes.io/serviceaccount + export NAMESPACE=$(cat ${SERVICEACCOUNT}/namespace) + export TOKEN=$(cat ${SERVICEACCOUNT}/token) + export CACERT=${SERVICEACCOUNT}/ca.crt + curl --cacert ${CACERT} --header "Authorization: Bearer ${TOKEN}" -X GET ${APISERVER}/api/v1/namespaces/tsg-os-system/services/{{ .Values.datapath_trace.cm.local_cache.cache_name }}-redis-master -o /tmp/cm-cache.txt + export CM_POLICY_LOCAL_CACHE_IP=$(cat /tmp/cm-cache.txt | jq -r '.spec.clusterIP') + echo "export CM_POLICY_LOCAL_CACHE_IP=${CM_POLICY_LOCAL_CACHE_IP}" > /etc/profile.d/cm-local-cache.sh + chmod 0755 /etc/profile.d/cm-local-cache.sh + sed -Ei -c "s|CM_POLICY_LOCAL_CACHE_IP_LOCATION|${CM_POLICY_LOCAL_CACHE_IP?}|g" /opt/tsg/dp_trace_telemetry/etc/dp_trace.conf + {{- end }} + ln -sf /opt/tsg/dp_trace_telemetry/etc/dynamic/config /opt/tsg/dp_trace_telemetry/etc/dp_trace_dy.conf + {{ template "dp_trace_telemetry.prestart" . }} {{ template "dp_trace_telemetry.start" . }} volumeMounts: - name: dp-telemetry-daemon @@ -93,11 +114,11 @@ spec: - name: ldconfig-mrzcpd mountPath: /etc/ld.so.conf.d/mrzcpd.conf readOnly: true - {{- if eq .Values.dp_trace_telemetry.debug.enable_mount_host_filesystem .Values.define_enable_val_yes }} + {{- if eq .Values.datapath_trace.debug.enable_mount_host_filesystem .Values.define_enable_val_yes }} - name: host-root mountPath: /host {{- end }} - {{- if eq .Values.dp_trace_telemetry.debug.enable_prestart_script .Values.define_enable_val_yes }} + {{- if eq .Values.datapath_trace.debug.enable_prestart_script .Values.define_enable_val_yes }} - name: prestart-dir mountPath: /tmp/prestart - name: dp-trace-prestart @@ -112,15 +133,17 @@ spec: value: dp-trace-telemetry securityContext: privileged: true -{{- if eq .Values.dp_trace_telemetry.debug.enable_liveness_probe .Values.define_enable_val_yes }} + ports: + - containerPort: 9086 +{{- if eq .Values.datapath_trace.debug.enable_liveness_probe .Values.define_enable_val_yes }} livenessProbe: httpGet: httpHeaders: - name: Custom-Header value: Awesome path: /probe - port: 10000 - initialDelaySeconds: 120 + port: 9086 + initialDelaySeconds: 30 periodSeconds: 3 successThreshold: 1 timeoutSeconds: 10 @@ -130,8 +153,8 @@ spec: - name: Custom-Header value: Awesome path: /probe - port: 10000 - initialDelaySeconds: 120 + port: 9086 + initialDelaySeconds: 30 periodSeconds: 3 successThreshold: 1 timeoutSeconds: 10 @@ -149,6 +172,25 @@ spec: - name: ex-trace-port containerPort: {{ .Values.mergeExporter.mergePort }} protocol: TCP + {{- if eq .Values.datapath_trace.cm.connectivity "local_cache" }} + initContainers: + - name: init-default-svc + image: "registry.gdnt-cloud.website/tsg-init:{{ .Chart.AppVersion }}" + imagePullPolicy: Never + command: + - "bash" + - "-ec" + - | + until nslookup kubernetes.default.svc; do echo waiting for kubernetes service; sleep 2; done + - name: init-cm-svc + image: "registry.gdnt-cloud.website/tsg-init:{{ .Chart.AppVersion }}" + imagePullPolicy: Never + command: + - "bash" + - "-ec" + - | + until nslookup {{ .Values.datapath_trace.cm.local_cache.cache_name }}-redis-master.tsg-os-system.svc; do echo waiting for cm cache service; sleep 2; done + {{- end }} volumes: - name: journal-volume hostPath: @@ -160,8 +202,8 @@ spec: type: DirectoryOrCreate - name: dp-trace-prestart hostPath: - {{- if .Values.dp_trace_telemetry.debug.prestart_script }} - path: {{ .Values.dp_trace_telemetry.debug.prestart_script }} + {{- if .Values.datapath_trace.debug.prestart_script }} + path: {{ .Values.datapath_trace.debug.prestart_script }} {{- else }} path: /etc/tsg-os/{{ .Release.Name }}/dp_trace_prestart_script.sh {{- end }} diff --git a/ansible/roles/dp-trace-telemetry/files/helm/templates/dp-trace.yaml b/ansible/roles/dp-trace-telemetry/files/helm/templates/dp-trace.yaml index 56e91589..297b41e4 100644 --- a/ansible/roles/dp-trace-telemetry/files/helm/templates/dp-trace.yaml +++ b/ansible/roles/dp-trace-telemetry/files/helm/templates/dp-trace.yaml @@ -16,16 +16,23 @@ data: monit_file_path=/var/run/mrzcpd/mrmonit.app.dp_trace_telemetry.saving [http_server] - listen_addr=127.0.0.1 - listen_port=10000 + listen_addr=0.0.0.0 + listen_port=9086 keep_alive_path=/probe [kafka] kafka_dump_to_log=0 - borker_list="192.168.44.12:9094" + {{- if .Values.datapath_trace.olap.kafka_brokers}} + borker_list="{{- range $index,$kafka := .Values.datapath_trace.olap.kafka_brokers.addresses }}{{- if ne $index 0 }},{{ end -}}{{ $kafka.address }}:{{ $kafka.port}}{{- end }}" topic_name="DATAPATH-TELEMETRY-RECORD" - sasl_username=admin - sasl_password=galaxy2019 + sasl_username={{.Values.datapath_trace.olap.kafka_brokers.sasl_username }} + sasl_password={{.Values.datapath_trace.olap.kafka_brokers.sasl_password }} + {{- else }} + borker_list="" + topic_name="DATAPATH-TELEMETRY-RECORD" + sasl_username= + sasl_password= + {{- end }} [maat] maat_log_level=3 @@ -34,8 +41,18 @@ data: deferred_load_on=0 table_schema=../etc/table_schema.json json_cfg_file=../etc/dp_telemetry_rules.json - maat_redis_server=192.168.44.3 - maat_redis_port_range=7002 + {{- if .Values.datapath_trace.cm}} + {{- if eq .Values.datapath_trace.cm.connectivity "local_cache"}} + maat_redis_server=CM_POLICY_LOCAL_CACHE_IP_LOCATION + maat_redis_port_range=6379 + {{- else }} + maat_redis_server={{ .Values.datapath_trace.cm.direct.address}} + maat_redis_port_range={{ .Values.datapath_trace.cm.direct.port}} + {{- end }} + {{- else }} + maat_redis_server= + maat_redis_port_range= + {{- end }} maat_redis_db_index=1 --- apiVersion: v1 diff --git a/ansible/roles/dp-trace-telemetry/files/helm/templates/serviceaccount.yaml b/ansible/roles/dp-trace-telemetry/files/helm/templates/serviceaccount.yaml new file mode 100644 index 00000000..0842acb1 --- /dev/null +++ b/ansible/roles/dp-trace-telemetry/files/helm/templates/serviceaccount.yaml @@ -0,0 +1,6 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + serviceFunction: {{ .Release.Name }} + name: {{ .Release.Name }} diff --git a/ansible/roles/dp-trace-telemetry/files/helm/values.yaml b/ansible/roles/dp-trace-telemetry/files/helm/values.yaml index 7a6f830c..90666ea8 100644 --- a/ansible/roles/dp-trace-telemetry/files/helm/values.yaml +++ b/ansible/roles/dp-trace-telemetry/files/helm/values.yaml @@ -85,15 +85,7 @@ mergeExporter: define_enable_val_yes: 'yes' -datapath_trace: {} +datapath_trace: { debug: { enable_liveness_probe:'no', enable_interactive_startup: 'no', enable_prestart_script: 'no', enable_mount_host_filesystem:'no', prestart_script: ""}} device: {} -dp_trace_telemetry: - debug: - enable_liveness_probe: 'no' - define_enable_val_yes: 'yes' - enable_prestart_script: 'yes' - enable_mount_host_filesystem: 'yes' - #default: /etc/tsg-os/${service_function_name}/shaping_prestart_script.sh - prestart_script: "" diff --git a/tools/mk-base-image b/tools/mk-base-image index 189ec3dd..f9187748 100755 --- a/tools/mk-base-image +++ b/tools/mk-base-image @@ -42,7 +42,7 @@ case $profile_id in containerd.io lrzsz python3 watchdog pcm git tmux fish kernel kernel-devel kernel-tools-libs kernel-modules kernel-tools kernel-core rpm-build libtool kernel-rpm-macros python36-devel tcsh kernel-modules-extra gcc-gfortran libdb-devel fuse-devel python3-Cython cmake perl-generators libstdc++-devel libmnl-devel bison flex gcc-c++ - python3-docutils libnsl liburing hwloc-gui perl-open perl python2 js-d3-flame-graph xmlstarlet conntrack-tools crudini inotify-tools" + python3-docutils libnsl liburing hwloc-gui perl-open perl python2 js-d3-flame-graph xmlstarlet conntrack-tools crudini" ;; "7400-MCN0-P01R01" | "7400-MCN123-P01R01")