✨ feat:dp-trace组件适配clixon配置下发

2024-04-30 18:12:58 +08:00
parent 2936c03f1a
commit 82ca73cec9
9 changed files with 120 additions and 33 deletions
--- a/ansible/roles/container-tools-install/tasks/main.yml
+++ b/ansible/roles/container-tools-install/tasks/main.yml
@@ -160,4 +160,10 @@
  yum:
    name: crudini
    conf_file: "{{ rpm_repo_config_path }}"
+    state: present
+
+- name: "install inotify-tools"
+  yum:
+    name: inotify-tools
+    conf_file: "{{ rpm_repo_config_path }}"
    state: present
--- a/ansible/roles/dp-trace-telemetry/files/helm/templates/_config.tpl
+++ b/ansible/roles/dp-trace-telemetry/files/helm/templates/_config.tpl
@@ -1,5 +1,5 @@
 {{- define "dp_trace_telemetry.start" -}}
-{{- if eq .Values.dp_trace_telemetry.debug.enable_interactive_startup .Values.define_enable_val_yes -}}
+{{- if eq .Values.datapath_trace.debug.enable_interactive_startup .Values.define_enable_val_yes -}}
            while true; do sleep 10;done
 {{- else -}}
           exec /opt/tsg/dp_trace_telemetry/bin/dp_trace_telemetry -c /opt/tsg/dp_trace_telemetry/etc/dp_trace.conf -d /opt/tsg/dp_trace_telemetry/etc/dp_trace_dy.conf 2>&1|tee >(systemd-cat -t $DEPLOYMENT_NAME)
@@ -7,10 +7,10 @@
 {{- end -}}

 {{- define "dp_trace_telemetry.volume.prestart" -}}
-{{- if eq .Values.dp_trace_telemetry.debug.enable_prestart_script  .Values.define_enable_val_yes }}
+{{- if eq .Values.datapath_trace.debug.enable_prestart_script  .Values.define_enable_val_yes }}
      - name: dp_trace_telemetry-prestart
        hostPath:
-{{- if .Values.dp_trace_telemetry.debug.prestart_script }}
+{{- if .Values.datapath_trace.debug.prestart_script }}
          path: {{ .Values.dp_trace_telemetry.debug.prestart_script }}
 {{- else }}
          path: /etc/tsg-os/{{ .Release.Name }}/dp_trace_telemetry_prestart_script.sh
@@ -20,7 +20,7 @@
 {{- end -}}

 {{- define "dp_trace_telemetry.mount.prestart" -}}
-{{- if eq .Values.dp_trace_telemetry.debug.enable_prestart_script  .Values.define_enable_val_yes }}
+{{- if eq .Values.datapath_trace.debug.enable_prestart_script  .Values.define_enable_val_yes }}
        - name: prestart-dir
          mountPath: /tmp/prestart
        - name: dp_trace_telemetry-prestart
@@ -29,7 +29,7 @@
 {{- end -}}

 {{- define "dp_trace_telemetry.prestart" -}}
-{{- if eq .Values.dp_trace_telemetry.debug.enable_prestart_script  .Values.define_enable_val_yes }}
+{{- if eq .Values.datapath_trace.debug.enable_prestart_script  .Values.define_enable_val_yes }}
            echo WARNING: PRESTART.sh is enable, the commands in PRESTART.sh is:
            cat /opt/tsg/scripts/prestart.sh
            
--- a/ansible/roles/dp-trace-telemetry/files/helm/templates/clusterrole.yaml
+++ b/ansible/roles/dp-trace-telemetry/files/helm/templates/clusterrole.yaml
@@ -0,0 +1,10 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  labels:
+    serviceFunction: {{ .Release.Name }}
+  name: {{ .Release.Name }}
+rules:
+  - apiGroups: [""]
+    resources: ["services", "nodes"]
+    verbs: ["get", "list", "watch"]
--- a/ansible/roles/dp-trace-telemetry/files/helm/templates/clusterrolebinding.yaml
+++ b/ansible/roles/dp-trace-telemetry/files/helm/templates/clusterrolebinding.yaml
@@ -0,0 +1,14 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  labels:
+    serviceFunction: {{ .Release.Name }}
+  name: {{ .Release.Name }}
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: {{ .Release.Name }}
+subjects:
+  - kind: ServiceAccount
+    name: {{ .Release.Name }}
+    namespace: {{ .Release.Namespace }}
--- a/ansible/roles/dp-trace-telemetry/files/helm/templates/deployment.yaml
+++ b/ansible/roles/dp-trace-telemetry/files/helm/templates/deployment.yaml
@@ -26,6 +26,7 @@ spec:
        prometheus.io/port: "9005"
        prometheus.io/scrape: "true"
    spec:
+      serviceAccountName: {{ .Release.Name }}
      shareProcessNamespace: true
      tolerations:
      - key: node-role.kubernetes.io/control-plane
@@ -42,6 +43,14 @@ spec:
          - "bash"
          - "-ec"
          - |
+            dnf -y install autoconf automake libtool
+            wget -c https://github.com/inotify-tools/inotify-tools/archive/refs/tags/3.21.9.6.tar.gz
+            tar -zvxf 3.21.9.6.tar.gz -C /usr/local/src/
+            cd /usr/local/src/inotify-tools-3.21.9.6/
+            ./autogen.sh && ./configure --prefix=/usr/local/inotify-tools && make && make install
+            echo 'export PATH=$PATH:/usr/local/inotify-tools/bin/' >> ~/.bashrc
+            source ~/.bashrc
+
            WATCH_DIR="/opt/tsg/dp_trace_telemetry/etc/dynamic"
            while inotifywait -r -e modify,create "$WATCH_DIR"; do
              echo "send HUB signal to dp_trace"
@@ -60,9 +69,21 @@ spec:
          - "-ec"
          - |
            ldconfig
-            {{ template "dp_trace_telemetry.prestart" . }}
            cp /opt/tsg/dp_trace_telemetry/etc/dp_trace_configmap.conf /opt/tsg/dp_trace_telemetry/etc/dp_trace.conf
-            cp /opt/tsg/dp_trace_telemetry/etc/dynamic/config /opt/tsg/dp_trace_telemetry/etc/dp_trace_dy.conf
+            {{- if eq .Values.datapath_trace.cm.connectivity "local_cache" }}
+            export APISERVER=https://kubernetes.default.svc
+            export SERVICEACCOUNT=/var/run/secrets/kubernetes.io/serviceaccount
+            export NAMESPACE=$(cat ${SERVICEACCOUNT}/namespace)
+            export TOKEN=$(cat ${SERVICEACCOUNT}/token)
+            export CACERT=${SERVICEACCOUNT}/ca.crt
+            curl --cacert ${CACERT} --header "Authorization: Bearer ${TOKEN}" -X GET ${APISERVER}/api/v1/namespaces/tsg-os-system/services/{{ .Values.datapath_trace.cm.local_cache.cache_name }}-redis-master -o /tmp/cm-cache.txt
+            export CM_POLICY_LOCAL_CACHE_IP=$(cat /tmp/cm-cache.txt  | jq -r '.spec.clusterIP')
+            echo "export CM_POLICY_LOCAL_CACHE_IP=${CM_POLICY_LOCAL_CACHE_IP}" > /etc/profile.d/cm-local-cache.sh
+            chmod 0755 /etc/profile.d/cm-local-cache.sh
+            sed -Ei -c "s|CM_POLICY_LOCAL_CACHE_IP_LOCATION|${CM_POLICY_LOCAL_CACHE_IP?}|g" /opt/tsg/dp_trace_telemetry/etc/dp_trace.conf
+            {{- end }}
+            ln -sf /opt/tsg/dp_trace_telemetry/etc/dynamic/config /opt/tsg/dp_trace_telemetry/etc/dp_trace_dy.conf
+            {{ template "dp_trace_telemetry.prestart" . }}
            {{ template "dp_trace_telemetry.start" . }}
        volumeMounts:
        - name: dp-telemetry-daemon
@@ -93,11 +114,11 @@ spec:
        - name: ldconfig-mrzcpd
          mountPath: /etc/ld.so.conf.d/mrzcpd.conf
          readOnly: true
-        {{- if eq .Values.dp_trace_telemetry.debug.enable_mount_host_filesystem .Values.define_enable_val_yes }}
+        {{- if eq .Values.datapath_trace.debug.enable_mount_host_filesystem .Values.define_enable_val_yes }}
        - name: host-root
          mountPath: /host
        {{- end }}
-        {{- if eq .Values.dp_trace_telemetry.debug.enable_prestart_script .Values.define_enable_val_yes }}
+        {{- if eq .Values.datapath_trace.debug.enable_prestart_script .Values.define_enable_val_yes }}
        - name: prestart-dir
          mountPath: /tmp/prestart
        - name: dp-trace-prestart
@@ -112,15 +133,17 @@ spec:
          value: dp-trace-telemetry
        securityContext:
          privileged: true
-{{- if eq .Values.dp_trace_telemetry.debug.enable_liveness_probe .Values.define_enable_val_yes }}
+        ports:
+          - containerPort: 9086
+{{- if eq .Values.datapath_trace.debug.enable_liveness_probe .Values.define_enable_val_yes }}
        livenessProbe:
          httpGet:
            httpHeaders:
            - name: Custom-Header
              value: Awesome
            path: /probe
-            port: 10000
-          initialDelaySeconds: 120
+            port: 9086
+          initialDelaySeconds: 30
          periodSeconds: 3
          successThreshold: 1
          timeoutSeconds: 10
@@ -130,8 +153,8 @@ spec:
            - name: Custom-Header
              value: Awesome
            path: /probe
-            port: 10000
-          initialDelaySeconds: 120
+            port: 9086
+          initialDelaySeconds: 30
          periodSeconds: 3
          successThreshold: 1
          timeoutSeconds: 10
@@ -149,6 +172,25 @@ spec:
        - name: ex-trace-port
          containerPort: {{ .Values.mergeExporter.mergePort }}
          protocol: TCP
+      {{- if eq .Values.datapath_trace.cm.connectivity "local_cache" }}
+      initContainers:
+      - name: init-default-svc
+        image: "registry.gdnt-cloud.website/tsg-init:{{ .Chart.AppVersion }}"
+        imagePullPolicy: Never
+        command:
+          - "bash"
+          - "-ec"
+          - |
+            until nslookup kubernetes.default.svc; do echo waiting for kubernetes service; sleep 2; done
+      - name: init-cm-svc
+        image: "registry.gdnt-cloud.website/tsg-init:{{ .Chart.AppVersion }}"
+        imagePullPolicy: Never
+        command:
+          - "bash"
+          - "-ec"
+          - |
+            until nslookup {{ .Values.datapath_trace.cm.local_cache.cache_name }}-redis-master.tsg-os-system.svc; do echo waiting for cm cache service; sleep 2; done
+      {{- end }}
      volumes:
      - name: journal-volume
        hostPath:
@@ -160,8 +202,8 @@ spec:
          type: DirectoryOrCreate
      - name: dp-trace-prestart
        hostPath:
-      {{- if .Values.dp_trace_telemetry.debug.prestart_script }}
-          path: {{ .Values.dp_trace_telemetry.debug.prestart_script }}
+      {{- if .Values.datapath_trace.debug.prestart_script }}
+          path: {{ .Values.datapath_trace.debug.prestart_script }}
      {{- else }}
          path: /etc/tsg-os/{{ .Release.Name }}/dp_trace_prestart_script.sh
      {{- end }}
--- a/ansible/roles/dp-trace-telemetry/files/helm/templates/dp-trace.yaml
+++ b/ansible/roles/dp-trace-telemetry/files/helm/templates/dp-trace.yaml
@@ -16,16 +16,23 @@ data:
    monit_file_path=/var/run/mrzcpd/mrmonit.app.dp_trace_telemetry.saving

    [http_server]
-    listen_addr=127.0.0.1
-    listen_port=10000
+    listen_addr=0.0.0.0
+    listen_port=9086
    keep_alive_path=/probe

    [kafka]
    kafka_dump_to_log=0
-    borker_list="192.168.44.12:9094"
+    {{- if .Values.datapath_trace.olap.kafka_brokers}}
+    borker_list="{{- range $index,$kafka := .Values.datapath_trace.olap.kafka_brokers.addresses }}{{- if ne $index 0 }},{{ end -}}{{ $kafka.address }}:{{ $kafka.port}}{{- end }}"
    topic_name="DATAPATH-TELEMETRY-RECORD"
-    sasl_username=admin
-    sasl_password=galaxy2019
+    sasl_username={{.Values.datapath_trace.olap.kafka_brokers.sasl_username }}
+    sasl_password={{.Values.datapath_trace.olap.kafka_brokers.sasl_password }}
+    {{- else }}
+    borker_list=""
+    topic_name="DATAPATH-TELEMETRY-RECORD"
+    sasl_username=
+    sasl_password=
+    {{- end }}

    [maat]
    maat_log_level=3
@@ -34,8 +41,18 @@ data:
    deferred_load_on=0
    table_schema=../etc/table_schema.json
    json_cfg_file=../etc/dp_telemetry_rules.json
-    maat_redis_server=192.168.44.3
-    maat_redis_port_range=7002
+    {{- if .Values.datapath_trace.cm}}
+    {{- if eq .Values.datapath_trace.cm.connectivity "local_cache"}}
+    maat_redis_server=CM_POLICY_LOCAL_CACHE_IP_LOCATION
+    maat_redis_port_range=6379
+    {{- else }}
+    maat_redis_server={{ .Values.datapath_trace.cm.direct.address}}
+    maat_redis_port_range={{ .Values.datapath_trace.cm.direct.port}}
+    {{- end }}
+    {{- else }}
+    maat_redis_server=
+    maat_redis_port_range=
+    {{- end }}
    maat_redis_db_index=1
 ---
 apiVersion: v1
--- a/ansible/roles/dp-trace-telemetry/files/helm/templates/serviceaccount.yaml
+++ b/ansible/roles/dp-trace-telemetry/files/helm/templates/serviceaccount.yaml
@@ -0,0 +1,6 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  labels:
+    serviceFunction: {{ .Release.Name }}
+  name: {{ .Release.Name }}
--- a/ansible/roles/dp-trace-telemetry/files/helm/values.yaml
+++ b/ansible/roles/dp-trace-telemetry/files/helm/values.yaml
@@ -85,15 +85,7 @@ mergeExporter:

 define_enable_val_yes: 'yes'

-datapath_trace: {}
+datapath_trace: { debug: { enable_liveness_probe:'no', enable_interactive_startup: 'no', enable_prestart_script: 'no', enable_mount_host_filesystem:'no', prestart_script: ""}}

 device: {}

-dp_trace_telemetry:
-  debug:
-    enable_liveness_probe: 'no'
-    define_enable_val_yes: 'yes'
-    enable_prestart_script: 'yes'
-    enable_mount_host_filesystem: 'yes'
-    #default: /etc/tsg-os/${service_function_name}/shaping_prestart_script.sh
-    prestart_script: ""
--- a/tools/mk-base-image
+++ b/tools/mk-base-image
@@ -42,7 +42,7 @@ case $profile_id in
                    containerd.io lrzsz python3 watchdog pcm git tmux fish kernel kernel-devel kernel-tools-libs kernel-modules
                    kernel-tools kernel-core rpm-build libtool kernel-rpm-macros python36-devel tcsh kernel-modules-extra gcc-gfortran
                    libdb-devel fuse-devel python3-Cython cmake perl-generators libstdc++-devel libmnl-devel bison flex gcc-c++
-                    python3-docutils libnsl liburing hwloc-gui perl-open perl python2 js-d3-flame-graph xmlstarlet conntrack-tools crudini inotify-tools"
+                    python3-docutils libnsl liburing hwloc-gui perl-open perl python2 js-d3-flame-graph xmlstarlet conntrack-tools crudini"

        ;;
    "7400-MCN0-P01R01" | "7400-MCN123-P01R01")