✨ feat: TSG-17785 为packet-io增加热加载功能

🌈 style: remove spaces of mrzcpd task.yml ✨ feat: TSG-17785 为packet-io增加热加载功能 🌈 style: remove spaces of mrzcpd task.yml 🌈 style: change packet-io reload config style 🐎 ci: change confd file name,redesign mrzcpd yaml ✨ feat: TSG-17785 为packet-io增加热加载功能 🌈 style: remove spaces of mrzcpd task.yml ✨ feat: TSG-17785 为packet-io增加热加载功能 🌈 style: remove spaces of mrzcpd task.yml 🌈 style: change packet-io reload config style 🐎 ci: change confd file name,redesign mrzcpd yaml 🌈 style: 更改packet-io 光保配置项名称为obp 🐞 fix: 修改packet-io动态配置文件的保存目录 ✨ feat: prometheus增加packet-io中obp光保信息
2023-11-21 15:26:51 +08:00
parent b545d66dd2
commit 4101a1b475
12 changed files with 178 additions and 41 deletions
--- a/ansible/HAL_deploy.yml
+++ b/ansible/HAL_deploy.yml
@@ -210,6 +210,7 @@
  roles:
    - {role: framework, tags: framework}
    - {role: mrzcpd, tags: mrzcpd}
+    - {role: confd, tags: confd}

 - hosts: server
  remote_user: root
--- a/ansible/roles/confd/files/confd
+++ b/ansible/roles/confd/files/confd
--- a/ansible/roles/confd/files/packet-io-conf.toml
+++ b/ansible/roles/confd/files/packet-io-conf.toml
@@ -0,0 +1,7 @@
+[template]
+src = "packet-io.conf.tmpl"
+dest = "/opt/tsg/config/shared/mrglobal.dynamic.conf"
+keys = [
+  "/obp_devices/"
+]
+reload_cmd = "/opt/tsg/mrzcpd/bin/signal.sh"
--- a/ansible/roles/confd/files/packet-io.conf.tmpl
+++ b/ansible/roles/confd/files/packet-io.conf.tmpl
@@ -0,0 +1,17 @@
+{{range $key, $dir := lsdir "/obp_devices"}}{{$name_dir := printf "/obp_devices/%s/name" $dir}}{{$type_dir := printf "/obp_devices/%s/type" $dir}}{{$connect_dir := printf "/obp_devices/%s/connect" $dir}}{{$address_dir := printf "/obp_devices/%s/address" $dir}}{{$port_dir := printf "/obp_devices/%s/port" $dir}}{{$segment_dir := printf "/obp_devices/%s/segment" $dir}}[olp_device:{{$key}}]
+name = {{getv $name_dir}}
+type = {{getv $type_dir}}
+connect = {{getv $connect_dir}}
+in_addr = {{getv $address_dir}}
+port = {{getv $port_dir}}
+{{$num := ""}}
+{{range $segkey, $segdir := lsdir $segment_dir}}{{$index:=len $num}}{{$id_dir := printf "/obp_devices/%s/segment/%d/id" $dir $index}}{{$workline_dir := printf "/obp_devices/%s/segment/%d/workline" $dir $index}}{{$heartbeat_modeline_dir := printf "/obp_devices/%s/segment/%d/heartbeat_mode" $dir $index}}{{$hb_send_interval_in_ms_dir := printf "/obp_devices/%s/segment/%d/hb_send_interval_in_ms" $dir $index}}{{$hb_timeout_interval_in_ms_dir := printf "/obp_devices/%s/segment/%d/hb_timeout_interval_in_ms" $dir $index}}{{$hb_lost_threshold_dir := printf "/obp_devices/%s/segment/%d/hb_lost_threshold" $dir $index}}{{$nonrevertive_mode_dir := printf "/obp_devices/%s/segment/%d/nonrevertive_mode" $dir $index}}{{$num = printf "%s1" $num}}
+[{{getv $name_dir}}:{{getv $id_dir}}]
+state = {{getv $workline_dir}}
+heartbeat = {{getv $heartbeat_modeline_dir}}
+heartbeat_send_interval_in_ms= {{getv $hb_send_interval_in_ms_dir}}
+heartbeat_timeout_interval_in_ms= {{getv $hb_timeout_interval_in_ms_dir}} 
+heartbeat_lost_threshold = {{getv $hb_lost_threshold_dir}}
+nonrevertive_mode = {{getv $nonrevertive_mode_dir}}
+{{end}}
+{{end}}
--- a/ansible/roles/confd/tasks/main.yml
+++ b/ansible/roles/confd/tasks/main.yml
@@ -0,0 +1,47 @@
+- name: "Create /opt/confd/bin directory if it does not exist"
+  file:
+    path: "{{ item }}"
+    state: directory
+    mode: '0755'
+  with_items:
+    - /opt/confd/bin
+  when: runtime_env == 'TSG-X-P0906'
+
+- name: "Create /etc/confd/conf.d directory if it does not exist"
+  file:
+    path: "{{ item }}"
+    state: directory
+    mode: '0755'
+  with_items:
+    - /etc/confd/conf.d
+  when: runtime_env == 'TSG-X-P0906'
+
+- name: "Create /etc/confd/templates directory if it does not exist"
+  file:
+    path: "{{ item }}"
+    state: directory
+    mode: '0755'
+  with_items:
+    - /etc/confd/templates
+  when: runtime_env == 'TSG-X-P0906'
+
+- name: "copy confd to dest"
+  copy:
+    src: "{{ role_path }}/files/confd"
+    dest: /opt/confd/bin/confd
+    mode: 0755
+  when: runtime_env == 'TSG-X-P0906'
+
+- name: "copy packet-io-conf.toml to dest"
+  copy:
+    src: "{{ role_path }}/files/packet-io-conf.toml"
+    dest: /etc/confd/conf.d/packet-io-conf.toml
+    mode: 0755
+  when: runtime_env == 'TSG-X-P0906'
+
+- name: "copy packet-io.conf.tmpl to dest"
+  copy:
+    src: "{{ role_path }}/files/packet-io.conf.tmpl"
+    dest: /etc/confd/templates/packet-io.conf.tmpl
+    mode: 0755
+  when: runtime_env == 'TSG-X-P0906'
--- a/ansible/roles/mrzcpd/files/helm/templates/_config.tpl
+++ b/ansible/roles/mrzcpd/files/helm/templates/_config.tpl
@@ -1,27 +1,3 @@
-{{- define "mrzcpd.mount.hostpath" -}}
-{{- if eq .Values.mrzcpd.debug.enable_mount_host_filesystem  .Values.define_enable_val_yes }}
-        - name: host-root
-          mountPath: /host
-{{- end }}
-{{- end -}}
-
-{{- define "mrzcpd.volume.hostpath" -}}
-      - name: host-root
-        hostPath:
-          path: /
-{{- end -}}
-
-{{- define "mrzcpd.mount.localtime" -}}
-        - name: localtime-node
-          mountPath: /etc/localtime
-          readOnly: true
-{{- end -}}
-
-{{- define "mrzcpd.volume.localtime" -}}
-      - name: localtime-node
-        hostPath:
-          path: /etc/localtime
-{{- end -}}

 {{- define "mrzcpd.resources" -}}
 {{- if .Values.mrzcpd.interfaces }}
@@ -35,7 +11,7 @@
 {{- if eq .Values.mrzcpd.debug.enable_interactive_startup .Values.define_enable_val_yes -}}
            while true; do sleep 10;done
 {{- else -}}
-           /opt/tsg/mrzcpd/bin/mrzcpd -c /opt/tsg/mrzcpd/etc/mrglobal.conf
+           /opt/tsg/mrzcpd/bin/mrzcpd -c /opt/tsg/mrzcpd/etc/mrglobal.conf -s /opt/tsg/mrzcpd/etc/mrglobal.dynamic.conf
 {{- end }}
 {{- end -}}

@@ -329,6 +305,12 @@ interface_ext=virtio_dign_s
 [vwire:{{ $virtual_wire.id }}]
 interface_int={{ $virtual_wire.internal_interface }}
 interface_ext={{ $virtual_wire.external_interface }}
+{{- if $virtual_wire.obp_device }}
+obp_device={{ $virtual_wire.obp_device }}
+{{- end }}
+{{- if $virtual_wire.obp_segment }}
+obp_segment={{ $virtual_wire.obp_segment }}
+{{- end }}
 {{$enter}}
 {{- end }}
 # classifier rule
--- a/ansible/roles/mrzcpd/files/helm/templates/deployment-mrzcpd.yaml
+++ b/ansible/roles/mrzcpd/files/helm/templates/deployment-mrzcpd.yaml
@@ -6,7 +6,7 @@ metadata:
    app: packet-io-engine
    component: packet-io-engine
  annotations:
-    reloader.stakater.com/auto: "true"
+    configmap.reloader.stakater.com/reload: mrzcpd-static-conf

 spec:
  replicas: 1
@@ -25,7 +25,8 @@ spec:
        prometheus.io/port: "9005"
        prometheus.io/scrape: "true"
    
-    spec: 
+    spec:
+      shareProcessNamespace: true
      tolerations:
      - key: node-role.kubernetes.io/control-plane
        operator: Exists
@@ -49,13 +50,16 @@ spec:
            cp -f /etc/profile.d/mrzcpd.sh /mnt/packet-io-engine/profile.d/mrzcpd.sh
            ldconfig -C /etc/ld.so.cache -r /host

-            cp -f /opt/tsg/config/mrzcpd.conf /opt/tsg/mrzcpd/etc/mrglobal.conf
+            cp -f /opt/tsg/config/mrzcpd-static-conf/static.conf /opt/tsg/mrzcpd/etc/mrglobal.conf
            chmod +x /opt/tsg/mrzcpd/bin/replaceconf.sh
            /opt/tsg/mrzcpd/bin/replaceconf.sh
-
+            ln -s /opt/tsg/mrzcpd/shared/mrglobal.dynamic.conf /opt/tsg/mrzcpd/etc/mrglobal.dynamic.conf
+            
            {{ template "mrzcpd.prestart" . }}
            {{ template "mrzcpd.start" . }}
        volumeMounts:
+        - name: shared-dir
+          mountPath: /opt/tsg/mrzcpd/shared
        - name: mrzcpd
          mountPath: /mnt/packet-io-engine/mrzcpd
          mountPropagation: Bidirectional
@@ -63,9 +67,8 @@ spec:
          mountPath: /mnt/packet-io-engine/ld.so.conf.d
        - name: etc-profile
          mountPath: /mnt/packet-io-engine/profile.d
-        - name: mrzcpd-conf
-          mountPath: "/opt/tsg/config/mrzcpd.conf"
-          subPath: mrzcpd.conf
+        - name: mrzcpd-static-conf
+          mountPath: /opt/tsg/config/mrzcpd-static-conf/
        - name: run-mrzcpd
          mountPath: /var/run/mrzcpd
        - name: hugepages
@@ -79,12 +82,18 @@ spec:
        - name: host-root
          mountPath: /host
          mountPropagation: HostToContainer
-        {{ template "mrzcpd.mount.localtime" . }}
+        - name: localtime-node
+          mountPath: /etc/localtime
+          readOnly: true
        {{ template "mrzcpd.mount.prestart" . }}
        resources: 
          limits:
            mrzcpd/mrzcpd: 1
-        {{ template "mrzcpd.resources" . }}
+{{- if .Values.mrzcpd.interfaces }}
+{{- range $interface := .Values.mrzcpd.interfaces }}
+            mrzcpd/{{ $interface.name }}: 1
+{{- end }}
+{{- end }}
            hugepages-1Gi: {{.Values.mrzcpd.hugepages }}
          requests:
            memory: 100Mi
@@ -131,6 +140,26 @@ spec:
        securityContext:
          privileged: true

+
+      - name: confd
+        image: "registry.gdnt-cloud.website/tsg-packet-io-engine:{{ .Chart.AppVersion }}"
+        imagePullPolicy: Never
+        lifecycle:
+          type: Sidecar
+        command:
+          - "bash"
+          - "-ec"
+          - |
+            chmod +x /opt/tsg/mrzcpd/bin/signal.sh
+            /opt/confd/bin/confd -watch -backend file -file /opt/tsg/config/mrzcpd-reload-conf/reload-conf.yaml -log-level debug
+        volumeMounts:
+        - name: shared-dir
+          mountPath: /opt/tsg/config/shared
+        - name: mrzcpd-reload-conf
+          mountPath: /opt/tsg/config/mrzcpd-reload-conf/
+        securityContext:
+          privileged: true
+
      - command:
        - /opt/tsg/mrzcpd/bin/monit_device
        - --prometheus-client
@@ -163,6 +192,22 @@ spec:
        - mountPath: /etc/localtime
          name: localtime-node
          readOnly: true
+      - command:
+        - /opt/tsg/mrzcpd/bin/monit_obp
+        - --prometheus-client
+        image: "registry.gdnt-cloud.website/tsg-packet-io-engine:{{ .Chart.AppVersion }}"
+        imagePullPolicy: Never
+        name: monit-obp
+        volumeMounts:
+        - mountPath: /var/run/mrzcpd
+          name: run-mrzcpd
+          readOnly: true
+        - mountPath: /var/run/dpdk
+          name: dpdk
+          readOnly: true
+        - mountPath: /etc/localtime
+          name: localtime-node
+          readOnly: true
      - env:
        - name: MERGER_URLS
          value: {{ .Values.mergeExporter.mergeUrls }}
@@ -176,13 +221,18 @@ spec:
          containerPort: {{ .Values.mergeExporter.mergePort }}
          protocol: TCP
      volumes:
+      - name: shared-dir
+        emptyDir: {}
      - name: prestart-dir
        hostPath: 
          path: /etc/tsg-os/{{ .Release.Name }}/
          type: DirectoryOrCreate
-      - name: mrzcpd-conf
+      - name: mrzcpd-static-conf
        configMap:
-          name: mrzcpd
+          name: mrzcpd-static-conf
+      - name: mrzcpd-reload-conf
+        configMap:
+          name: mrzcpd-reload-conf
      - name: mrzcpd
        hostPath:
          path: /opt/tsg/mrzcpd
@@ -214,7 +264,11 @@ spec:
      - name: dev-vfio
        hostPath:
          path: /dev/vfio
-      {{ template "mrzcpd.volume.localtime" . }}
+      - name: localtime-node
+        hostPath:
+          path: /etc/localtime
+      - name: host-root
+        hostPath:
+          path: /
      {{ template "mrzcpd.volume.prestart" . }}
-      {{ template "mrzcpd.volume.hostpath" . }}
      hostNetwork: true
--- a/ansible/roles/mrzcpd/files/helm/templates/mrzcpd.yaml
+++ b/ansible/roles/mrzcpd/files/helm/templates/mrzcpd.yaml
@@ -1,6 +1,15 @@
 apiVersion: v1
 kind: ConfigMap
 metadata:
-  name: mrzcpd
+  name: mrzcpd-static-conf
 data:
-  mrzcpd.conf: {{ tpl (.Files.Get "conf/mrzcpd.conf") . | quote }}
+  static.conf: {{ tpl (.Files.Get "conf/mrzcpd.conf") . | quote }}
+
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: mrzcpd-reload-conf
+data:
+  reload-conf.yaml: |
+    obp_devices: {{ .Values.mrzcpd.obp_devices | toJson }}
--- a/ansible/roles/mrzcpd/files/helm/values.yaml
+++ b/ansible/roles/mrzcpd/files/helm/values.yaml
@@ -85,7 +85,7 @@ mergeExporter:
    pullPolicy: Never
    tag: "v0.2.0"
  mergePort: "9005"
-  mergeUrls: "http://0.0.0.0:8901/metrics http://0.0.0.0:8902/metrics"
+  mergeUrls: "http://0.0.0.0:8901/metrics http://0.0.0.0:8902/metrics http://0.0.0.0:8903/metrics"

 service:
  type: ClusterIP
@@ -105,6 +105,7 @@ mrzcpd:
  traffic_mirror: {}
  distmode: 0
  hugepages: 32Gi
+  obp_devices: [{'name': 'obp1', 'type': '2', 'connect': '3', 'address': '2.2.2.2', 'port': '6800', 'segment': [{'id': '1', 'workline':'0', 'heartbeat_mode': '1', 'hb_send_interval_in_ms': '100', 'hb_timeout_interval_in_ms': '100', 'hb_lost_threshold': '5', 'nonrevertive_mode': '1'}, {'id': '1', 'workline':'0', 'heartbeat_mode': '1', 'hb_send_interval_in_ms': '100', 'hb_timeout_interval_in_ms': '100', 'hb_lost_threshold': '5', 'nonrevertive_mode': '1'}]}]
  debug:
    enable_liveness_probe: yes
    define_enable_val_yes: yes
--- a/ansible/roles/mrzcpd/files/signal.sh
+++ b/ansible/roles/mrzcpd/files/signal.sh
@@ -0,0 +1,10 @@
+#!/bin/sh
+# Find the process PID
+pid=$(pidof mrzcpd)
+if [ -z "$pid" ]; then
+  echo "No process found that matches the criteria"
+  exit 0
+fi
+# Send SIGHUP signal
+kill -HUP "$pid"
+echo "SIGHUP signal sent to process with PID: $pid"
--- a/ansible/roles/mrzcpd/tasks/main.yml
+++ b/ansible/roles/mrzcpd/tasks/main.yml
@@ -77,6 +77,14 @@
    mode: 0644
  when: runtime_env == 'TSG-X-P0906'

+- name: "copy signal.sh to dest"
+  copy:
+    src: "{{ role_path }}/files/signal.sh"
+    dest: /opt/tsg/mrzcpd/bin/signal.sh
+    mode: 0644
+  when: runtime_env == 'TSG-X-P0906'
+
+
 - name: "replace action: replace Requires to Wants"
  replace:
    path: "{{ item }}"
--- a/ansible/roles/tsg-os-HAL/files/script/tsg-os-HAL.sh
+++ b/ansible/roles/tsg-os-HAL/files/script/tsg-os-HAL.sh
@@ -100,6 +100,7 @@ cat <<EOF > ${k3s_cfg_file}
 kubelet-arg:
  - "kube-reserved=memory=${KUBE_RESERVED}Gi"
  - "system-reserved=memory=${system_mem}Gi"
+  - "sync-frequency=1s"
 EOF
 }