feature:TSG-17387:add packet-io-engine prometheus exporter.

2023-10-12 10:40:06 +08:00
parent 013d4f8700
commit 5ff7fc65bb
7 changed files with 123 additions and 83 deletions
--- a/ansible/HAL_deploy.yml
+++ b/ansible/HAL_deploy.yml
@@ -270,7 +270,7 @@
    - {role: framework, tags: framework}
    - {role: packet_adapter, tags: packet_adapter}

- hosts: TSG-X-NXR620G40-R01-P0906-mrzcpd
+- hosts: TSG-X-NXR620G40-R01-P0906-packet-io-engine
  remote_user: root
  vars_files:
    - install_config/group_vars/HAL_TSGXNXR620G40R01P0906.yml
@@ -294,7 +294,7 @@
  roles:
    - {role: container-tools-install, tags: container-tools-install}

- hosts: TSG-X-NXR620G40-R01-P0906-mrzcpd-deviceplugin
+- hosts: TSG-X-NXR620G40-R01-P0906-packet-io-engine-deviceplugin
  remote_user: root
  vars_files:
    - install_config/group_vars/HAL_TSGXNXR620G40R01P0906.yml
--- a/ansible/roles/k3s-exporter/tasks/main.yml
+++ b/ansible/roles/k3s-exporter/tasks/main.yml
@@ -73,10 +73,10 @@
    url_username: "{{ lookup('env', 'PULP_REPO_USERNAME') }}"
    url_password: "{{ lookup('env', 'PULP_REPO_PASSWORD') }}"

- name: "copy prometheus crd file to dest"
-  copy:
-    src: "{{ role_path }}/files/mrzcpd/exporter-mrzcpd-crd.yaml"
-    dest: /var/lib/rancher/k3s/server/manifests/exporter-mrzcpd-crd.yaml
+# - name: "copy prometheus crd file to dest"
+#   copy:
+#     src: "{{ role_path }}/files/mrzcpd/exporter-mrzcpd-crd.yaml"
+#     dest: /var/lib/rancher/k3s/server/manifests/exporter-mrzcpd-crd.yaml

 #### mrzcpd exporter end

--- a/ansible/roles/k3s-install/files/daemonset-mrzcpd-deviceplugin.yaml
+++ b/ansible/roles/k3s-install/files/daemonset-mrzcpd-deviceplugin.yaml
@@ -1,22 +1,22 @@
 apiVersion: apps/v1
 kind: DaemonSet
 metadata:
-  name: packet-io-deviceplugin
+  name: packet-io-engine-deviceplugin
  labels:
-    app: packet-io-deviceplugin
+    app: packet-io-engine-deviceplugin
  namespace: tsg-os-system
 spec:
  selector:
    matchLabels:
-      app: packet-io-deviceplugin
+      app: packet-io-engine-deviceplugin
  template:
    metadata:
      labels:
-        app: packet-io-deviceplugin
+        app: packet-io-engine-deviceplugin
    spec:
      containers:
-      - name: packet-io-deviceplugin
-        image: registry.gdnt-cloud.website/tsg-mrzcpd-deviceplugin:%%MRZCPD_DEVICE_PLUGIN_IMAGE_TAG%%
+      - name: packet-io-engine-deviceplugin
+        image: registry.gdnt-cloud.website/tsg-packet-io-engine-deviceplugin:%%MRZCPD_DEVICE_PLUGIN_IMAGE_TAG%%
        command:
          - "bash"
          - "-ec"
@@ -31,13 +31,11 @@ spec:
          mountPath: /var/lib/kubelet
        - name: modules
          mountPath: /lib/modules
-        - name: mrzcpdtest
-          mountPath: /opt/tsg/mrzcpd
        securityContext:
          privileged: true
      initContainers:
-      - name: init-mrzcpd-deviceplugin
-        image: registry.gdnt-cloud.website/tsg-mrzcpd-deviceplugin:%%MRZCPD_DEVICE_PLUGIN_IMAGE_TAG%%
+      - name: init-packet-io-engine-deviceplugin
+        image: registry.gdnt-cloud.website/tsg-packet-io-engine-deviceplugin:%%MRZCPD_DEVICE_PLUGIN_IMAGE_TAG%%
        command: 
          - "bash"
          - "-ec"
@@ -52,8 +50,6 @@ spec:
          mountPath: /var/lib/kubelet
        - name: modules
          mountPath: /lib/modules
-        - name: mrzcpdtest
-          mountPath: /opt/tsg/mrzcpd
        securityContext:
          privileged: true
      volumes:
@@ -70,7 +66,4 @@ spec:
      - name: modules
        hostPath: 
          path: /lib/modules
-      - name: mrzcpdtest
-        hostPath:
-          path: /root/mrzcpd
      hostNetwork: true
--- a/ansible/roles/mrzcpd/files/helm/templates/deployment-mrzcpd.yaml
+++ b/ansible/roles/mrzcpd/files/helm/templates/deployment-mrzcpd.yaml
@@ -1,10 +1,10 @@
 apiVersion: apps/v1
 kind: Deployment
 metadata:
-  name: {{ .Release.Name }}
+  name: packet-io-engine
  labels:
-    app: {{ .Release.Name }}
-    component: mrzcpd
+    app: packet-io-engine
+    component: packet-io-engine
  annotations:
    reloader.stakater.com/auto: "true"

@@ -12,17 +12,17 @@ spec:
  replicas: 1
  selector:
    matchLabels:
-      app: {{ .Release.Name }}-mrzcpd
+      app: packet-io-engine
  strategy:
    type: Recreate
  template:
    metadata:
      labels:
-        app: {{ .Release.Name }}-mrzcpd
+        app: packet-io-engine
        serviceFunction: {{ .Release.Name }}
-        component: mrzcpd
+        component: packet-io-engine
      annotations:
-        prometheus.io/port: "9086"
+        prometheus.io/port: "9005"
        prometheus.io/scrape: "true"
    
    spec: 
@@ -34,8 +34,8 @@ spec:
        operator: Exists
        effect: NoSchedule
      containers:
-      - name: mrzcpd
-        image: "registry.gdnt-cloud.website/tsg-mrzcpd:{{ .Chart.AppVersion }}"
+      - name: packet-io-engine
+        image: "registry.gdnt-cloud.website/tsg-packet-io-engine:{{ .Chart.AppVersion }}"
        imagePullPolicy: Never
        workingDir: /opt/tsg/mrzcpd
        command: 
@@ -43,16 +43,26 @@ spec:
          - "-ec"
          - |
            /opt/tsg/mrzcpd/bin/mrmarch --auto
-            mount --rbind /opt/tsg/mrzcpd /root/hostmount/mrzcpd
-            cp -f /etc/ld.so.conf.d/mrzcpd.conf /root/hostmount/ld.so.conf.d/mrzcpd.conf
-            cp -f /etc/profile.d/mrzcpd.sh /root/hostmount/profile.d/mrzcpd.sh
+            mount --rbind /opt/tsg/mrzcpd /mnt/packet-io-engine/mrzcpd
+
+            cp -f /etc/ld.so.conf.d/mrzcpd.conf /mnt/packet-io-engine/ld.so.conf.d/mrzcpd.conf
+            cp -f /etc/profile.d/mrzcpd.sh /mnt/packet-io-engine/profile.d/mrzcpd.sh
+            ldconfig -C /etc/ld.so.cache -r /host
+
            cp -f /opt/tsg/config/mrzcpd.conf /opt/tsg/mrzcpd/etc/mrglobal.conf
            chmod +x /opt/tsg/mrzcpd/bin/replaceconf.sh
            /opt/tsg/mrzcpd/bin/replaceconf.sh
-            ldconfig -C /etc/ld.so.cache -r /host
+
            {{ template "mrzcpd.prestart" . }}
            {{ template "mrzcpd.start" . }}
        volumeMounts:
+        - name: mrzcpd
+          mountPath: /mnt/packet-io-engine/mrzcpd
+          mountPropagation: Bidirectional
+        - name: ldconfig-mrzcpd
+          mountPath: /mnt/packet-io-engine/ld.so.conf.d
+        - name: env-sh
+          mountPath: /mnt/packet-io-engine/profile.d
        - name: mrzcpd-conf
          mountPath: "/opt/tsg/config/mrzcpd.conf"
          subPath: mrzcpd.conf
@@ -60,13 +70,6 @@ spec:
          mountPath: /var/run/mrzcpd
        - name: hugepages
          mountPath: /var/run/mrzcpd/hugepages
-        - name: mrzcpd
-          mountPath: /root/hostmount/mrzcpd
-          mountPropagation: Bidirectional
-        - name: ldconfig-mrzcpd
-          mountPath: /root/hostmount/ld.so.conf.d
-        - name: env-sh
-          mountPath: /root/hostmount/profile.d
        - name: dpdk
          mountPath: /var/run/dpdk
        - name: container-mrzcpd
@@ -87,8 +90,6 @@ spec:
            hugepages-1Gi: {{.Values.mrzcpd.hugepages }}
          requests:
            memory: 100Mi
-        ports:
-          - containerPort: 9086
        env:
        - name: MLX5_GLUE_PATH
          value: /opt/tsg/mrzcpd/lib
@@ -106,45 +107,57 @@ spec:
 {{- end }}
        securityContext:
          privileged: true
-      
-      - name: mrapm-device
-        image: "registry.gdnt-cloud.website/tsg-mrzcpd:{{ .Chart.AppVersion }}"
-        imagePullPolicy: Never
-        workingDir: /opt/tsg/mrzcpd
-        command: 
-          - "bash"
-          - "-ec"
-          - |
-            /opt/tsg/mrzcpd/bin/monit_device --prometheus-client
-        volumeMounts:
-        - name: run-mrzcpd
-          mountPath: /var/run/mrzcpd
-        - name: dpdk
-          mountPath: /var/run/dpdk
-        - name: root-sys
-          mountPath: /root/sys
-        {{ template "mrzcpd.mount.localtime" . }}
-        {{ template "mrzcpd.mount.prestart" . }}
-        {{ template "mrzcpd.mount.hostpath" . }}

-      - name: mrapm-stream
-        image: "registry.gdnt-cloud.website/tsg-mrzcpd:{{ .Chart.AppVersion }}"
+      - command:
+        - /opt/tsg/mrzcpd/bin/monit_device
+        - --prometheus-client
+        image: "registry.gdnt-cloud.website/tsg-packet-io-engine:{{ .Chart.AppVersion }}"
        imagePullPolicy: Never
-        workingDir: /opt/tsg/mrzcpd
-        command: 
-          - "bash"
-          - "-ec"
-          - |
-            /opt/tsg/mrzcpd/bin/monit_stream --prometheus-client
+        name: mrapm-device
        volumeMounts:
-        - name: run-mrzcpd
-          mountPath: /var/run/mrzcpd
-        - name: dpdk
-          mountPath: /var/run/dpdk
-        {{ template "mrzcpd.mount.localtime" . }}
-        {{ template "mrzcpd.mount.prestart" . }}
-        {{ template "mrzcpd.mount.hostpath" . }}
-
+        - mountPath: /var/run/mrzcpd
+          name: run-mrzcpd
+          readOnly: true
+        - mountPath: /var/run/dpdk
+          name: dpdk
+          readOnly: true
+        - mountPath: /root/sys
+          name: root-sys
+          readOnly: true
+        - mountPath: /etc/localtime
+          name: localtime-node
+          readOnly: true
+      - command:
+        - /opt/tsg/mrzcpd/bin/monit_stream
+        - --prometheus-client
+        image: "registry.gdnt-cloud.website/tsg-packet-io-engine:{{ .Chart.AppVersion }}"
+        imagePullPolicy: Never
+        name: monit-stream
+        volumeMounts:
+        - mountPath: /var/run/mrzcpd
+          name: run-mrzcpd
+          readOnly: true
+        - mountPath: /var/run/dpdk
+          name: dpdk
+          readOnly: true
+        - mountPath: /root/sys
+          name: root-sys
+          readOnly: true
+        - mountPath: /etc/localtime
+          name: localtime-node
+          readOnly: true
+      - env:
+        - name: MERGER_URLS
+          value: {{ .Values.mergeExporter.mergeUrls }}
+        - name: MERGER_PORT
+          value: "{{ .Values.mergeExporter.mergePort }}"
+        image: "{{ .Values.mergeExporter.image.repository }}:{{ .Values.mergeExporter.image.tag }}"
+        imagePullPolicy: {{ .Values.mergeExporter.image.pullPolicy }}
+        name: exporter-merger
+        ports:
+        - name: ex-mrzcpd-port
+          containerPort: {{ .Values.mergeExporter.mergePort }}
+          protocol: TCP
      volumes:
      - name: prestart-dir
        hostPath: 
--- a/ansible/roles/mrzcpd/files/helm/templates/service.yaml
+++ b/ansible/roles/mrzcpd/files/helm/templates/service.yaml
@@ -0,0 +1,20 @@
+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+    app: packet-io-engine
+  name: packet-io-engine
+  annotations:
+    prometheus.io/scrape: "true"
+
+spec:
+  ports:
+  - name: mrzcpd-exporter-port
+    {{- if .Values.service.nodePort }}
+    nodePort: {{ .Values.service.nodePort }}
+    {{- end }}
+    port: {{ .Values.service.port }}
+    targetPort: ex-mrzcpd-port
+  selector:
+    app: packet-io-engine
+  type: {{ .Values.service.type }}
--- a/ansible/roles/mrzcpd/files/helm/values.yaml
+++ b/ansible/roles/mrzcpd/files/helm/values.yaml
@@ -79,6 +79,20 @@ tolerations: []
 affinity: {}


+mergeExporter:
+  image:
+    repository: quay.io/rebuy/exporter-merger
+    pullPolicy: Never
+    tag: "v0.2.0"
+  mergePort: "9005"
+  mergeUrls: "http://0.0.0.0:8901/metrics http://0.0.0.0:8902/metrics"
+
+service:
+  type: ClusterIP
+#  nodePort: "30081"
+  port: "9005"
+  annotations: {}
+
 define_enable_val_yes: 'yes'

 mrzcpd:
--- a/tools/build_container_images.sh
+++ b/tools/build_container_images.sh
@@ -66,9 +66,9 @@ function export_images()
            ${IMAGE_REPO_PREFIX}/tsg-sce:${OS_RELEASE_VER} \
            ${IMAGE_REPO_PREFIX}/tsg-bfdd:${OS_RELEASE_VER} \
            ${IMAGE_REPO_PREFIX}/tsg-inject:${OS_RELEASE_VER} \
-            ${IMAGE_REPO_PREFIX}/tsg-mrzcpd:${OS_RELEASE_VER} \
+            ${IMAGE_REPO_PREFIX}/tsg-packet-io-engine:${OS_RELEASE_VER} \
            ${IMAGE_REPO_PREFIX}/tsg-shaping:${OS_RELEASE_VER} \
-            ${IMAGE_REPO_PREFIX}/tsg-mrzcpd-deviceplugin:${OS_RELEASE_VER}
+            ${IMAGE_REPO_PREFIX}/tsg-packet-io-engine-deviceplugin:${OS_RELEASE_VER}
 }
 ##function define end

@@ -86,8 +86,8 @@ build_image "sce"       "${BASE_IMAGE_NAME}" "${IMAGE_REPO_PREFIX}/tsg-sce:${OS_
 build_image "bfdd"      "${BASE_IMAGE_NAME}" "${IMAGE_REPO_PREFIX}/tsg-bfdd:${OS_RELEASE_VER}"
 build_image "shaping"   "${BASE_IMAGE_NAME}" "${IMAGE_REPO_PREFIX}/tsg-shaping:${OS_RELEASE_VER}"
 build_image "inject"    "${BASE_IMAGE_NAME}" "${IMAGE_REPO_PREFIX}/tsg-inject:${OS_RELEASE_VER}"
-build_image "mrzcpd"    "${BASE_IMAGE_NAME}" "${IMAGE_REPO_PREFIX}/tsg-mrzcpd:${OS_RELEASE_VER}"
-build_image "mrzcpd-deviceplugin"    "${BASE_IMAGE_NAME}" "${IMAGE_REPO_PREFIX}/tsg-mrzcpd-deviceplugin:${OS_RELEASE_VER}"
+build_image "packet-io-engine"    "${BASE_IMAGE_NAME}" "${IMAGE_REPO_PREFIX}/tsg-packet-io-engine:${OS_RELEASE_VER}"
+build_image "packet-io-engine-deviceplugin"    "${BASE_IMAGE_NAME}" "${IMAGE_REPO_PREFIX}/tsg-packet-io-engine-deviceplugin:${OS_RELEASE_VER}"

 export_images