From 5ff7fc65bb170814a45a5857d19aa7cc5ac78ae2 Mon Sep 17 00:00:00 2001 From: linxin Date: Thu, 12 Oct 2023 10:40:06 +0800 Subject: [PATCH] feature:TSG-17387:add packet-io-engine prometheus exporter. --- ansible/HAL_deploy.yml | 4 +- ansible/roles/k3s-exporter/tasks/main.yml | 8 +- .../files/daemonset-mrzcpd-deviceplugin.yaml | 23 ++-- .../helm/templates/deployment-mrzcpd.yaml | 129 ++++++++++-------- .../mrzcpd/files/helm/templates/service.yaml | 20 +++ ansible/roles/mrzcpd/files/helm/values.yaml | 14 ++ tools/build_container_images.sh | 8 +- 7 files changed, 123 insertions(+), 83 deletions(-) create mode 100644 ansible/roles/mrzcpd/files/helm/templates/service.yaml diff --git a/ansible/HAL_deploy.yml b/ansible/HAL_deploy.yml index 0f46029f..48043f6c 100644 --- a/ansible/HAL_deploy.yml +++ b/ansible/HAL_deploy.yml @@ -270,7 +270,7 @@ - {role: framework, tags: framework} - {role: packet_adapter, tags: packet_adapter} -- hosts: TSG-X-NXR620G40-R01-P0906-mrzcpd +- hosts: TSG-X-NXR620G40-R01-P0906-packet-io-engine remote_user: root vars_files: - install_config/group_vars/HAL_TSGXNXR620G40R01P0906.yml @@ -294,7 +294,7 @@ roles: - {role: container-tools-install, tags: container-tools-install} -- hosts: TSG-X-NXR620G40-R01-P0906-mrzcpd-deviceplugin +- hosts: TSG-X-NXR620G40-R01-P0906-packet-io-engine-deviceplugin remote_user: root vars_files: - install_config/group_vars/HAL_TSGXNXR620G40R01P0906.yml diff --git a/ansible/roles/k3s-exporter/tasks/main.yml b/ansible/roles/k3s-exporter/tasks/main.yml index a93e75d0..9533116e 100644 --- a/ansible/roles/k3s-exporter/tasks/main.yml +++ b/ansible/roles/k3s-exporter/tasks/main.yml @@ -73,10 +73,10 @@ url_username: "{{ lookup('env', 'PULP_REPO_USERNAME') }}" url_password: "{{ lookup('env', 'PULP_REPO_PASSWORD') }}" -- name: "copy prometheus crd file to dest" - copy: - src: "{{ role_path }}/files/mrzcpd/exporter-mrzcpd-crd.yaml" - dest: /var/lib/rancher/k3s/server/manifests/exporter-mrzcpd-crd.yaml +# - name: "copy prometheus crd file to dest" +# copy: +# src: "{{ role_path }}/files/mrzcpd/exporter-mrzcpd-crd.yaml" +# dest: /var/lib/rancher/k3s/server/manifests/exporter-mrzcpd-crd.yaml #### mrzcpd exporter end diff --git a/ansible/roles/k3s-install/files/daemonset-mrzcpd-deviceplugin.yaml b/ansible/roles/k3s-install/files/daemonset-mrzcpd-deviceplugin.yaml index 9499b4cf..91def5bf 100644 --- a/ansible/roles/k3s-install/files/daemonset-mrzcpd-deviceplugin.yaml +++ b/ansible/roles/k3s-install/files/daemonset-mrzcpd-deviceplugin.yaml @@ -1,22 +1,22 @@ apiVersion: apps/v1 kind: DaemonSet metadata: - name: packet-io-deviceplugin + name: packet-io-engine-deviceplugin labels: - app: packet-io-deviceplugin + app: packet-io-engine-deviceplugin namespace: tsg-os-system spec: selector: matchLabels: - app: packet-io-deviceplugin + app: packet-io-engine-deviceplugin template: metadata: labels: - app: packet-io-deviceplugin + app: packet-io-engine-deviceplugin spec: containers: - - name: packet-io-deviceplugin - image: registry.gdnt-cloud.website/tsg-mrzcpd-deviceplugin:%%MRZCPD_DEVICE_PLUGIN_IMAGE_TAG%% + - name: packet-io-engine-deviceplugin + image: registry.gdnt-cloud.website/tsg-packet-io-engine-deviceplugin:%%MRZCPD_DEVICE_PLUGIN_IMAGE_TAG%% command: - "bash" - "-ec" @@ -31,13 +31,11 @@ spec: mountPath: /var/lib/kubelet - name: modules mountPath: /lib/modules - - name: mrzcpdtest - mountPath: /opt/tsg/mrzcpd securityContext: privileged: true initContainers: - - name: init-mrzcpd-deviceplugin - image: registry.gdnt-cloud.website/tsg-mrzcpd-deviceplugin:%%MRZCPD_DEVICE_PLUGIN_IMAGE_TAG%% + - name: init-packet-io-engine-deviceplugin + image: registry.gdnt-cloud.website/tsg-packet-io-engine-deviceplugin:%%MRZCPD_DEVICE_PLUGIN_IMAGE_TAG%% command: - "bash" - "-ec" @@ -52,8 +50,6 @@ spec: mountPath: /var/lib/kubelet - name: modules mountPath: /lib/modules - - name: mrzcpdtest - mountPath: /opt/tsg/mrzcpd securityContext: privileged: true volumes: @@ -70,7 +66,4 @@ spec: - name: modules hostPath: path: /lib/modules - - name: mrzcpdtest - hostPath: - path: /root/mrzcpd hostNetwork: true \ No newline at end of file diff --git a/ansible/roles/mrzcpd/files/helm/templates/deployment-mrzcpd.yaml b/ansible/roles/mrzcpd/files/helm/templates/deployment-mrzcpd.yaml index 25235801..07584b78 100644 --- a/ansible/roles/mrzcpd/files/helm/templates/deployment-mrzcpd.yaml +++ b/ansible/roles/mrzcpd/files/helm/templates/deployment-mrzcpd.yaml @@ -1,10 +1,10 @@ apiVersion: apps/v1 kind: Deployment metadata: - name: {{ .Release.Name }} + name: packet-io-engine labels: - app: {{ .Release.Name }} - component: mrzcpd + app: packet-io-engine + component: packet-io-engine annotations: reloader.stakater.com/auto: "true" @@ -12,17 +12,17 @@ spec: replicas: 1 selector: matchLabels: - app: {{ .Release.Name }}-mrzcpd + app: packet-io-engine strategy: type: Recreate template: metadata: labels: - app: {{ .Release.Name }}-mrzcpd + app: packet-io-engine serviceFunction: {{ .Release.Name }} - component: mrzcpd + component: packet-io-engine annotations: - prometheus.io/port: "9086" + prometheus.io/port: "9005" prometheus.io/scrape: "true" spec: @@ -34,8 +34,8 @@ spec: operator: Exists effect: NoSchedule containers: - - name: mrzcpd - image: "registry.gdnt-cloud.website/tsg-mrzcpd:{{ .Chart.AppVersion }}" + - name: packet-io-engine + image: "registry.gdnt-cloud.website/tsg-packet-io-engine:{{ .Chart.AppVersion }}" imagePullPolicy: Never workingDir: /opt/tsg/mrzcpd command: @@ -43,16 +43,26 @@ spec: - "-ec" - | /opt/tsg/mrzcpd/bin/mrmarch --auto - mount --rbind /opt/tsg/mrzcpd /root/hostmount/mrzcpd - cp -f /etc/ld.so.conf.d/mrzcpd.conf /root/hostmount/ld.so.conf.d/mrzcpd.conf - cp -f /etc/profile.d/mrzcpd.sh /root/hostmount/profile.d/mrzcpd.sh + mount --rbind /opt/tsg/mrzcpd /mnt/packet-io-engine/mrzcpd + + cp -f /etc/ld.so.conf.d/mrzcpd.conf /mnt/packet-io-engine/ld.so.conf.d/mrzcpd.conf + cp -f /etc/profile.d/mrzcpd.sh /mnt/packet-io-engine/profile.d/mrzcpd.sh + ldconfig -C /etc/ld.so.cache -r /host + cp -f /opt/tsg/config/mrzcpd.conf /opt/tsg/mrzcpd/etc/mrglobal.conf chmod +x /opt/tsg/mrzcpd/bin/replaceconf.sh /opt/tsg/mrzcpd/bin/replaceconf.sh - ldconfig -C /etc/ld.so.cache -r /host + {{ template "mrzcpd.prestart" . }} {{ template "mrzcpd.start" . }} volumeMounts: + - name: mrzcpd + mountPath: /mnt/packet-io-engine/mrzcpd + mountPropagation: Bidirectional + - name: ldconfig-mrzcpd + mountPath: /mnt/packet-io-engine/ld.so.conf.d + - name: env-sh + mountPath: /mnt/packet-io-engine/profile.d - name: mrzcpd-conf mountPath: "/opt/tsg/config/mrzcpd.conf" subPath: mrzcpd.conf @@ -60,13 +70,6 @@ spec: mountPath: /var/run/mrzcpd - name: hugepages mountPath: /var/run/mrzcpd/hugepages - - name: mrzcpd - mountPath: /root/hostmount/mrzcpd - mountPropagation: Bidirectional - - name: ldconfig-mrzcpd - mountPath: /root/hostmount/ld.so.conf.d - - name: env-sh - mountPath: /root/hostmount/profile.d - name: dpdk mountPath: /var/run/dpdk - name: container-mrzcpd @@ -87,8 +90,6 @@ spec: hugepages-1Gi: {{.Values.mrzcpd.hugepages }} requests: memory: 100Mi - ports: - - containerPort: 9086 env: - name: MLX5_GLUE_PATH value: /opt/tsg/mrzcpd/lib @@ -106,45 +107,57 @@ spec: {{- end }} securityContext: privileged: true - - - name: mrapm-device - image: "registry.gdnt-cloud.website/tsg-mrzcpd:{{ .Chart.AppVersion }}" - imagePullPolicy: Never - workingDir: /opt/tsg/mrzcpd - command: - - "bash" - - "-ec" - - | - /opt/tsg/mrzcpd/bin/monit_device --prometheus-client - volumeMounts: - - name: run-mrzcpd - mountPath: /var/run/mrzcpd - - name: dpdk - mountPath: /var/run/dpdk - - name: root-sys - mountPath: /root/sys - {{ template "mrzcpd.mount.localtime" . }} - {{ template "mrzcpd.mount.prestart" . }} - {{ template "mrzcpd.mount.hostpath" . }} - - name: mrapm-stream - image: "registry.gdnt-cloud.website/tsg-mrzcpd:{{ .Chart.AppVersion }}" + - command: + - /opt/tsg/mrzcpd/bin/monit_device + - --prometheus-client + image: "registry.gdnt-cloud.website/tsg-packet-io-engine:{{ .Chart.AppVersion }}" imagePullPolicy: Never - workingDir: /opt/tsg/mrzcpd - command: - - "bash" - - "-ec" - - | - /opt/tsg/mrzcpd/bin/monit_stream --prometheus-client + name: mrapm-device volumeMounts: - - name: run-mrzcpd - mountPath: /var/run/mrzcpd - - name: dpdk - mountPath: /var/run/dpdk - {{ template "mrzcpd.mount.localtime" . }} - {{ template "mrzcpd.mount.prestart" . }} - {{ template "mrzcpd.mount.hostpath" . }} - + - mountPath: /var/run/mrzcpd + name: run-mrzcpd + readOnly: true + - mountPath: /var/run/dpdk + name: dpdk + readOnly: true + - mountPath: /root/sys + name: root-sys + readOnly: true + - mountPath: /etc/localtime + name: localtime-node + readOnly: true + - command: + - /opt/tsg/mrzcpd/bin/monit_stream + - --prometheus-client + image: "registry.gdnt-cloud.website/tsg-packet-io-engine:{{ .Chart.AppVersion }}" + imagePullPolicy: Never + name: monit-stream + volumeMounts: + - mountPath: /var/run/mrzcpd + name: run-mrzcpd + readOnly: true + - mountPath: /var/run/dpdk + name: dpdk + readOnly: true + - mountPath: /root/sys + name: root-sys + readOnly: true + - mountPath: /etc/localtime + name: localtime-node + readOnly: true + - env: + - name: MERGER_URLS + value: {{ .Values.mergeExporter.mergeUrls }} + - name: MERGER_PORT + value: "{{ .Values.mergeExporter.mergePort }}" + image: "{{ .Values.mergeExporter.image.repository }}:{{ .Values.mergeExporter.image.tag }}" + imagePullPolicy: {{ .Values.mergeExporter.image.pullPolicy }} + name: exporter-merger + ports: + - name: ex-mrzcpd-port + containerPort: {{ .Values.mergeExporter.mergePort }} + protocol: TCP volumes: - name: prestart-dir hostPath: diff --git a/ansible/roles/mrzcpd/files/helm/templates/service.yaml b/ansible/roles/mrzcpd/files/helm/templates/service.yaml new file mode 100644 index 00000000..b986486d --- /dev/null +++ b/ansible/roles/mrzcpd/files/helm/templates/service.yaml @@ -0,0 +1,20 @@ +apiVersion: v1 +kind: Service +metadata: + labels: + app: packet-io-engine + name: packet-io-engine + annotations: + prometheus.io/scrape: "true" + +spec: + ports: + - name: mrzcpd-exporter-port + {{- if .Values.service.nodePort }} + nodePort: {{ .Values.service.nodePort }} + {{- end }} + port: {{ .Values.service.port }} + targetPort: ex-mrzcpd-port + selector: + app: packet-io-engine + type: {{ .Values.service.type }} \ No newline at end of file diff --git a/ansible/roles/mrzcpd/files/helm/values.yaml b/ansible/roles/mrzcpd/files/helm/values.yaml index a0de4b17..53afb837 100644 --- a/ansible/roles/mrzcpd/files/helm/values.yaml +++ b/ansible/roles/mrzcpd/files/helm/values.yaml @@ -79,6 +79,20 @@ tolerations: [] affinity: {} +mergeExporter: + image: + repository: quay.io/rebuy/exporter-merger + pullPolicy: Never + tag: "v0.2.0" + mergePort: "9005" + mergeUrls: "http://0.0.0.0:8901/metrics http://0.0.0.0:8902/metrics" + +service: + type: ClusterIP +# nodePort: "30081" + port: "9005" + annotations: {} + define_enable_val_yes: 'yes' mrzcpd: diff --git a/tools/build_container_images.sh b/tools/build_container_images.sh index 248bef87..36f723b6 100644 --- a/tools/build_container_images.sh +++ b/tools/build_container_images.sh @@ -66,9 +66,9 @@ function export_images() ${IMAGE_REPO_PREFIX}/tsg-sce:${OS_RELEASE_VER} \ ${IMAGE_REPO_PREFIX}/tsg-bfdd:${OS_RELEASE_VER} \ ${IMAGE_REPO_PREFIX}/tsg-inject:${OS_RELEASE_VER} \ - ${IMAGE_REPO_PREFIX}/tsg-mrzcpd:${OS_RELEASE_VER} \ + ${IMAGE_REPO_PREFIX}/tsg-packet-io-engine:${OS_RELEASE_VER} \ ${IMAGE_REPO_PREFIX}/tsg-shaping:${OS_RELEASE_VER} \ - ${IMAGE_REPO_PREFIX}/tsg-mrzcpd-deviceplugin:${OS_RELEASE_VER} + ${IMAGE_REPO_PREFIX}/tsg-packet-io-engine-deviceplugin:${OS_RELEASE_VER} } ##function define end @@ -86,8 +86,8 @@ build_image "sce" "${BASE_IMAGE_NAME}" "${IMAGE_REPO_PREFIX}/tsg-sce:${OS_ build_image "bfdd" "${BASE_IMAGE_NAME}" "${IMAGE_REPO_PREFIX}/tsg-bfdd:${OS_RELEASE_VER}" build_image "shaping" "${BASE_IMAGE_NAME}" "${IMAGE_REPO_PREFIX}/tsg-shaping:${OS_RELEASE_VER}" build_image "inject" "${BASE_IMAGE_NAME}" "${IMAGE_REPO_PREFIX}/tsg-inject:${OS_RELEASE_VER}" -build_image "mrzcpd" "${BASE_IMAGE_NAME}" "${IMAGE_REPO_PREFIX}/tsg-mrzcpd:${OS_RELEASE_VER}" -build_image "mrzcpd-deviceplugin" "${BASE_IMAGE_NAME}" "${IMAGE_REPO_PREFIX}/tsg-mrzcpd-deviceplugin:${OS_RELEASE_VER}" +build_image "packet-io-engine" "${BASE_IMAGE_NAME}" "${IMAGE_REPO_PREFIX}/tsg-packet-io-engine:${OS_RELEASE_VER}" +build_image "packet-io-engine-deviceplugin" "${BASE_IMAGE_NAME}" "${IMAGE_REPO_PREFIX}/tsg-packet-io-engine-deviceplugin:${OS_RELEASE_VER}" export_images