bugfix:TSG-18297:将sn获取流程为从HAL中获取,将sn写入node annotations中,在pod init容器中通过k8s API获取.

2024-01-05 19:05:24 +08:00
parent 71506130fd
commit 0549a5e2b5
7 changed files with 89 additions and 12 deletions
--- a/ansible/roles/traffic-engine/files/helm/templates/_config.tpl
+++ b/ansible/roles/traffic-engine/files/helm/templates/_config.tpl
@@ -163,13 +163,23 @@ enable_breakpad_upload=0
 {{- end -}}


+{{- define "traffic-engine.public.prepare-access-API" -}}
+export APISERVER=https://kubernetes.default.svc
+export SERVICEACCOUNT=/var/run/secrets/kubernetes.io/serviceaccount
+export NAMESPACE=$(cat ${SERVICEACCOUNT}/namespace)
+export TOKEN=$(cat ${SERVICEACCOUNT}/token)
+export CACERT=${SERVICEACCOUNT}/ca.crt
+{{- end -}}
+
+
 {{- define "traffic-engine.init" -}}
-            mkdir -p /target_config/opt/tsg/etc
-            mkdir -p /target_config/etc/profile.d
-            /opt/tsg/scripts/read_sn_to_env.sh
-            source /etc/profile.d/sn.sh
-            echo "{\"sn\": \"$SN\"}" > /target_config/opt/tsg/etc/tsg_sn.json
-            echo "export device_id=${SN}" > /target_config/etc/profile.d/device_id.sh
+{{- include "traffic-engine.public.prepare-access-API" . }}
+curl --cacert ${CACERT} --header "Authorization: Bearer ${TOKEN}" -X GET ${APISERVER}/api/v1/nodes/${NODE_NAME} -o /tmp/node-${NODE_NAME}.json
+export DEVICE_SN=$(cat /tmp/node-${NODE_NAME}.json | jq -r '.metadata.annotations."tsg-os/device-sn"')
+mkdir -p /target_config/opt/tsg/etc
+mkdir -p /target_config/etc/profile.d
+echo "{\"sn\": \"$DEVICE_SN\"}" > /target_config/opt/tsg/etc/tsg_sn.json
+echo "export device_id=${DEVICE_SN}" > /target_config/etc/profile.d/device_id.sh
 {{- end -}}

 {{- define "traffic-engine.init.sce" -}}
--- a/ansible/roles/traffic-engine/files/helm/templates/clusterrole.yaml
+++ b/ansible/roles/traffic-engine/files/helm/templates/clusterrole.yaml
@@ -7,5 +7,5 @@ metadata:
  name: {{ .Release.Name }}
 rules:
  - apiGroups: [""]
-    resources: ["services"]
+    resources: ["services", "nodes"]
    verbs: ["get", "list", "watch"]
--- a/ansible/roles/traffic-engine/files/helm/templates/deployment-firewall.yaml
+++ b/ansible/roles/traffic-engine/files/helm/templates/deployment-firewall.yaml
@@ -162,7 +162,7 @@ spec:
          - "bash"
          - "-ec"
          - |
-            {{ template "traffic-engine.init" . }}
+            {{- include "traffic-engine.init" . | nindent 12 }}
        securityContext:
          privileged: true
        env:
@@ -170,6 +170,10 @@ spec:
          valueFrom:
            fieldRef:
              fieldPath: status.hostIP
+        - name: NODE_NAME
+          valueFrom:
+            fieldRef:
+              fieldPath: spec.nodeName
        volumeMounts:
        - name: config-volume
          mountPath: /target_config 
--- a/ansible/roles/traffic-engine/files/helm/templates/deployment-proxy.yaml
+++ b/ansible/roles/traffic-engine/files/helm/templates/deployment-proxy.yaml
@@ -212,8 +212,7 @@ spec:
            sysctl -w net.ipv4.tcp_window_scaling=0


-
-            {{ template "traffic-engine.init" . }}
+            {{ include "traffic-engine.init" . | nindent 12 }}
            ip tuntap add dev tap0 mode tap multi_queue
            /usr/sbin/ip link set tap0 address fe:65:b7:03:50:bd
            /usr/sbin/ip link set tap0 up
@@ -284,6 +283,10 @@ spec:
          valueFrom:
            fieldRef:
              fieldPath: status.hostIP
+        - name: NODE_NAME
+          valueFrom:
+            fieldRef:
+              fieldPath: spec.nodeName
        volumeMounts:
        - name: config-volume
          mountPath: /target_config 
--- a/ansible/roles/traffic-engine/files/helm/templates/deployment-service-chaining.yaml
+++ b/ansible/roles/traffic-engine/files/helm/templates/deployment-service-chaining.yaml
@@ -175,7 +175,7 @@ spec:
          - "bash"
          - "-ec"
          - |
-            {{ template "traffic-engine.init" . }}
+            {{- include "traffic-engine.init" . | nindent 12 }}
            {{ template "traffic-engine.init.sce" . }}
        securityContext:
          privileged: true
@@ -184,6 +184,10 @@ spec:
          valueFrom:
            fieldRef:
              fieldPath: status.hostIP
+        - name: NODE_NAME
+          valueFrom:
+            fieldRef:
+              fieldPath: spec.nodeName
        volumeMounts:
        - name: config-volume
          mountPath: /target_config 
--- a/ansible/roles/traffic-engine/files/helm/templates/deployment-shaping.yaml
+++ b/ansible/roles/traffic-engine/files/helm/templates/deployment-shaping.yaml
@@ -146,7 +146,7 @@ spec:
          - "bash"
          - "-ec"
          - |
-            {{ template "traffic-engine.init" . }}
+            {{- include "traffic-engine.init" . | nindent 12 }}
        securityContext:
          privileged: true
        env:
@@ -154,6 +154,11 @@ spec:
          valueFrom:
            fieldRef:
              fieldPath: status.hostIP
+        - name: NODE_NAME
+          valueFrom:
+            fieldRef:
+              fieldPath: spec.nodeName
+      
        volumeMounts:
        - name: config-volume
          mountPath: /target_config 
--- a/ansible/roles/tsg-os-HAL/files/script/tsg-os-HAL.sh
+++ b/ansible/roles/tsg-os-HAL/files/script/tsg-os-HAL.sh
@@ -29,6 +29,8 @@ TRAFFIC_ENGINE_LOGS_VDISK_PATH="/data/vdisks"
 TRAFFIC_ENGINE_LOGS_VDISK_SIZE_BYTES=
 NF_COUNT=16
 NIC_CPU_Affinity_Switch=
+DEVICE_TYPE=
+SN=

 cpu_format_conversion()
 {
@@ -273,12 +275,56 @@ isolated_cores=${IOCORE}
 EOF
 }

+read_device_type()
+{
+    product_name=`ipmitool fru list | grep 'Product Name' | awk '{print $4}' | head -n 1`
+
+    case ${product_name} in
+        "ACB300-040-00" | "9000-NPB-P01R01")
+            DEVICE_TYPE="9000-NPB"
+            ;;
+        *)
+        ;;
+    esac
+}
+
+read_sn()
+{
+    if [ ${DEVICE_TYPE} == "9000-NPB" ]; then
+        SN=`ipmitool fru -t 130 | awk 'NR==12' | awk '{ print $4}'`
+    else
+        SN=`ipmitool fru list | grep 'Product Serial' | awk '{ print $4}' | head -n 1`
+    fi
+
+    if [ -z "${SN}" ]; then
+        SN="unknown"
+    fi
+}
+
+add_node_metadata()
+{
+    local node_name="localhost"
+    local tsg_os_manifest="/var/lib/rancher/k3s/server/manifests/tsg-os"
+    local node_manifest="${tsg_os_manifest}/add-node-metadata.yaml"
+
+    mkdir -p ${tsg_os_manifest}
+    cat > ${node_manifest} << EOF
+apiVersion: v1
+kind: Node
+metadata:
+  name: ${node_name}
+  annotations:
+    tsg-os/device-sn: "${SN}"
+EOF
+}
+
 load_tsg_os_HAL_config
 mkdir -p /run/mrzcpd/hugepages
 mkdir -p /var/run/share

 build_and_mount_traffic_engine_logs_vdisk
 set_hugepages
+
 set_cpu_list
 set_grub_cmdline

@@ -295,3 +341,8 @@ set_mrzcpd_direct_pktmbuf
 set_k3s_config ${mem_num}
 set_tsg_clixon_conf
 set_mrzcpd_conf
+
+# add sn to k8s node annotations.
+read_device_type
+read_sn
+add_node_metadata