✨ feat更新Clixon到1.0.42

TSG-15468: 支持根据NUMA节点下CPU的个数，动态分配CPU给MRZCPD TSG-15467: Clixon支持session_distribution_policy命令 TSG-15466: Clixon支持traffic_mirror命令
2023-06-08 19:12:57 +08:00
parent 5b1c5576f1
commit e00c2c5f00
7 changed files with 137 additions and 41 deletions
--- a/ansible/install_config/group_vars/rpm_version.yml
+++ b/ansible/install_config/group_vars/rpm_version.yml
@@ -100,7 +100,7 @@ hasp_update_rpm_version:
 tsg_os_clixon_rpm_version:
  cligen: cligen-5.8.0-release
  clixon: clixon-5.8.0-release
-  tsg-os-mgnt-srv: tsg-os-mgnt-srv-1.0.41.7b86d6b
+  tsg-os-mgnt-srv: tsg-os-mgnt-srv-1.0.42.b5d7a44

 sce_rpm_version:
  sce: sce-1.0.13.23d05e7
--- a/ansible/roles/traffic-engine/files/helm/conf/main.conf
+++ b/ansible/roles/traffic-engine/files/helm/conf/main.conf
@@ -136,7 +136,9 @@ TRAFFIC_MIRROR_ENABLE=1
 {{- else }}
 TRAFFIC_MIRROR_ENABLE=0
 {{- end }}
-NIC_NAME="{{ .Values.nic_mirror_name }}"
+{{- if .Values.nic_mirror_name.firewall }}
+NIC_NAME="{{ .Values.nic_mirror_name.firewall }}"
+{{- end }}
 DEFAULT_VLAN_ID=0
 [RADIUS_PLUG]
 DEVICE_TAGS={"tags":[{{- include "traffic-engine.device-tag-list" . }}]}
--- a/ansible/roles/traffic-engine/files/helm/conf/tfe.conf
+++ b/ansible/roles/traffic-engine/files/helm/conf/tfe.conf
@@ -163,7 +163,9 @@ enable=1
 {{- else }}
 enable=0
 {{- end }}
-device={{ .Values.nic_mirror_name }}
+{{- if .Values.nic_mirror_name.proxy }}
+device={{ .Values.nic_mirror_name.proxy }}
+{{- end }}
 # 0:TRAFFIC_MIRROR_ETHDEV_AF_PACKET; 1:TRAFFIC_MIRROR_ETHDEV_MARSIO
 type=1

@@ -239,7 +241,7 @@ bpf_obj=/opt/tsg/tfe/resource/bpf/bpf_tun_rss_steering.o
 bpf_debug_log=0
 # 2: BPF 使用二元组分流
 # 4: BPF 使用四元组分流
-bpf_hash_mode=4
+bpf_hash_mode={{ .Values.distmode }}

 # 配置 tap 网卡的 RPS
 tap_rps_enable=1
--- a/ansible/roles/traffic-engine/files/helm/values.yaml
+++ b/ansible/roles/traffic-engine/files/helm/values.yaml
@@ -126,7 +126,9 @@ tfe_rps_mask: "00000000"

 nic_policy_log_name: eth0
 nic_raw_name: eth0
-nic_mirror_name: eth0
+nic_mirror_name: 
+  firewall: eth0
+  proxy: eth0

 define_enable_val_yes: yes
 define_enable_val_no: no
@@ -188,6 +190,7 @@ shaping_config:
  shaping_nic: nf_1_shaping_engine

 app_symbol_index: 1
+distmode: 2

 debug:
  firewall:
--- a/ansible/roles/tsg-os-HAL/files/conf/tsg-os-HAL.conf
+++ b/ansible/roles/tsg-os-HAL/files/conf/tsg-os-HAL.conf
@@ -0,0 +1,10 @@
+
+# HUGEPAGES=
+
+# 需要同时指定才能生效
+# NUMA_NODE_CNT=
+# MRZCPD_IOCORE=
+# CLIXON_IOCORE=
+
+# MRZCPD_DIRECT_PKTMBUF=
+# MRZCPD_INDIRECT_PKTMBUF=
--- a/ansible/roles/tsg-os-HAL/files/script/tsg-os-HAL.sh
+++ b/ansible/roles/tsg-os-HAL/files/script/tsg-os-HAL.sh
@@ -1,8 +1,8 @@
 #!/bin/sh

-set -xe
+set -x

-tsg_os_HAL_cfg_file="/etc/sysconfig/tsg-os-HAL"
+tsg_os_HAL_cfg_file="/etc/sysconfig/tsg-os-HAL.conf"

 grub_cfg_file="/boot/grub/grub.cfg"
 mrzcpd_cfg_file="/etc/sysconfig/mrzcpd"
@@ -11,46 +11,20 @@ k3s_cfg_file="/etc/rancher/k3s/config.yaml"
 mrglobal_cfg_template="/opt/tsg/clixon/etc/mrglobal.conf.jinja"

 cpu_core_num=$(lscpu | grep "CPU(s):" | head -n 1 | sed  -r 's/CPU\(s\):\s{1,}//g')
+numa_nodes=$(numactl --hardware | grep available | awk -F"[()]" '{print $2}')
 mem_num=$(free -m | grep Mem | awk '{print $2}')

 HUGEPAGES=
+NUMA_NODE_CNT=
 MRZCPD_IOCORE=
+CLIXON_IOCORE=
 MRZCPD_DIRECT_PKTMBUF=
-TSG_CLIXON_CPU_QUOTAS_MIN=
+MRZCPD_INDIRECT_PKTMBUF=

 load_tsg_os_HAL_config()
 {
-    [ -f ${tsg_os_HAL_cfg_file} ] || return 1
-    
-    source ${tsg_os_HAL_cfg_file}
-    return 0
-}
-
-set_resouce_by_HAL()
-{
-    local mem_num="$1"
-    local cpu_core_num="$2"
-
-    if [ $mem_num -le "32768" ]; then
-        HUGEPAGES=4
-        MRZCPD_DIRECT_PKTMBUF=262144
-    elif [ $mem_num -le "65536" ]; then
-        HUGEPAGES=8
-        MRZCPD_DIRECT_PKTMBUF=1048576
-    else
-        HUGEPAGES=16
-        MRZCPD_DIRECT_PKTMBUF=2097152
-    fi
-
-    if [ ${cpu_core_num} -le "4" ]; then
-        MRZCPD_IOCORE="1"
-        TSG_CLIXON_CPU_QUOTAS_MIN=2
-    elif [ ${cpu_core_num} -le "16" ]; then
-        MRZCPD_IOCORE="1,2"
-        TSG_CLIXON_CPU_QUOTAS_MIN=4
-    else
-        MRZCPD_IOCORE="1,2,3,4"
-        TSG_CLIXON_CPU_QUOTAS_MIN=9
+    if [ -f "${tsg_os_HAL_cfg_file}" ]; then
+        source ${tsg_os_HAL_cfg_file}
    fi
 }

@@ -83,10 +57,11 @@ EOF
 set_tsg_clixon_conf()
 {
    [ -f ${mrglobal_cfg_template} ] && sed -i "s/^sz_direct_pktmbuf=.*$/sz_direct_pktmbuf=${MRZCPD_DIRECT_PKTMBUF}/g" ${mrglobal_cfg_template}
+    [ -f ${mrglobal_cfg_template} ] && sed -i "s/^sz_indirect_pktmbuf=.*$/sz_indirect_pktmbuf=${MRZCPD_INDIRECT_PKTMBUF}/g" ${mrglobal_cfg_template}

    [ -f ${mrglobal_cfg_template} ] && sed -i "s/^iocore=.*$/iocore=${MRZCPD_IOCORE}/g" ${mrglobal_cfg_template}

-    [ -f ${tsg_clixon_cfg_file} ] && sed -i "s/^cpu_range=.*$/cpu_range=${TSG_CLIXON_CPU_QUOTAS_MIN}/g" ${tsg_clixon_cfg_file}
+    [ -f ${tsg_clixon_cfg_file} ] && sed -i "s/^cpu_range=.*$/cpu_range=${CLIXON_IOCORE}/g" ${tsg_clixon_cfg_file}
 }

 set_os_hugepages()
@@ -97,6 +72,90 @@ set_os_hugepages()
    return 0
 }

+set_hugepages()
+{
+    if [ ! -n "$HUGEPAGES" ]; then
+        if [ $mem_num -le "32768" ]; then
+            HUGEPAGES=4
+        elif [ $mem_num -le "65536" ]; then
+            HUGEPAGES=8
+        else
+            HUGEPAGES=16
+        fi
+    fi
+}
+
+set_mrzcpd_indirect_pktmbuf()
+{
+    if [ ! -n "$MRZCPD_INDIRECT_PKTMBUF" ]; then
+        MRZCPD_INDIRECT_PKTMBUF=`expr 8192 / $NUMA_NODE_CNT`
+    fi
+}
+
+set_mrzcpd_direct_pktmbuf()
+{
+    if [ ! -n "$MRZCPD_DIRECT_PKTMBUF" ]; then
+        if [ $mem_num -le "32768" ]; then
+            MRZCPD_DIRECT_PKTMBUF=`expr 262144 / $NUMA_NODE_CNT`
+        elif [ $mem_num -le "65536" ]; then
+            MRZCPD_DIRECT_PKTMBUF=`expr 1048576 / $NUMA_NODE_CNT`
+        else
+            MRZCPD_DIRECT_PKTMBUF=`expr 2097152 / $NUMA_NODE_CNT`
+        fi
+    fi
+}
+
+allocate_cpu()
+{
+    local numa_node_min=0
+    local numa_node_max=0
+    local numa_node_cpu_cnt=0
+    local numa_cpu_cnt_min=0
+    local numa_list
+
+    if [ -n "$MRZCPD_IOCORE" ] && [ -n "$CLIXON_IOCORE" ] && [ -n "$NUMA_NODE_CNT" ]; then
+        return 0
+    fi
+
+    if [[ $numa_nodes == *-* ]]
+    then
+        read numa_node_min numa_node_max <<< $(echo $numa_nodes | awk -F- '{print $1" "$2}')
+    else
+        numa_node_min=$numa_nodes
+        numa_node_max=$numa_nodes
+    fi
+    NUMA_NODE_CNT=`expr $numa_node_max - $numa_node_min + 1`
+    for((i=$numa_node_min;i<=$numa_node_max;i++));
+    do
+        numa_node_cpu_cnt=$(numactl --hardware | awk '/^node ['$i']+ cpus:/ {print NF-3}')
+        if [ $numa_cpu_cnt_min -eq 0 ] || [ $numa_node_cpu_cnt -lt $numa_cpu_cnt_min ]; then
+            numa_cpu_cnt_min=$numa_node_cpu_cnt
+        fi
+    done
+
+    if [ $numa_cpu_cnt_min -le 4 ]; then
+        mrzcpd_cnt=1
+        clixon_min_index=2
+    elif [ $numa_cpu_cnt_min -le 16 ]; then
+        mrzcpd_cnt=2
+        clixon_min_index=3
+    else
+        mrzcpd_cnt=4
+        clixon_min_index=5
+    fi
+
+    for((i=$numa_node_min;i<=$numa_node_max;i++));
+    do
+        numa_list=$(numactl --hardware | grep "node $i cpus: " | sed -r "s/node $i cpus: //g" | sed -r "s/^0 //g")
+        mrzcpd_core=$(echo $numa_list | cut -d ' ' -f 1-$mrzcpd_cnt | tr ' ' ',')
+        MRZCPD_IOCORE=$MRZCPD_IOCORE","$mrzcpd_core
+        clixon_core=$(echo $numa_list | cut -d ' ' -f $clixon_min_index- | tr ' ' ',')
+        CLIXON_IOCORE=$CLIXON_IOCORE","$clixon_core
+    done
+    MRZCPD_IOCORE=${MRZCPD_IOCORE#*,}
+    CLIXON_IOCORE=${CLIXON_IOCORE#*,}
+}
+
 set_bridge()
 {
    ip link add br_dign_c type bridge
@@ -105,7 +164,12 @@ set_bridge()
    ip link set br_dign_s up
 }

-load_tsg_os_HAL_config || set_resouce_by_HAL ${mem_num} ${cpu_core_num}
+load_tsg_os_HAL_config
+
+allocate_cpu
+set_hugepages
+set_mrzcpd_indirect_pktmbuf
+set_mrzcpd_direct_pktmbuf

 set_k3s_config ${mem_num}
 set_tsg_clixon_conf
--- a/ansible/roles/tsg-os-HAL/tasks/main.yml
+++ b/ansible/roles/tsg-os-HAL/tasks/main.yml
@@ -7,6 +7,12 @@
    - tsg-os-HAL.sh
  when: runtime_env == 'TSG-X-P0906'

+- name: "tsg-os-HAL: install numactl"
+  yum:
+    name: numactl
+    conf_file: "{{ rpm_repo_config_path }}"
+    state: present
+
 - name: "install tsg-os-HAL.service -- P0906"
  copy:
    src: "{{ role_path }}/files/service/{{ item.src }}"
@@ -23,3 +29,12 @@
  with_items:
    - tsg-os-HAL.service
  when: runtime_env == 'TSG-X-P0906'
+
+- name: "tsg-os-HAL.conf"
+  copy:
+    src: "{{ role_path }}/files/conf/{{ item }}"
+    dest: /etc/sysconfig/
+    mode: 0755
+  with_items:
+    - tsg-os-HAL.conf
+  when: runtime_env == 'TSG-X-P0906'