HAL support nf_count and NIC_CPU_Affinity_Switch config

2023-11-24 17:11:54 +08:00
parent bba9c5680e
commit 41e235aff3
4 changed files with 200 additions and 102 deletions
--- a/ansible/roles/tsg-os-HAL/files/conf/tsg-os-HAL.conf
+++ b/ansible/roles/tsg-os-HAL/files/conf/tsg-os-HAL.conf
@@ -4,8 +4,10 @@
 # SYSTEM_RESERVED=

 # NUMA_NODE_CNT=
-# MRZCPD_IOCORE=
-# CLIXON_IOCORE=
+# IOCORE=
+# WORKLOAD_CORE=
+
+# NF_COUNT=

 # MRZCPD_DIRECT_PKTMBUF=
 # MRZCPD_INDIRECT_PKTMBUF=
@@ -13,3 +15,5 @@
 # MRZCPD_SZ_DATA=
 # MRZCPD_SZ_TUNNEL=
 # MRZCPD_CHECK_BUFFER_LEAK=
+
+# NIC_CPU_Affinity_Switch=
--- a/ansible/roles/tsg-os-HAL/files/script/cpu_amd.sh
+++ b/ansible/roles/tsg-os-HAL/files/script/cpu_amd.sh
@@ -1,8 +1,16 @@
 #!/bin/sh

-set -x
+write_devices_disable()
+{
+(cat <<EOF1
+[device]
+rxcore_enable=0

-write_devices_conf()
+EOF1
+) >> $1
+}
+
+write_devices_enable()
 {
 (cat <<EOF1
 [device]
@@ -38,74 +46,110 @@ set_mrzcpd_rx_core()
    devices=$(lshw -c network -businfo | grep network | awk '{print $2}')
    for device in $devices
    do
+        if [ $device == "network" ]; then
+            continue
+        fi
        businfo=$(ethtool -i $device | grep "bus-info:"| awk '{print $2}')
        numa_node=$(lspci -s $businfo -vv | grep "NUMA node:" | awk '{print $3}')
+
+        if [ ! -n "$numa_node" ] || [ $numa_node -ge $NUMA_NODE_CNT ]; then
+            continue
+        fi
        write_device_rxcore_conf $device ${mrzcpd_rx_core[$numa_node]} ${tsg_clixon_cfg_file}
        str_device=$str_device","$device
    done
    str_device=${str_device#*,}
-    write_devices_conf $str_device ${tsg_clixon_cfg_file}
+
+    if [ ! -n "$str_device" ]; then
+        write_devices_disable ${tsg_clixon_cfg_file}
+    else
+        write_devices_enable $str_device ${tsg_clixon_cfg_file}
+    fi
+}
+
+output_common_str()
+{
+    local common_list
+
+    list1=$(echo $1 | tr ',' ' ')
+    list2=$(echo $2 | tr ',' ' ')
+
+    for numa_core in $list1;
+    do
+        for iocore in $list2;
+        do
+            if [ $numa_core -eq $iocore ]; then
+                common_list=$common_list","$numa_core
+                break
+            fi
+        done
+    done
+    common_list=${common_list#*,}
+    echo $common_list
 }

 allocate_cpu()
 {
-    local numa_node_min=0
-    local numa_node_max=0
    local numa_node_cpu_cnt=0
    local numa_cpu_cnt_min=0
    local numa_list
-    local numa_nodes
    local mrzcpd_start
    local mrzcpd_end
    local clixon_start

-    numa_nodes=$(numactl --hardware | grep available | awk -F"[()]" '{print $2}')
-
-    if [[ $numa_nodes == *-* ]]
-    then
-        read numa_node_min numa_node_max <<< $(echo $numa_nodes | awk -F- '{print $1" "$2}')
-    else
-        numa_node_min=$numa_nodes
-        numa_node_max=$numa_nodes
+    if [ ! -n "$NUMA_NODE_CNT" ]; then
+        NUMA_NODE_CNT=`lscpu | grep "NUMA node(s):" | head -n 1 | sed  -r 's/NUMA node\(s\):\s{1,}//g'`
    fi
-    NUMA_NODE_CNT=`expr $numa_node_max - $numa_node_min + 1`
-    for((i=$numa_node_min;i<=$numa_node_max;i++));
-    do
-        numa_node_cpu_cnt=$(numactl --hardware | awk '/^node ['$i']+ cpus:/ {print NF-3}')
-        if [ $numa_cpu_cnt_min -eq 0 ] || [ $numa_node_cpu_cnt -lt $numa_cpu_cnt_min ]; then
-            numa_cpu_cnt_min=$numa_node_cpu_cnt
+
+    if [ -n "$IOCORE" ] && [ -n "$WORKLOAD_CORE" ]; then
+        for((i=0;i<$NUMA_NODE_CNT;i++));
+        do
+            numa_list=$(numactl --hardware | grep "node $i cpus: " | sed -r "s/node $i cpus: //g" | tr ' ' ',')
+            mrzcpd_core=$(output_common_str $numa_list $IOCORE)
+            mrzcpd_rx_core+=($mrzcpd_core)
+        done
+    else
+        unset IOCORE
+        unset WORKLOAD_CORE
+
+        for((i=0;i<$NUMA_NODE_CNT;i++));
+        do
+            numa_node_cpu_cnt=$(numactl --hardware | awk '/^node ['$i']+ cpus:/ {print NF-3}')
+            if [ $numa_cpu_cnt_min -eq 0 ] || [ $numa_node_cpu_cnt -lt $numa_cpu_cnt_min ]; then
+                numa_cpu_cnt_min=$numa_node_cpu_cnt
+            fi
+        done
+
+        if [ $numa_cpu_cnt_min -le 4 ]; then
+            mrzcpd_start=2
+            mrzcpd_end=2
+            clixon_start=3
+        elif [ $numa_cpu_cnt_min -le 16 ]; then
+            mrzcpd_start=3
+            mrzcpd_end=4
+            clixon_start=5
+        elif [ $numa_cpu_cnt_min -le 32 ]; then
+            mrzcpd_start=5
+            mrzcpd_end=8
+            clixon_start=9
+        else
+            mrzcpd_start=5
+            mrzcpd_end=12
+            clixon_start=13
        fi
-    done

-    if [ $numa_cpu_cnt_min -le 4 ]; then
-        mrzcpd_start=2
-        mrzcpd_end=2
-        clixon_start=3
-    elif [ $numa_cpu_cnt_min -le 16 ]; then
-        mrzcpd_start=3
-        mrzcpd_end=4
-        clixon_start=5
-    elif [ $numa_cpu_cnt_min -le 32 ]; then
-        mrzcpd_start=5
-        mrzcpd_end=8
-        clixon_start=9
-    else
-        mrzcpd_start=5
-        mrzcpd_end=12
-        clixon_start=13
+        for((i=0;i<$NUMA_NODE_CNT;i++));
+        do
+            numa_list=$(numactl --hardware | grep "node $i cpus: " | sed -r "s/node $i cpus: //g")
+            mrzcpd_core=$(echo $numa_list | cut -d ' ' -f $mrzcpd_start-$mrzcpd_end | tr ' ' ',')
+            IOCORE=$IOCORE","$mrzcpd_core
+            mrzcpd_rx_core+=($mrzcpd_core)
+            clixon_core=$(echo $numa_list | cut -d ' ' -f $clixon_start- | tr ' ' ',')
+            WORKLOAD_CORE=$WORKLOAD_CORE","$clixon_core
+        done
+        IOCORE=${IOCORE#*,}
+        WORKLOAD_CORE=${WORKLOAD_CORE#*,}
    fi

-    for((i=$numa_node_min;i<=$numa_node_max;i++));
-    do
-        numa_list=$(numactl --hardware | grep "node $i cpus: " | sed -r "s/node $i cpus: //g")
-        mrzcpd_core=$(echo $numa_list | cut -d ' ' -f $mrzcpd_start-$mrzcpd_end | tr ' ' ',')
-        MRZCPD_IOCORE=$MRZCPD_IOCORE","$mrzcpd_core
-        mrzcpd_rx_core+=($mrzcpd_core)
-        clixon_core=$(echo $numa_list | cut -d ' ' -f $clixon_start- | tr ' ' ',')
-        CLIXON_IOCORE=$CLIXON_IOCORE","$clixon_core
-    done
-    MRZCPD_IOCORE=${MRZCPD_IOCORE#*,}
-    CLIXON_IOCORE=${CLIXON_IOCORE#*,}
-
    set_mrzcpd_rx_core
 }
--- a/ansible/roles/tsg-os-HAL/files/script/cpu_default.sh
+++ b/ansible/roles/tsg-os-HAL/files/script/cpu_default.sh
@@ -1,56 +1,66 @@
 #!/bin/sh

-set -x
+write_devices_conf()
+{
+(cat <<EOF1
+[device]
+rxcore_enable=0
+
+EOF1
+) >> ${tsg_clixon_cfg_file}
+}
+
+del_device_rxcore_conf()
+{
+    sed -i '/^\[device.*\]/,/^$/d' ${tsg_clixon_cfg_file}
+}

 allocate_cpu()
 {
-    local numa_node_min=0
-    local numa_node_max=0
    local numa_node_cpu_cnt=0
    local numa_cpu_cnt_min=0
    local numa_list
-    local numa_nodes
    local mrzcpd_cnt
    local clixon_min_index

-    numa_nodes=$(numactl --hardware | grep available | awk -F"[()]" '{print $2}')
-
-    if [[ $numa_nodes == *-* ]]
-    then
-        read numa_node_min numa_node_max <<< $(echo $numa_nodes | awk -F- '{print $1" "$2}')
-    else
-        numa_node_min=$numa_nodes
-        numa_node_max=$numa_nodes
+    if [ ! -n "$NUMA_NODE_CNT" ]; then
+        NUMA_NODE_CNT=`lscpu | grep "NUMA node(s):" | head -n 1 | sed  -r 's/NUMA node\(s\):\s{1,}//g'`
    fi
-    NUMA_NODE_CNT=`expr $numa_node_max - $numa_node_min + 1`
-    for((i=$numa_node_min;i<=$numa_node_max;i++));
-    do
-        numa_node_cpu_cnt=$(numactl --hardware | awk '/^node ['$i']+ cpus:/ {print NF-3}')
-        if [ $numa_cpu_cnt_min -eq 0 ] || [ $numa_node_cpu_cnt -lt $numa_cpu_cnt_min ]; then
-            numa_cpu_cnt_min=$numa_node_cpu_cnt
+
+    if [ ! -n "$IOCORE" ] || [ ! -n "$WORKLOAD_CORE" ]; then
+        unset IOCORE
+        unset WORKLOAD_CORE
+
+        for((i=0;i<$NUMA_NODE_CNT;i++));
+        do
+            numa_node_cpu_cnt=$(numactl --hardware | awk '/^node ['$i']+ cpus:/ {print NF-3}')
+            if [ $numa_cpu_cnt_min -eq 0 ] || [ $numa_node_cpu_cnt -lt $numa_cpu_cnt_min ]; then
+                numa_cpu_cnt_min=$numa_node_cpu_cnt
+            fi
+        done
+
+        if [ $numa_cpu_cnt_min -le 4 ]; then
+            mrzcpd_cnt=1
+            clixon_min_index=2
+        elif [ $numa_cpu_cnt_min -le 16 ]; then
+            mrzcpd_cnt=2
+            clixon_min_index=3
+        else
+            mrzcpd_cnt=4
+            clixon_min_index=5
        fi
-    done

-    if [ $numa_cpu_cnt_min -le 4 ]; then
-        mrzcpd_cnt=1
-        clixon_min_index=2
-    elif [ $numa_cpu_cnt_min -le 16 ]; then
-        mrzcpd_cnt=2
-        clixon_min_index=3
-    else
-        mrzcpd_cnt=4
-        clixon_min_index=5
+        for((i=0;i<$NUMA_NODE_CNT;i++));
+        do
+            numa_list=$(numactl --hardware | grep "node $i cpus: " | sed -r "s/node $i cpus: //g" | sed -r "s/^0 //g")
+            mrzcpd_core=$(echo $numa_list | cut -d ' ' -f 1-$mrzcpd_cnt | tr ' ' ',')
+            IOCORE=$IOCORE","$mrzcpd_core
+            clixon_core=$(echo $numa_list | cut -d ' ' -f $clixon_min_index- | tr ' ' ',')
+            WORKLOAD_CORE=$WORKLOAD_CORE","$clixon_core
+        done
+        IOCORE=${IOCORE#*,}
+        WORKLOAD_CORE=${WORKLOAD_CORE#*,}
    fi
-
-
-    for((i=$numa_node_min;i<=$numa_node_max;i++));
-    do
-        numa_list=$(numactl --hardware | grep "node $i cpus: " | sed -r "s/node $i cpus: //g" | sed -r "s/^0 //g")
-        mrzcpd_core=$(echo $numa_list | cut -d ' ' -f 1-$mrzcpd_cnt | tr ' ' ',')
-        MRZCPD_IOCORE=$MRZCPD_IOCORE","$mrzcpd_core
-        clixon_core=$(echo $numa_list | cut -d ' ' -f $clixon_min_index- | tr ' ' ',')
-        CLIXON_IOCORE=$CLIXON_IOCORE","$clixon_core
-    done
-    MRZCPD_IOCORE=${MRZCPD_IOCORE#*,}
-    CLIXON_IOCORE=${CLIXON_IOCORE#*,}
+    del_device_rxcore_conf
+    write_devices_conf
 }
--- a/ansible/roles/tsg-os-HAL/files/script/tsg-os-HAL.sh
+++ b/ansible/roles/tsg-os-HAL/files/script/tsg-os-HAL.sh
@@ -17,8 +17,8 @@ HUGEPAGES=
 KUBE_RESERVED=
 SYSTEM_RESERVED=
 NUMA_NODE_CNT=
-MRZCPD_IOCORE=
-CLIXON_IOCORE=
+IOCORE=
+WORKLOAD_CORE=
 MRZCPD_DIRECT_PKTMBUF=
 MRZCPD_INDIRECT_PKTMBUF=
 MRZCPD_POLL_WAIT_THROTTLE=512
@@ -27,11 +27,50 @@ MRZCPD_SZ_TUNNEL=8192
 MRZCPD_CHECK_BUFFER_LEAK=1
 TRAFFIC_ENGINE_LOGS_VDISK_PATH="/data/vdisks"
 TRAFFIC_ENGINE_LOGS_VDISK_SIZE_BYTES=
+NF_COUNT=16
+NIC_CPU_Affinity_Switch=
+
+cpu_format_conversion()
+{
+    local core_min
+    local core_max
+    local core_list
+
+    if [[ $1 == *-* ]]; then
+        read core_min core_max <<< $(echo $1 | awk -F- '{print $1" "$2}')
+        for((i=$core_min;i<=$core_max;i++));
+        do
+            core_list=$core_list","$i
+        done
+        core_list=${core_list#*,}
+    else
+        core_list=$1
+    fi
+    echo $core_list
+}

 load_tsg_os_HAL_config()
 {
    if [ -f "${tsg_os_HAL_cfg_file}" ]; then
        source ${tsg_os_HAL_cfg_file}
+
+        if [ ! -n "$IOCORE" ] && [ -n "$MRZCPD_IOCORE" ]; then
+            IOCORE=$MRZCPD_IOCORE
+        fi
+
+        if [ ! -n "$WORKLOAD_CORE" ] && [ -n "$CLIXON_IOCORE" ]; then
+            WORKLOAD_CORE=$CLIXON_IOCORE
+        fi
+
+        if [ -n $IOCORE ]; then
+            core_list=$(echo $IOCORE | tr ',' ' ')
+            for core in $core_list;
+            do
+                core_format=$core_format","$(cpu_format_conversion $core)
+            done
+            core_format=${core_format#*,}
+            IOCORE=$core_format
+        fi
    fi
 }

@@ -66,15 +105,16 @@ EOF

 set_tsg_clixon_conf()
 {
-    [ -f ${tsg_clixon_cfg_file} ] && sed -i "s/^cpu_range=.*$/cpu_range=${CLIXON_IOCORE}/g" ${tsg_clixon_cfg_file}
+    [ -f ${tsg_clixon_cfg_file} ] && sed -i "s/^cpu_range=.*$/cpu_range=${WORKLOAD_CORE}/g" ${tsg_clixon_cfg_file}
    [ -f ${tsg_clixon_cfg_file} ] && sed -i "s/^hugepages=.*$/hugepages=${HUGEPAGES}/g" ${tsg_clixon_cfg_file}
+    [ -f ${tsg_clixon_cfg_file} ] && sed -i "s/^nf_count=.*$/nf_count=${NF_COUNT}/g" ${tsg_clixon_cfg_file}
 }

 set_mrzcpd_conf()
 {
    echo "sz_indirect_pktmbuf=${MRZCPD_INDIRECT_PKTMBUF}" > ${mrzcpd_conf_file}
    echo "sz_direct_pktmbuf=${MRZCPD_DIRECT_PKTMBUF}" >> ${mrzcpd_conf_file}
-    echo "iocore=${MRZCPD_IOCORE}" >> ${mrzcpd_conf_file}
+    echo "iocore=${IOCORE}" >> ${mrzcpd_conf_file}
    echo "poll_wait_throttle=${MRZCPD_POLL_WAIT_THROTTLE}" >> ${mrzcpd_conf_file}
    echo "sz_data=${MRZCPD_SZ_DATA}" >> ${mrzcpd_conf_file}
    echo "sz_tunnel=${MRZCPD_SZ_TUNNEL}" >> ${mrzcpd_conf_file}
@@ -121,12 +161,12 @@ set_mrzcpd_direct_pktmbuf()
 set_cpu_list()
 {
    local vendor
-    if [ -n "$MRZCPD_IOCORE" ] && [ -n "$CLIXON_IOCORE" ] && [ -n "$NUMA_NODE_CNT" ]; then
-        return 0
-    fi

-    vendor=$(lscpu | grep "^Vendor ID" | awk '{print $3}')
-    if [ $vendor == 'AuthenticAMD' ]; then
+    if [ -n "$NIC_CPU_Affinity_Switch" ] && [ "$NIC_CPU_Affinity_Switch" == "1" ]; then
+        . /opt/tsg/tsg-os-HAL/scripts/cpu_amd.sh
+    elif [ -n "$NIC_CPU_Affinity_Switch" ] && [ "$NIC_CPU_Affinity_Switch" == "0" ]; then
+        . /opt/tsg/tsg-os-HAL/scripts/cpu_default.sh
+    elif [ $vendor == 'AuthenticAMD' ]; then
        . /opt/tsg/tsg-os-HAL/scripts/cpu_amd.sh
    else
        . /opt/tsg/tsg-os-HAL/scripts/cpu_default.sh
@@ -156,7 +196,7 @@ set_grub_cmdline()
    local set_grub_flag=0

    grub_cmdline_key=("hugepagesz" "hugepages" "isolcpus")
-    grub_cmdline_value=("1G" ${HUGEPAGES} ${MRZCPD_IOCORE})
+    grub_cmdline_value=("1G" ${HUGEPAGES} ${IOCORE})

    for index in "${!grub_cmdline_key[@]}"
    do
@@ -224,7 +264,7 @@ build_and_mount_traffic_engine_logs_vdisk()
 set_cpu_partitioning()
 {
 cat <<EOF > ${cpu_partitioning_conf_file}
-isolated_cores=${MRZCPD_IOCORE}
+isolated_cores=${IOCORE}
 EOF
 }