diff --git a/ansible/roles/tsg-os-HAL/files/conf/tsg-os-HAL.conf b/ansible/roles/tsg-os-HAL/files/conf/tsg-os-HAL.conf index 68acdbc6..ae11ff5f 100644 --- a/ansible/roles/tsg-os-HAL/files/conf/tsg-os-HAL.conf +++ b/ansible/roles/tsg-os-HAL/files/conf/tsg-os-HAL.conf @@ -4,8 +4,10 @@ # SYSTEM_RESERVED= # NUMA_NODE_CNT= -# MRZCPD_IOCORE= -# CLIXON_IOCORE= +# IOCORE= +# WORKLOAD_CORE= + +# NF_COUNT= # MRZCPD_DIRECT_PKTMBUF= # MRZCPD_INDIRECT_PKTMBUF= @@ -13,3 +15,5 @@ # MRZCPD_SZ_DATA= # MRZCPD_SZ_TUNNEL= # MRZCPD_CHECK_BUFFER_LEAK= + +# NIC_CPU_Affinity_Switch= \ No newline at end of file diff --git a/ansible/roles/tsg-os-HAL/files/script/cpu_amd.sh b/ansible/roles/tsg-os-HAL/files/script/cpu_amd.sh index ca72763b..9a404be3 100644 --- a/ansible/roles/tsg-os-HAL/files/script/cpu_amd.sh +++ b/ansible/roles/tsg-os-HAL/files/script/cpu_amd.sh @@ -1,8 +1,16 @@ #!/bin/sh -set -x +write_devices_disable() +{ +(cat <> $1 +} + +write_devices_enable() { (cat <> ${tsg_clixon_cfg_file} +} + +del_device_rxcore_conf() +{ + sed -i '/^\[device.*\]/,/^$/d' ${tsg_clixon_cfg_file} +} allocate_cpu() { - local numa_node_min=0 - local numa_node_max=0 local numa_node_cpu_cnt=0 local numa_cpu_cnt_min=0 local numa_list - local numa_nodes local mrzcpd_cnt local clixon_min_index - numa_nodes=$(numactl --hardware | grep available | awk -F"[()]" '{print $2}') - - if [[ $numa_nodes == *-* ]] - then - read numa_node_min numa_node_max <<< $(echo $numa_nodes | awk -F- '{print $1" "$2}') - else - numa_node_min=$numa_nodes - numa_node_max=$numa_nodes + if [ ! -n "$NUMA_NODE_CNT" ]; then + NUMA_NODE_CNT=`lscpu | grep "NUMA node(s):" | head -n 1 | sed -r 's/NUMA node\(s\):\s{1,}//g'` fi - NUMA_NODE_CNT=`expr $numa_node_max - $numa_node_min + 1` - for((i=$numa_node_min;i<=$numa_node_max;i++)); - do - numa_node_cpu_cnt=$(numactl --hardware | awk '/^node ['$i']+ cpus:/ {print NF-3}') - if [ $numa_cpu_cnt_min -eq 0 ] || [ $numa_node_cpu_cnt -lt $numa_cpu_cnt_min ]; then - numa_cpu_cnt_min=$numa_node_cpu_cnt + + if [ ! -n "$IOCORE" ] || [ ! -n "$WORKLOAD_CORE" ]; then + unset IOCORE + unset WORKLOAD_CORE + + for((i=0;i<$NUMA_NODE_CNT;i++)); + do + numa_node_cpu_cnt=$(numactl --hardware | awk '/^node ['$i']+ cpus:/ {print NF-3}') + if [ $numa_cpu_cnt_min -eq 0 ] || [ $numa_node_cpu_cnt -lt $numa_cpu_cnt_min ]; then + numa_cpu_cnt_min=$numa_node_cpu_cnt + fi + done + + if [ $numa_cpu_cnt_min -le 4 ]; then + mrzcpd_cnt=1 + clixon_min_index=2 + elif [ $numa_cpu_cnt_min -le 16 ]; then + mrzcpd_cnt=2 + clixon_min_index=3 + else + mrzcpd_cnt=4 + clixon_min_index=5 fi - done - if [ $numa_cpu_cnt_min -le 4 ]; then - mrzcpd_cnt=1 - clixon_min_index=2 - elif [ $numa_cpu_cnt_min -le 16 ]; then - mrzcpd_cnt=2 - clixon_min_index=3 - else - mrzcpd_cnt=4 - clixon_min_index=5 + for((i=0;i<$NUMA_NODE_CNT;i++)); + do + numa_list=$(numactl --hardware | grep "node $i cpus: " | sed -r "s/node $i cpus: //g" | sed -r "s/^0 //g") + mrzcpd_core=$(echo $numa_list | cut -d ' ' -f 1-$mrzcpd_cnt | tr ' ' ',') + IOCORE=$IOCORE","$mrzcpd_core + clixon_core=$(echo $numa_list | cut -d ' ' -f $clixon_min_index- | tr ' ' ',') + WORKLOAD_CORE=$WORKLOAD_CORE","$clixon_core + done + IOCORE=${IOCORE#*,} + WORKLOAD_CORE=${WORKLOAD_CORE#*,} fi - - - for((i=$numa_node_min;i<=$numa_node_max;i++)); - do - numa_list=$(numactl --hardware | grep "node $i cpus: " | sed -r "s/node $i cpus: //g" | sed -r "s/^0 //g") - mrzcpd_core=$(echo $numa_list | cut -d ' ' -f 1-$mrzcpd_cnt | tr ' ' ',') - MRZCPD_IOCORE=$MRZCPD_IOCORE","$mrzcpd_core - clixon_core=$(echo $numa_list | cut -d ' ' -f $clixon_min_index- | tr ' ' ',') - CLIXON_IOCORE=$CLIXON_IOCORE","$clixon_core - done - MRZCPD_IOCORE=${MRZCPD_IOCORE#*,} - CLIXON_IOCORE=${CLIXON_IOCORE#*,} + del_device_rxcore_conf + write_devices_conf } diff --git a/ansible/roles/tsg-os-HAL/files/script/tsg-os-HAL.sh b/ansible/roles/tsg-os-HAL/files/script/tsg-os-HAL.sh index 827a7e61..23b2f731 100644 --- a/ansible/roles/tsg-os-HAL/files/script/tsg-os-HAL.sh +++ b/ansible/roles/tsg-os-HAL/files/script/tsg-os-HAL.sh @@ -17,8 +17,8 @@ HUGEPAGES= KUBE_RESERVED= SYSTEM_RESERVED= NUMA_NODE_CNT= -MRZCPD_IOCORE= -CLIXON_IOCORE= +IOCORE= +WORKLOAD_CORE= MRZCPD_DIRECT_PKTMBUF= MRZCPD_INDIRECT_PKTMBUF= MRZCPD_POLL_WAIT_THROTTLE=512 @@ -27,11 +27,50 @@ MRZCPD_SZ_TUNNEL=8192 MRZCPD_CHECK_BUFFER_LEAK=1 TRAFFIC_ENGINE_LOGS_VDISK_PATH="/data/vdisks" TRAFFIC_ENGINE_LOGS_VDISK_SIZE_BYTES= +NF_COUNT=16 +NIC_CPU_Affinity_Switch= + +cpu_format_conversion() +{ + local core_min + local core_max + local core_list + + if [[ $1 == *-* ]]; then + read core_min core_max <<< $(echo $1 | awk -F- '{print $1" "$2}') + for((i=$core_min;i<=$core_max;i++)); + do + core_list=$core_list","$i + done + core_list=${core_list#*,} + else + core_list=$1 + fi + echo $core_list +} load_tsg_os_HAL_config() { if [ -f "${tsg_os_HAL_cfg_file}" ]; then source ${tsg_os_HAL_cfg_file} + + if [ ! -n "$IOCORE" ] && [ -n "$MRZCPD_IOCORE" ]; then + IOCORE=$MRZCPD_IOCORE + fi + + if [ ! -n "$WORKLOAD_CORE" ] && [ -n "$CLIXON_IOCORE" ]; then + WORKLOAD_CORE=$CLIXON_IOCORE + fi + + if [ -n $IOCORE ]; then + core_list=$(echo $IOCORE | tr ',' ' ') + for core in $core_list; + do + core_format=$core_format","$(cpu_format_conversion $core) + done + core_format=${core_format#*,} + IOCORE=$core_format + fi fi } @@ -66,15 +105,16 @@ EOF set_tsg_clixon_conf() { - [ -f ${tsg_clixon_cfg_file} ] && sed -i "s/^cpu_range=.*$/cpu_range=${CLIXON_IOCORE}/g" ${tsg_clixon_cfg_file} + [ -f ${tsg_clixon_cfg_file} ] && sed -i "s/^cpu_range=.*$/cpu_range=${WORKLOAD_CORE}/g" ${tsg_clixon_cfg_file} [ -f ${tsg_clixon_cfg_file} ] && sed -i "s/^hugepages=.*$/hugepages=${HUGEPAGES}/g" ${tsg_clixon_cfg_file} + [ -f ${tsg_clixon_cfg_file} ] && sed -i "s/^nf_count=.*$/nf_count=${NF_COUNT}/g" ${tsg_clixon_cfg_file} } set_mrzcpd_conf() { echo "sz_indirect_pktmbuf=${MRZCPD_INDIRECT_PKTMBUF}" > ${mrzcpd_conf_file} echo "sz_direct_pktmbuf=${MRZCPD_DIRECT_PKTMBUF}" >> ${mrzcpd_conf_file} - echo "iocore=${MRZCPD_IOCORE}" >> ${mrzcpd_conf_file} + echo "iocore=${IOCORE}" >> ${mrzcpd_conf_file} echo "poll_wait_throttle=${MRZCPD_POLL_WAIT_THROTTLE}" >> ${mrzcpd_conf_file} echo "sz_data=${MRZCPD_SZ_DATA}" >> ${mrzcpd_conf_file} echo "sz_tunnel=${MRZCPD_SZ_TUNNEL}" >> ${mrzcpd_conf_file} @@ -121,12 +161,12 @@ set_mrzcpd_direct_pktmbuf() set_cpu_list() { local vendor - if [ -n "$MRZCPD_IOCORE" ] && [ -n "$CLIXON_IOCORE" ] && [ -n "$NUMA_NODE_CNT" ]; then - return 0 - fi - vendor=$(lscpu | grep "^Vendor ID" | awk '{print $3}') - if [ $vendor == 'AuthenticAMD' ]; then + if [ -n "$NIC_CPU_Affinity_Switch" ] && [ "$NIC_CPU_Affinity_Switch" == "1" ]; then + . /opt/tsg/tsg-os-HAL/scripts/cpu_amd.sh + elif [ -n "$NIC_CPU_Affinity_Switch" ] && [ "$NIC_CPU_Affinity_Switch" == "0" ]; then + . /opt/tsg/tsg-os-HAL/scripts/cpu_default.sh + elif [ $vendor == 'AuthenticAMD' ]; then . /opt/tsg/tsg-os-HAL/scripts/cpu_amd.sh else . /opt/tsg/tsg-os-HAL/scripts/cpu_default.sh @@ -156,7 +196,7 @@ set_grub_cmdline() local set_grub_flag=0 grub_cmdline_key=("hugepagesz" "hugepages" "isolcpus") - grub_cmdline_value=("1G" ${HUGEPAGES} ${MRZCPD_IOCORE}) + grub_cmdline_value=("1G" ${HUGEPAGES} ${IOCORE}) for index in "${!grub_cmdline_key[@]}" do @@ -224,7 +264,7 @@ build_and_mount_traffic_engine_logs_vdisk() set_cpu_partitioning() { cat < ${cpu_partitioning_conf_file} -isolated_cores=${MRZCPD_IOCORE} +isolated_cores=${IOCORE} EOF }