gfwleak/tsg-tsg-os-buildimage
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HAL渲染Clixon DefaultDB

Browse Source
This commit is contained in:
wangmenglan
2024-06-13 10:48:18 +08:00
committed by 林鑫
parent f5d436990d
commit 2b5461f9a0
6 changed files with 181 additions and 259 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
ansible/roles/tsg-os-HAL/files/conf/tsg-os-HAL.conf
View File

@@ -1,7 +1,5 @@
# HUGEPAGES=
# KUBE_RESERVED=
# SYSTEM_RESERVED=
# NUMA_NODE_CNT=
# IOCORE=

16
ansible/roles/tsg-os-HAL/files/script/build_clixon_default_db.sh Normal file
View File

@@ -0,0 +1,16 @@
#!/bin/sh
build_default_db()
{
local resources="<config>
<system xmlns=\"urn:yang:tsg-os-mgnt-srv\">
<resources>
<packet_io_engine_hugepages>${HUGEPAGES}</packet_io_engine_hugepages>
<packet_io_engine_core>${IOCORE}</packet_io_engine_core>
<workload_core>${WORKLOAD_CORE}</workload_core>
</resources>
</system>
</config>"
echo ${resources} | xmlstarlet fo > ${tsg_clixon_default_db_file}
sed -i '/<?xml version="1.0"?>/d' ${tsg_clixon_default_db_file}
}

122
ansible/roles/tsg-os-HAL/files/script/cpu_amd.sh
View File

@@ -1,114 +1,17 @@
#!/bin/sh
write_devices_disable()
{
(cat <<EOF1
[device]
rxcore_enable=0
EOF1
) >> $1
}
write_devices_enable()
{
(cat <<EOF1
[device]
rxcore_enable=1
device=$1
EOF1
) >> $2
}
write_device_rxcore_conf()
{
(cat <<EOF1
[device:$1]
rx_cores=$2
EOF1
) >> $3
}
del_device_rxcore_conf()
{
sed -i '/^\[device.*\]/,/^$/d' ${tsg_clixon_cfg_file}
}
set_mrzcpd_rx_core()
{
local devices
local str_device
del_device_rxcore_conf
devices=$(lshw -c network -businfo | grep network | awk '{print $2}')
for device in $devices
do
if [ $device == "network" ]; then
continue
fi
businfo=$(ethtool -i $device | grep "bus-info:"| awk '{print $2}')
numa_node=$(lspci -s $businfo -vv | grep "NUMA node:" | awk '{print $3}')
if [ ! -n "$numa_node" ] || [ $numa_node -ge $NUMA_NODE_CNT ]; then
continue
fi
write_device_rxcore_conf $device ${mrzcpd_rx_core[$numa_node]} ${tsg_clixon_cfg_file}
str_device=$str_device","$device
done
str_device=${str_device#*,}
if [ ! -n "$str_device" ]; then
write_devices_disable ${tsg_clixon_cfg_file}
else
write_devices_enable $str_device ${tsg_clixon_cfg_file}
fi
}
output_common_str()
{
local common_list
list1=$(echo $1 | tr ',' ' ')
list2=$(echo $2 | tr ',' ' ')
for numa_core in $list1;
do
for iocore in $list2;
do
if [ $numa_core -eq $iocore ]; then
common_list=$common_list","$numa_core
break
fi
done
done
common_list=${common_list#*,}
echo $common_list
}
allocate_cpu()
{
local numa_node_cpu_cnt=0
local numa_cpu_cnt_min=0
local numa_list
local cpus
local numa_cpus
local clixon_cpus
local mrzcpd_start
local mrzcpd_end
local clixon_start
local numa_cpu_cnt_min=0
local numa_node_cpu_cnt=0
if [ ! -n "$NUMA_NODE_CNT" ]; then
NUMA_NODE_CNT=`lscpu | grep "NUMA node(s):" | head -n 1 | sed -r 's/NUMA node\(s\):\s{1,}//g'`
fi
if [ -n "$IOCORE" ] && [ -n "$WORKLOAD_CORE" ]; then
for((i=0;i<$NUMA_NODE_CNT;i++));
do
numa_list=$(numactl --hardware | grep "node $i cpus: " | sed -r "s/node $i cpus: //g" | tr ' ' ',')
mrzcpd_core=$(output_common_str $numa_list $IOCORE)
mrzcpd_rx_core+=($mrzcpd_core)
done
else
if [ ! -n "$IOCORE" ] || [ ! -n "$WORKLOAD_CORE" ]; then
unset IOCORE
unset WORKLOAD_CORE
@@ -140,16 +43,13 @@ allocate_cpu()
for((i=0;i<$NUMA_NODE_CNT;i++));
do
numa_list=$(numactl --hardware | grep "node $i cpus: " | sed -r "s/node $i cpus: //g")
mrzcpd_core=$(echo $numa_list | cut -d ' ' -f $mrzcpd_start-$mrzcpd_end | tr ' ' ',')
IOCORE=$IOCORE","$mrzcpd_core
mrzcpd_rx_core+=($mrzcpd_core)
clixon_core=$(echo $numa_list | cut -d ' ' -f $clixon_start- | tr ' ' ',')
WORKLOAD_CORE=$WORKLOAD_CORE","$clixon_core
numa_cpus=$(numactl --hardware | grep "node $i cpus: " | sed -r "s/node $i cpus: //g")
cpus=$(echo $numa_cpus | cut -d ' ' -f $mrzcpd_start-$mrzcpd_end | tr ' ' ',')
IOCORE=$IOCORE","$cpus
clixon_cpus=$(echo $numa_cpus | cut -d ' ' -f $clixon_start- | tr ' ' ',')
WORKLOAD_CORE=$WORKLOAD_CORE","$clixon_cpus
done
IOCORE=${IOCORE#*,}
WORKLOAD_CORE=${WORKLOAD_CORE#*,}
fi
set_mrzcpd_rx_core
}

53
ansible/roles/tsg-os-HAL/files/script/cpu_default.sh
View File

@@ -1,31 +1,14 @@
#!/bin/sh
write_devices_conf()
{
(cat <<EOF1
[device]
rxcore_enable=0
EOF1
) >> ${tsg_clixon_cfg_file}
}
del_device_rxcore_conf()
{
sed -i '/^\[device.*\]/,/^$/d' ${tsg_clixon_cfg_file}
}
allocate_cpu()
{
local numa_node_cpu_cnt=0
local cpus
local numa_cpus
local clixon_cpus
local mrzcpd_end
local clixon_start
local numa_cpu_cnt_min=0
local numa_list
local mrzcpd_cnt
local clixon_min_index
if [ ! -n "$NUMA_NODE_CNT" ]; then
NUMA_NODE_CNT=`lscpu | grep "NUMA node(s):" | head -n 1 | sed -r 's/NUMA node\(s\):\s{1,}//g'`
fi
local numa_node_cpu_cnt=0
if [ ! -n "$IOCORE" ] || [ ! -n "$WORKLOAD_CORE" ]; then
unset IOCORE
@@ -40,27 +23,25 @@ allocate_cpu()
done
if [ $numa_cpu_cnt_min -le 4 ]; then
mrzcpd_cnt=1
clixon_min_index=2
mrzcpd_end=1
clixon_start=2
elif [ $numa_cpu_cnt_min -le 16 ]; then
mrzcpd_cnt=2
clixon_min_index=3
mrzcpd_end=2
clixon_start=3
else
mrzcpd_cnt=4
clixon_min_index=5
mrzcpd_end=4
clixon_start=5
fi
for((i=0;i<$NUMA_NODE_CNT;i++));
do
numa_list=$(numactl --hardware | grep "node $i cpus: " | sed -r "s/node $i cpus: //g" | sed -r "s/^0 //g")
mrzcpd_core=$(echo $numa_list | cut -d ' ' -f 1-$mrzcpd_cnt | tr ' ' ',')
IOCORE=$IOCORE","$mrzcpd_core
clixon_core=$(echo $numa_list | cut -d ' ' -f $clixon_min_index- | tr ' ' ',')
WORKLOAD_CORE=$WORKLOAD_CORE","$clixon_core
numa_cpus=$(numactl --hardware | grep "node $i cpus: " | sed -r "s/node $i cpus: //g" | sed -r "s/^0 //g")
cpus=$(echo $numa_cpus | cut -d ' ' -f 1-$mrzcpd_end | tr ' ' ',')
IOCORE=$IOCORE","$cpus
clixon_cpus=$(echo $numa_cpus | cut -d ' ' -f $clixon_start- | tr ' ' ',')
WORKLOAD_CORE=$WORKLOAD_CORE","$clixon_cpus
done
IOCORE=${IOCORE#*,}
WORKLOAD_CORE=${WORKLOAD_CORE#*,}
fi
del_device_rxcore_conf
write_devices_conf
}

101
ansible/roles/tsg-os-HAL/files/script/set_nic_cpu_affinity.sh Normal file
View File

@@ -0,0 +1,101 @@
#!/bin/sh
nic_cpu_affinity_disable()
{
(cat <<EOF1
[device]
rxcore_enable=0
EOF1
) >> ${tsg_clixon_cfg_file}
}
nic_cpu_affinity_enable()
{
(cat <<EOF1
[device]
rxcore_enable=1
device=$1
EOF1
) >> ${tsg_clixon_cfg_file}
}
set_nic_cpu_affinity()
{
(cat <<EOF1
[device:$1]
rx_cores=$2
EOF1
) >> ${tsg_clixon_cfg_file}
}
delete_nic_cpu_affinity_config()
{
sed -i '/^\[device.*\]/,/^$/d' ${tsg_clixon_cfg_file}
}
output_common_cpu()
{
local common_list
list1=$(echo $1 | tr ',' ' ')
list2=$(echo $2 | tr ',' ' ')
for numa_core in $list1;
do
for iocore in $list2;
do
if [ $numa_core -eq $iocore ]; then
common_list=$common_list","$numa_core
break
fi
done
done
common_list=${common_list#*,}
echo $common_list
}
build_nic_cpu_affinity_configuration()
{
local cpus
local cpu_list
local numa_cpus
local devices
local device_list
local NIC_CPU_Affinity_Switch=$1
delete_nic_cpu_affinity_config
if [ -z "$NIC_CPU_Affinity_Switch" ] || [ "$NIC_CPU_Affinity_Switch" == "0" ]; then
nic_cpu_affinity_disable
return
fi
for((i=0;i<$NUMA_NODE_CNT;i++));
do
numa_cpus=$(numactl --hardware | grep "node $i cpus: " | sed -r "s/node $i cpus: //g" | tr ' ' ',')
cpus=$(output_common_cpu $numa_cpus $IOCORE)
cpu_list+=("$cpus")
done
devices=$(lshw -c network -businfo | grep network | awk '{print $2}')
for device in $devices
do
if [ $device == "network" ]; then
continue
fi
businfo=$(ethtool -i $device | grep "bus-info:"| awk '{print $2}')
numa_node=$(lspci -s $businfo -vv | grep "NUMA node:" | awk '{print $3}')
if [ -z "$numa_node" ] || [ $numa_node -ge $NUMA_NODE_CNT ] || [ -z ${cpu_list[$numa_node]} ]; then
continue
fi
set_nic_cpu_affinity $device ${cpu_list[$numa_node]} ${tsg_clixon_cfg_file}
device_list=$device_list","$device
done
device_list=${device_list#*,}
nic_cpu_affinity_enable $device_list
}

146
ansible/roles/tsg-os-HAL/files/script/tsg-os-HAL.sh
View File

@@ -4,11 +4,10 @@ set -x
tsg_os_HAL_cfg_file="/etc/sysconfig/tsg-os-HAL.conf"
grub_cfg_file="/boot/grub/grub.cfg"
tsg_clixon_cfg_file="/opt/tsg/clixon/etc/mgnt-srv.conf"
k3s_cfg_file="/etc/rancher/k3s/config.yaml"
mrzcpd_conf_file="/var/run/share/container_mrzcpd.conf"
cpu_partitioning_conf_file='/etc/tuned/cpu-partitioning-variables.conf'
tsg_clixon_default_db_file='/etc/clixon/default_db'
cpu_core_num=$(lscpu | grep "CPU(s):" | head -n 1 | sed -r 's/CPU\(s\):\s{1,}//g')
mem_num=$(free -m | grep Mem | awk '{print $2}')
@@ -77,50 +76,15 @@ load_tsg_os_HAL_config()
fi
}
set_k3s_config()
set_clixon_configuration()
{
local mem_num="$1"
local system_mem
if [ ! -n "$KUBE_RESERVED" ]; then
KUBE_RESERVED=2
fi
if [ ! -n "$SYSTEM_RESERVED" ]; then
if [ $mem_num -le "32768" ]; then
SYSTEM_RESERVED=8
elif [ $mem_num -le "65536" ]; then
SYSTEM_RESERVED=8
elif [ $mem_num -le "131072" ]; then
SYSTEM_RESERVED=16
else
SYSTEM_RESERVED=32
fi
fi
system_mem=$(($HUGEPAGES + $SYSTEM_RESERVED))
cat <<EOF > ${k3s_cfg_file}
kubelet-arg:
- "kube-reserved=memory=${KUBE_RESERVED}Gi"
- "system-reserved=memory=${system_mem}Gi"
- "sync-frequency=1s"
kube-apiserver-arg:
- "event-ttl=48h0m0s"
EOF
}
set_tsg_clixon_conf()
{
[ -f ${tsg_clixon_cfg_file} ] && sed -i "s/^cpu_range=.*$/cpu_range=${WORKLOAD_CORE}/g" ${tsg_clixon_cfg_file}
[ -f ${tsg_clixon_cfg_file} ] && sed -i "s/^hugepages=.*$/hugepages=${HUGEPAGES}/g" ${tsg_clixon_cfg_file}
[ -f ${tsg_clixon_cfg_file} ] && sed -i "s/^nf_count=.*$/nf_count=${NF_COUNT}/g" ${tsg_clixon_cfg_file}
}
set_mrzcpd_conf()
set_mrzcpd_configuration()
{
echo "sz_indirect_pktmbuf=${MRZCPD_INDIRECT_PKTMBUF}" > ${mrzcpd_conf_file}
echo "sz_direct_pktmbuf=${MRZCPD_DIRECT_PKTMBUF}" >> ${mrzcpd_conf_file}
echo "iocore=${IOCORE}" >> ${mrzcpd_conf_file}
echo "poll_wait_throttle=${MRZCPD_POLL_WAIT_THROTTLE}" >> ${mrzcpd_conf_file}
echo "sz_data=${MRZCPD_SZ_DATA}" >> ${mrzcpd_conf_file}
echo "sz_tunnel=${MRZCPD_SZ_TUNNEL}" >> ${mrzcpd_conf_file}
@@ -128,7 +92,7 @@ set_mrzcpd_conf()
echo "create_mode=${MRZCPD_CREATE_MODE}" >> ${mrzcpd_conf_file}
}
set_hugepages()
calculate_hugepages()
{
if [ ! -n "$HUGEPAGES" ]; then
if [ $mem_num -le "32768" ]; then
@@ -143,14 +107,14 @@ set_hugepages()
fi
}
set_mrzcpd_indirect_pktmbuf()
calculate_mrzcpd_indirect_pktmbuf()
{
if [ ! -n "$MRZCPD_INDIRECT_PKTMBUF" ]; then
MRZCPD_INDIRECT_PKTMBUF=`expr 8192 / $NUMA_NODE_CNT`
fi
}
set_mrzcpd_sz_tunnel()
calculate_mrzcpd_sz_tunnel()
{
if [ ! -n "$MRZCPD_SZ_TUNNEL" ]; then
if [ $mem_num -le "32768" ]; then
@@ -165,7 +129,7 @@ set_mrzcpd_sz_tunnel()
fi
}
set_mrzcpd_direct_pktmbuf()
calculate_mrzcpd_direct_pktmbuf()
{
if [ ! -n "$MRZCPD_DIRECT_PKTMBUF" ]; then
if [ $mem_num -le "32768" ]; then
@@ -180,17 +144,16 @@ set_mrzcpd_direct_pktmbuf()
fi
}
set_cpu_list()
calculate_cpu_list()
{
local vendor
vendor=$(lscpu | grep "^Vendor ID" | awk '{print $3}')
if [ -n "$NIC_CPU_Affinity_Switch" ] && [ "$NIC_CPU_Affinity_Switch" == "1" ]; then
. /opt/tsg/tsg-os-HAL/scripts/cpu_amd.sh
elif [ -n "$NIC_CPU_Affinity_Switch" ] && [ "$NIC_CPU_Affinity_Switch" == "0" ]; then
. /opt/tsg/tsg-os-HAL/scripts/cpu_default.sh
elif [ "$vendor" == 'AuthenticAMD' ]; then
if [ "$vendor" == 'AuthenticAMD' ]; then
. /opt/tsg/tsg-os-HAL/scripts/cpu_amd.sh
if [ ! -n "$NIC_CPU_Affinity_Switch" ]; then
NIC_CPU_Affinity_Switch=1
fi
else
. /opt/tsg/tsg-os-HAL/scripts/cpu_default.sh
fi
@@ -198,42 +161,6 @@ set_cpu_list()
allocate_cpu
}
set_grub_cmdline_file()
{
local result=0
local grub_cmdline_key="$1"
local grub_cmdline_value="$2"
if ! grep -q " ${grub_cmdline_key}=" "$grub_cfg_file"; then
sed -i "/ linux / s/$/ ${grub_cmdline_key}=${grub_cmdline_value} /" $grub_cfg_file
result=1
elif ! grep -qE " ${grub_cmdline_key}=${grub_cmdline_value}\s| ${grub_cmdline_key}=${grub_cmdline_value}$" "$grub_cfg_file"; then
sed -i "s/ ${grub_cmdline_key}=\(\w\+,\)*\(\w\+-\w\+,*\)*\w*,*\s*/ ${grub_cmdline_key}=${grub_cmdline_value} /" $grub_cfg_file
result=1
fi
return $result
}
set_grub_cmdline()
{
local set_grub_flag=0
grub_cmdline_key=("hugepagesz" "hugepages" "isolcpus")
grub_cmdline_value=("1G" ${HUGEPAGES} ${IOCORE})
for index in "${!grub_cmdline_key[@]}"
do
set_grub_cmdline_file ${grub_cmdline_key[$index]} ${grub_cmdline_value[$index]}
if [ "$?" == "1" ]; then
set_grub_flag=1
fi
done
if [ "$set_grub_flag" == "1" ]; then
reboot
fi
}
build_and_mount_traffic_engine_logs_vdisk()
{
local vdisk_path=${TRAFFIC_ENGINE_LOGS_VDISK_PATH}
@@ -283,14 +210,6 @@ build_and_mount_traffic_engine_logs_vdisk()
fi
}
set_cpu_partitioning()
{
cat <<EOF > ${cpu_partitioning_conf_file}
isolated_cores=${IOCORE}
EOF
}
read_device_type()
{
product_name=`ipmitool fru list | grep 'Product Name' | awk '{print $4}' | head -n 1`
@@ -346,33 +265,40 @@ metadata:
EOF
}
load_tsg_os_HAL_config
mkdir -p /run/mrzcpd/hugepages
mkdir -p /var/run/share
build_and_mount_traffic_engine_logs_vdisk
set_hugepages
set_cpu_list
set_grub_cmdline
set_cpu_partitioning
mkdir -p /run/mrzcpd/hugepages
mount -t hugetlbfs -o pagesize=1073741824 nodev /var/run/mrzcpd/hugepages
modprobe vfio
echo 1 > /sys/module/vfio/parameters/enable_unsafe_noiommu_mode
modprobe vfio_pci
set_mrzcpd_indirect_pktmbuf
set_mrzcpd_sz_tunnel
set_mrzcpd_direct_pktmbuf
load_tsg_os_HAL_config
set_k3s_config ${mem_num}
set_tsg_clixon_conf
set_mrzcpd_conf
if [ ! -n "$NUMA_NODE_CNT" ]; then
NUMA_NODE_CNT=`lscpu | grep "NUMA node(s):" | head -n 1 | sed -r 's/NUMA node\(s\):\s{1,}//g'`
fi
build_and_mount_traffic_engine_logs_vdisk
calculate_hugepages
calculate_cpu_list
. /opt/tsg/tsg-os-HAL/scripts/set_nic_cpu_affinity.sh
build_nic_cpu_affinity_configuration ${NIC_CPU_Affinity_Switch}
calculate_mrzcpd_indirect_pktmbuf
calculate_mrzcpd_sz_tunnel
calculate_mrzcpd_direct_pktmbuf
set_k3s_configuration ${mem_num}
set_clixon_configuration
set_mrzcpd_configuration
# add sn to k8s node annotations.
read_device_type
read_sn
retry_read_sn_when_sn_unavailable
add_node_metadata
. /opt/tsg/tsg-os-HAL/scripts/build_clixon_default_db.sh
build_default_db