diff --git a/ansible/install_config/group_vars/rpm_version.yml b/ansible/install_config/group_vars/rpm_version.yml index dfdb2896..d0115639 100644 --- a/ansible/install_config/group_vars/rpm_version.yml +++ b/ansible/install_config/group_vars/rpm_version.yml @@ -100,7 +100,7 @@ hasp_update_rpm_version: tsg_os_clixon_rpm_version: cligen: cligen-5.8.0-release clixon: clixon-5.8.0-release - tsg-os-mgnt-srv: tsg-os-mgnt-srv-1.0.41.7b86d6b + tsg-os-mgnt-srv: tsg-os-mgnt-srv-1.0.42.b5d7a44 sce_rpm_version: sce: sce-1.0.13.23d05e7 diff --git a/ansible/roles/traffic-engine/files/helm/conf/main.conf b/ansible/roles/traffic-engine/files/helm/conf/main.conf index 471bde33..59ac609c 100644 --- a/ansible/roles/traffic-engine/files/helm/conf/main.conf +++ b/ansible/roles/traffic-engine/files/helm/conf/main.conf @@ -136,7 +136,9 @@ TRAFFIC_MIRROR_ENABLE=1 {{- else }} TRAFFIC_MIRROR_ENABLE=0 {{- end }} -NIC_NAME="{{ .Values.nic_mirror_name }}" +{{- if .Values.nic_mirror_name.firewall }} +NIC_NAME="{{ .Values.nic_mirror_name.firewall }}" +{{- end }} DEFAULT_VLAN_ID=0 [RADIUS_PLUG] DEVICE_TAGS={"tags":[{{- include "traffic-engine.device-tag-list" . }}]} diff --git a/ansible/roles/traffic-engine/files/helm/conf/tfe.conf b/ansible/roles/traffic-engine/files/helm/conf/tfe.conf index 69e6b775..b5bec628 100644 --- a/ansible/roles/traffic-engine/files/helm/conf/tfe.conf +++ b/ansible/roles/traffic-engine/files/helm/conf/tfe.conf @@ -163,7 +163,9 @@ enable=1 {{- else }} enable=0 {{- end }} -device={{ .Values.nic_mirror_name }} +{{- if .Values.nic_mirror_name.proxy }} +device={{ .Values.nic_mirror_name.proxy }} +{{- end }} # 0:TRAFFIC_MIRROR_ETHDEV_AF_PACKET; 1:TRAFFIC_MIRROR_ETHDEV_MARSIO type=1 @@ -239,7 +241,7 @@ bpf_obj=/opt/tsg/tfe/resource/bpf/bpf_tun_rss_steering.o bpf_debug_log=0 # 2: BPF 使用二元组分流 # 4: BPF 使用四元组分流 -bpf_hash_mode=4 +bpf_hash_mode={{ .Values.distmode }} # 配置 tap 网卡的 RPS tap_rps_enable=1 diff --git a/ansible/roles/traffic-engine/files/helm/values.yaml b/ansible/roles/traffic-engine/files/helm/values.yaml index 58335a59..69045662 100644 --- a/ansible/roles/traffic-engine/files/helm/values.yaml +++ b/ansible/roles/traffic-engine/files/helm/values.yaml @@ -126,7 +126,9 @@ tfe_rps_mask: "00000000" nic_policy_log_name: eth0 nic_raw_name: eth0 -nic_mirror_name: eth0 +nic_mirror_name: + firewall: eth0 + proxy: eth0 define_enable_val_yes: yes define_enable_val_no: no @@ -188,6 +190,7 @@ shaping_config: shaping_nic: nf_1_shaping_engine app_symbol_index: 1 +distmode: 2 debug: firewall: diff --git a/ansible/roles/tsg-os-HAL/files/conf/tsg-os-HAL.conf b/ansible/roles/tsg-os-HAL/files/conf/tsg-os-HAL.conf new file mode 100644 index 00000000..8b2578f9 --- /dev/null +++ b/ansible/roles/tsg-os-HAL/files/conf/tsg-os-HAL.conf @@ -0,0 +1,10 @@ + +# HUGEPAGES= + +# 需要同时指定才能生效 +# NUMA_NODE_CNT= +# MRZCPD_IOCORE= +# CLIXON_IOCORE= + +# MRZCPD_DIRECT_PKTMBUF= +# MRZCPD_INDIRECT_PKTMBUF= diff --git a/ansible/roles/tsg-os-HAL/files/script/tsg-os-HAL.sh b/ansible/roles/tsg-os-HAL/files/script/tsg-os-HAL.sh index 5fa0580d..82918963 100644 --- a/ansible/roles/tsg-os-HAL/files/script/tsg-os-HAL.sh +++ b/ansible/roles/tsg-os-HAL/files/script/tsg-os-HAL.sh @@ -1,8 +1,8 @@ #!/bin/sh -set -xe +set -x -tsg_os_HAL_cfg_file="/etc/sysconfig/tsg-os-HAL" +tsg_os_HAL_cfg_file="/etc/sysconfig/tsg-os-HAL.conf" grub_cfg_file="/boot/grub/grub.cfg" mrzcpd_cfg_file="/etc/sysconfig/mrzcpd" @@ -11,46 +11,20 @@ k3s_cfg_file="/etc/rancher/k3s/config.yaml" mrglobal_cfg_template="/opt/tsg/clixon/etc/mrglobal.conf.jinja" cpu_core_num=$(lscpu | grep "CPU(s):" | head -n 1 | sed -r 's/CPU\(s\):\s{1,}//g') +numa_nodes=$(numactl --hardware | grep available | awk -F"[()]" '{print $2}') mem_num=$(free -m | grep Mem | awk '{print $2}') HUGEPAGES= +NUMA_NODE_CNT= MRZCPD_IOCORE= +CLIXON_IOCORE= MRZCPD_DIRECT_PKTMBUF= -TSG_CLIXON_CPU_QUOTAS_MIN= +MRZCPD_INDIRECT_PKTMBUF= load_tsg_os_HAL_config() { - [ -f ${tsg_os_HAL_cfg_file} ] || return 1 - - source ${tsg_os_HAL_cfg_file} - return 0 -} - -set_resouce_by_HAL() -{ - local mem_num="$1" - local cpu_core_num="$2" - - if [ $mem_num -le "32768" ]; then - HUGEPAGES=4 - MRZCPD_DIRECT_PKTMBUF=262144 - elif [ $mem_num -le "65536" ]; then - HUGEPAGES=8 - MRZCPD_DIRECT_PKTMBUF=1048576 - else - HUGEPAGES=16 - MRZCPD_DIRECT_PKTMBUF=2097152 - fi - - if [ ${cpu_core_num} -le "4" ]; then - MRZCPD_IOCORE="1" - TSG_CLIXON_CPU_QUOTAS_MIN=2 - elif [ ${cpu_core_num} -le "16" ]; then - MRZCPD_IOCORE="1,2" - TSG_CLIXON_CPU_QUOTAS_MIN=4 - else - MRZCPD_IOCORE="1,2,3,4" - TSG_CLIXON_CPU_QUOTAS_MIN=9 + if [ -f "${tsg_os_HAL_cfg_file}" ]; then + source ${tsg_os_HAL_cfg_file} fi } @@ -83,10 +57,11 @@ EOF set_tsg_clixon_conf() { [ -f ${mrglobal_cfg_template} ] && sed -i "s/^sz_direct_pktmbuf=.*$/sz_direct_pktmbuf=${MRZCPD_DIRECT_PKTMBUF}/g" ${mrglobal_cfg_template} + [ -f ${mrglobal_cfg_template} ] && sed -i "s/^sz_indirect_pktmbuf=.*$/sz_indirect_pktmbuf=${MRZCPD_INDIRECT_PKTMBUF}/g" ${mrglobal_cfg_template} [ -f ${mrglobal_cfg_template} ] && sed -i "s/^iocore=.*$/iocore=${MRZCPD_IOCORE}/g" ${mrglobal_cfg_template} - [ -f ${tsg_clixon_cfg_file} ] && sed -i "s/^cpu_range=.*$/cpu_range=${TSG_CLIXON_CPU_QUOTAS_MIN}/g" ${tsg_clixon_cfg_file} + [ -f ${tsg_clixon_cfg_file} ] && sed -i "s/^cpu_range=.*$/cpu_range=${CLIXON_IOCORE}/g" ${tsg_clixon_cfg_file} } set_os_hugepages() @@ -97,6 +72,90 @@ set_os_hugepages() return 0 } +set_hugepages() +{ + if [ ! -n "$HUGEPAGES" ]; then + if [ $mem_num -le "32768" ]; then + HUGEPAGES=4 + elif [ $mem_num -le "65536" ]; then + HUGEPAGES=8 + else + HUGEPAGES=16 + fi + fi +} + +set_mrzcpd_indirect_pktmbuf() +{ + if [ ! -n "$MRZCPD_INDIRECT_PKTMBUF" ]; then + MRZCPD_INDIRECT_PKTMBUF=`expr 8192 / $NUMA_NODE_CNT` + fi +} + +set_mrzcpd_direct_pktmbuf() +{ + if [ ! -n "$MRZCPD_DIRECT_PKTMBUF" ]; then + if [ $mem_num -le "32768" ]; then + MRZCPD_DIRECT_PKTMBUF=`expr 262144 / $NUMA_NODE_CNT` + elif [ $mem_num -le "65536" ]; then + MRZCPD_DIRECT_PKTMBUF=`expr 1048576 / $NUMA_NODE_CNT` + else + MRZCPD_DIRECT_PKTMBUF=`expr 2097152 / $NUMA_NODE_CNT` + fi + fi +} + +allocate_cpu() +{ + local numa_node_min=0 + local numa_node_max=0 + local numa_node_cpu_cnt=0 + local numa_cpu_cnt_min=0 + local numa_list + + if [ -n "$MRZCPD_IOCORE" ] && [ -n "$CLIXON_IOCORE" ] && [ -n "$NUMA_NODE_CNT" ]; then + return 0 + fi + + if [[ $numa_nodes == *-* ]] + then + read numa_node_min numa_node_max <<< $(echo $numa_nodes | awk -F- '{print $1" "$2}') + else + numa_node_min=$numa_nodes + numa_node_max=$numa_nodes + fi + NUMA_NODE_CNT=`expr $numa_node_max - $numa_node_min + 1` + for((i=$numa_node_min;i<=$numa_node_max;i++)); + do + numa_node_cpu_cnt=$(numactl --hardware | awk '/^node ['$i']+ cpus:/ {print NF-3}') + if [ $numa_cpu_cnt_min -eq 0 ] || [ $numa_node_cpu_cnt -lt $numa_cpu_cnt_min ]; then + numa_cpu_cnt_min=$numa_node_cpu_cnt + fi + done + + if [ $numa_cpu_cnt_min -le 4 ]; then + mrzcpd_cnt=1 + clixon_min_index=2 + elif [ $numa_cpu_cnt_min -le 16 ]; then + mrzcpd_cnt=2 + clixon_min_index=3 + else + mrzcpd_cnt=4 + clixon_min_index=5 + fi + + for((i=$numa_node_min;i<=$numa_node_max;i++)); + do + numa_list=$(numactl --hardware | grep "node $i cpus: " | sed -r "s/node $i cpus: //g" | sed -r "s/^0 //g") + mrzcpd_core=$(echo $numa_list | cut -d ' ' -f 1-$mrzcpd_cnt | tr ' ' ',') + MRZCPD_IOCORE=$MRZCPD_IOCORE","$mrzcpd_core + clixon_core=$(echo $numa_list | cut -d ' ' -f $clixon_min_index- | tr ' ' ',') + CLIXON_IOCORE=$CLIXON_IOCORE","$clixon_core + done + MRZCPD_IOCORE=${MRZCPD_IOCORE#*,} + CLIXON_IOCORE=${CLIXON_IOCORE#*,} +} + set_bridge() { ip link add br_dign_c type bridge @@ -105,7 +164,12 @@ set_bridge() ip link set br_dign_s up } -load_tsg_os_HAL_config || set_resouce_by_HAL ${mem_num} ${cpu_core_num} +load_tsg_os_HAL_config + +allocate_cpu +set_hugepages +set_mrzcpd_indirect_pktmbuf +set_mrzcpd_direct_pktmbuf set_k3s_config ${mem_num} set_tsg_clixon_conf diff --git a/ansible/roles/tsg-os-HAL/tasks/main.yml b/ansible/roles/tsg-os-HAL/tasks/main.yml index 0c169425..bd9cef5f 100644 --- a/ansible/roles/tsg-os-HAL/tasks/main.yml +++ b/ansible/roles/tsg-os-HAL/tasks/main.yml @@ -7,6 +7,12 @@ - tsg-os-HAL.sh when: runtime_env == 'TSG-X-P0906' +- name: "tsg-os-HAL: install numactl" + yum: + name: numactl + conf_file: "{{ rpm_repo_config_path }}" + state: present + - name: "install tsg-os-HAL.service -- P0906" copy: src: "{{ role_path }}/files/service/{{ item.src }}" @@ -23,3 +29,12 @@ with_items: - tsg-os-HAL.service when: runtime_env == 'TSG-X-P0906' + +- name: "tsg-os-HAL.conf" + copy: + src: "{{ role_path }}/files/conf/{{ item }}" + dest: /etc/sysconfig/ + mode: 0755 + with_items: + - tsg-os-HAL.conf + when: runtime_env == 'TSG-X-P0906'