✨ feat(TSG-12762): TSG-OS自动构建虚拟硬盘

2022-12-22 19:31:14 +08:00
parent 51d771ba3c
commit 0eee6a54c4
10 changed files with 402 additions and 1 deletions
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -4,6 +4,10 @@ variables:
  BUILD_BASED_IMAGE_CENTOS7: "git.mesalab.cn:7443/mesa_platform/build-env:master"
  BUILD_BASED_IMAGE_ROCKYLINUX8: "git.mesalab.cn:7443/mesa_platform/build-env:rockylinux-dindind"

+cache:
+   paths:
+     - images
+
 .build_tsg-buildimage:
  script:
    - env | sort
@@ -14,6 +18,35 @@ variables:
  tags:
    - tsg-os-installer

+.build_tsg_vm_image:
+  cache:
+    key: $BUILD_OS_CACHE_KEY
+    paths:
+      - images/
+    policy: pull
+  script:
+    - yum install -y qemu-img qemu-kvm telnet net-tools procps-ng httpd
+    - find / -name images
+    - ls -lh
+    - export VM_IMG_NAME=$(basename -s .bin images/*.bin)
+    - echo $VM_IMG_NAME
+    - mv images/*.bin /var/www/html/onie-installer-x86_64.bin
+    - httpd
+    - chmod +x ./tools/mk-kvm.sh
+    - cp ./tools/onie-base-kvm.img images/$VM_IMG_NAME.img
+    - ./tools/mk-kvm.sh images/$VM_IMG_NAME.img
+    - qemu-img convert -O qcow2 -c images/$VM_IMG_NAME.img images/$VM_IMG_NAME-kvm.img
+    - rm -rf images/$VM_IMG_NAME.img
+    - ls -lh images
+    - chmod +x ./tools/mk-kvm-gns3a.sh
+    - ./tools/mk-kvm-gns3a.sh images/$VM_IMG_NAME-kvm.img images/$VM_IMG_NAME.gns3a
+    - cat images/$VM_IMG_NAME.gns3a
+    - ls -lh images
+    - chmod +x ./tools/upload.sh
+    - ./tools/upload.sh
+  tags:
+    - tsg-os-installer
+
 feature_branch_build_7400_mcn0:
  image: $BUILD_BASED_IMAGE_CENTOS7
  stage: build
@@ -86,6 +119,24 @@ feature_branch_build_TSGXP0906:
  variables:
    PROFILE_LIST: TSGXNXR620G40R01P0906
    DALIY_BUILD_VERSION: 1
+  cache:
+    key: TSGXP0906
+    paths:
+      - images/
+    policy: push
+  except:
+    - tags
+    - /^dev-.*$/i
+    - /^rel-.*$/i
+    - /^update-.*$/i
+
+feature_branch_build_kvm_image_TSGXP0906:
+  image: $BUILD_BASED_IMAGE_ROCKYLINUX8
+  stage: deploy
+  extends: .build_tsg_vm_image
+  needs: ["feature_branch_build_TSGXP0906"]
+  variables:
+    BUILD_OS_CACHE_KEY: TSGXP0906
  except:
    - tags
    - /^dev-.*$/i
@@ -201,6 +252,26 @@ develop_build_TSGXP0906:
    PULP3_FILE_DIST_NAME: tsg-os-images-develop
    DALIY_BUILD_VERSION: 1
    FILE_REPO_PATH: install/develop/tsg-os-images
+  cache:
+    key: TSGXP0906
+    paths:
+      - images/
+    policy: push
+  only:
+    - /^dev-.*$/i
+
+develop_build_kvm_image_TSGXP0906:
+  image: $BUILD_BASED_IMAGE_ROCKYLINUX8
+  stage: deploy
+  extends: .build_tsg_vm_image
+  needs: ["develop_build_TSGXP0906"]
+  variables:
+    BUILD_OS_CACHE_KEY: TSGXP0906
+    UPLOAD_TO_FILE_REPO: 1
+    PROFILE_LIST: KVM
+    PULP3_FILE_REPO_NAME: tsg-os-images-develop
+    PULP3_FILE_DIST_NAME: tsg-os-images-develop
+    FILE_REPO_PATH: install/develop/tsg-os-images
  only:
    - /^dev-.*$/i

@@ -315,6 +386,26 @@ testing_build_TSGXP0906:
    PULP3_FILE_DIST_NAME: tsg-os-images-testing
    FILE_REPO_PATH: install/testing/tsg-os-images
    DALIY_BUILD_VERSION: 1
+  cache:
+    key: TSGXP0906
+    paths:
+      - images/
+    policy: push
+  only:
+    - /^rel-.*$/i
+
+testing_build_kvm_image_TSGXP0906:
+  image: $BUILD_BASED_IMAGE_ROCKYLINUX8
+  stage: deploy
+  extends: .build_tsg_vm_image
+  needs: ["testing_build_TSGXP0906"]
+  variables:
+    BUILD_OS_CACHE_KEY: TSGXP0906
+    UPLOAD_TO_FILE_REPO: 1
+    PROFILE_LIST: KVM
+    PULP3_FILE_REPO_NAME: tsg-os-images-testing
+    PULP3_FILE_DIST_NAME: tsg-os-images-testing
+    FILE_REPO_PATH: install/testing/tsg-os-images
  only:
    - /^rel-.*$/i

@@ -429,6 +520,26 @@ rc_build_TSGXP0906:
    PULP3_FILE_REPO_NAME: tsg-os-images-rc
    PULP3_FILE_DIST_NAME: tsg-os-images-rc
    FILE_REPO_PATH: install/rc/tsg-os-images
+  cache:
+    key: TSGXP0906
+    paths:
+      - images/
+    policy: push
+  only:
+    - /^.*-rc.*$/i
+
+rc_build_kvm_image_TSGXP0906:
+  image: $BUILD_BASED_IMAGE_ROCKYLINUX8
+  stage: deploy
+  extends: .build_tsg_vm_image
+  needs: ["rc_build_TSGXP0906"]
+  variables:
+    BUILD_OS_CACHE_KEY: TSGXP0906
+    UPLOAD_TO_FILE_REPO: 1
+    PROFILE_LIST: KVM
+    PULP3_FILE_REPO_NAME: tsg-os-images-rc
+    PULP3_FILE_DIST_NAME: tsg-os-images-rc
+    FILE_REPO_PATH: install/rc/tsg-os-images
  only:
    - /^.*-rc.*$/i

@@ -553,6 +664,28 @@ release_build_TSGXP0906:
    PULP3_FILE_REPO_NAME: tsg-os-images-release
    PULP3_FILE_DIST_NAME: tsg-os-images-release
    FILE_REPO_PATH: install/release/tsg-os-images
+  cache:
+    key: TSGXP0906
+    paths:
+      - images/
+    policy: push
+  only:
+    - tags
+  except:
+    - /^.*-rc.*$/i
+
+release_build_kvm_image_TSGXP0906:
+  image: $BUILD_BASED_IMAGE_ROCKYLINUX8
+  stage: deploy
+  extends: .build_tsg_vm_image
+  needs: ["release_build_TSGXP0906"]
+  variables:
+    BUILD_OS_CACHE_KEY: TSGXP0906
+    UPLOAD_TO_FILE_REPO: 1
+    PROFILE_LIST: KVM
+    PULP3_FILE_REPO_NAME: tsg-os-images-release
+    PULP3_FILE_DIST_NAME: tsg-os-images-release
+    FILE_REPO_PATH: install/release/tsg-os-images
  only:
    - tags
  except:
--- a/ansible/HAL_deploy.yml
+++ b/ansible/HAL_deploy.yml
@@ -151,6 +151,7 @@
    - {role: coredump, tags: coredump}
    - {role: mrzcpd, tags: mrzcpd}
    - {role: clixon, tags: clixon}
+    - {role: tsg-os-HAL, tags: tsg-os-HAL}
    - {role: traffic-engine, tags: traffic-engine}
    - {role: k8s-dashboard, tags: k8s-dashboard}
    - {role: k3s-exporter, tags: k3s-exporter}
--- a/ansible/roles/tsg-os-HAL/files/script/tsg-os-HAL.sh
+++ b/ansible/roles/tsg-os-HAL/files/script/tsg-os-HAL.sh
@@ -0,0 +1,104 @@
+#!/bin/sh
+
+set -xe
+
+tsg_os_HAL_cfg_file="/etc/sysconfig/tsg-os-HAL"
+
+grub_cfg_file="/boot/grub/grub.cfg"
+mrzcpd_cfg_file="/etc/sysconfig/mrzcpd"
+tsg_clixon_cfg_file="/opt/tsg/clixon/etc/mgnt-srv.conf"
+k3s_cfg_file="/etc/rancher/k3s/config.yaml"
+mrglobal_cfg_template="/opt/tsg/clixon/etc/mrglobal.conf.jinja"
+
+cpu_core_num=$(lscpu | grep "CPU(s):" | head -n 1 | sed  -r 's/CPU\(s\):\s{1,}//g')
+mem_num=$(free -m | grep Mem | awk '{print $2}')
+
+HUGEPAGES=
+MRZCPD_IOCORE=
+MRZCPD_DIRECT_PKTMBUF=
+TSG_CLIXON_CPU_QUOTAS_MIN=
+
+load_tsg_os_HAL_config()
+{
+    [ -f ${tsg_os_HAL_cfg_file} ] || return 1
+    
+    source ${tsg_os_HAL_cfg_file}
+    return 0
+}
+
+set_resouce_by_HAL()
+{
+    local mem_num="$1"
+    local cpu_core_num="$2"
+
+    if [ $mem_num -le "32768" ]; then
+        HUGEPAGES=4
+        MRZCPD_DIRECT_PKTMBUF=262144
+    elif [ $mem_num -le "65536" ]; then
+        HUGEPAGES=8
+        MRZCPD_DIRECT_PKTMBUF=1048576
+    else
+        HUGEPAGES=16
+        MRZCPD_DIRECT_PKTMBUF=2097152
+    fi
+
+    if [ ${cpu_core_num} -le "4" ]; then
+        MRZCPD_IOCORE="1"
+        TSG_CLIXON_CPU_QUOTAS_MIN=2
+    elif [ ${cpu_core_num} -le "16" ]; then
+        MRZCPD_IOCORE="1,2"
+        TSG_CLIXON_CPU_QUOTAS_MIN=4
+    else
+        MRZCPD_IOCORE="1,2,3,4"
+        TSG_CLIXON_CPU_QUOTAS_MIN=9
+    fi
+}
+
+set_k3s_config()
+{
+    local mem_num="$1"
+
+    if [ $mem_num -le "32768" ]; then
+        kube_reserved="512Mi"
+        system_reserved="512Mi"
+        eviction_hard="64Mi"
+    elif [ $mem_num -le "65536" ]; then
+        kube_reserved="1Gi"
+        system_reserved="1Gi"
+        eviction_hard="1Gi"
+    else
+        kube_reserved="2Gi"
+        system_reserved="2Gi"
+        eviction_hard="2Gi"
+    fi
+
+cat <<EOF > ${k3s_cfg_file}
+kubelet-arg:
+  - "kube-reserved=memory=${kube_reserved}"
+  - "system-reserved=memory=${system_reserved}"
+  - "eviction-hard=memory.available<${eviction_hard}"
+EOF
+}
+
+set_tsg_clixon_conf()
+{
+    [ -f ${mrglobal_cfg_template} ] && sed -i "s/^sz_direct_pktmbuf=.*$/sz_direct_pktmbuf=${MRZCPD_DIRECT_PKTMBUF}/g" ${mrglobal_cfg_template}
+
+    [ -f ${mrglobal_cfg_template} ] && sed -i "s/^iocore=.*$/iocore=${MRZCPD_IOCORE}/g" ${mrglobal_cfg_template}
+
+    [ -f ${tsg_clixon_cfg_file} ] && sed -i "s/^cpu_range=.*$/cpu_range=${TSG_CLIXON_CPU_QUOTAS_MIN}/g" ${tsg_clixon_cfg_file}
+}
+
+set_os_hugepages()
+{
+    [ -f ${mrzcpd_cfg_file} ] && sed -i "s/HUGEPAGE_NUM_1G=\w*/HUGEPAGE_NUM_1G=${HUGEPAGES}/g" ${mrzcpd_cfg_file}
+
+    sysctl -w vm.nr_hugepages=${HUGEPAGES}
+    return 0
+}
+
+load_tsg_os_HAL_config || set_resouce_by_HAL ${mem_num} ${cpu_core_num}
+
+set_k3s_config ${mem_num}
+set_tsg_clixon_conf
+set_os_hugepages
--- a/ansible/roles/tsg-os-HAL/files/service/tsg-os-HAL.service.TSGXP0906
+++ b/ansible/roles/tsg-os-HAL/files/service/tsg-os-HAL.service.TSGXP0906
@@ -0,0 +1,11 @@
+[Unit]
+Description=Tsg os HAL
+Before=mrenv.service
+
+[Service]
+ExecStart=/bin/sh -c "/opt/tsg/tsg-os-HAL/scripts/tsg-os-HAL.sh"
+Type=oneshot
+RemainAfterExit=yes
+
+[Install]
+WantedBy=multi-user.target
--- a/ansible/roles/tsg-os-HAL/tasks/main.yml
+++ b/ansible/roles/tsg-os-HAL/tasks/main.yml
@@ -0,0 +1,25 @@
+- name: "tsg-os-HAL: copy shell file to dest"
+  copy:
+    src: "{{ role_path }}/files/script/{{ item }}"
+    dest: /opt/tsg/tsg-os-HAL/scripts/
+    mode: 0755
+  with_items:
+    - tsg-os-HAL.sh
+  when: runtime_env == 'TSG-X-P0906'
+
+- name: "install tsg-os-HAL.service -- P0906"
+  copy:
+    src: "{{ role_path }}/files/service/{{ item.src }}"
+    dest: /usr/lib/systemd/system/{{ item.dest }}
+    mode: 0644
+  with_items:
+    - { "src": tsg-os-HAL.service.TSGXP0906, "dest": tsg-os-HAL.service }
+  when: runtime_env == 'TSG-X-P0906'
+
+- name: "tsg-os-HAL.service: enable -- P0906"
+  systemd:
+    name: "{{ item }}"
+    enabled: yes
+  with_items:
+    - tsg-os-HAL.service
+  when: runtime_env == 'TSG-X-P0906'
--- a/make/Makefile.TSGXNXR620G40R01P0906
+++ b/make/Makefile.TSGXNXR620G40R01P0906
@@ -1,7 +1,7 @@

 PROFILE_ID 				:= TSG-X-NXR620G40-R01-P0906
 SUPPORTED_MACHINE_ID 	:= TSG-X-NXR620G40-R01-P0906
-KERNEL_ARGS 			:= crashkernel=512M default_hugepagesz=1G hugepagesz=1G hugepages=16 intel_iommu=on iommu=pt mitigations=off psi=1
+KERNEL_ARGS 			:= crashkernel=512M default_hugepagesz=1G intel_iommu=on iommu=pt mitigations=off psi=1
 GRUB_SERIAL_COMMAND 	:=
 SIZE_PART_SYSROOT		:= 16384M
 SIZE_PART_UPDATE		:= 16384M
--- a/tools/mk-kvm-gns3a.sh
+++ b/tools/mk-kvm-gns3a.sh
@@ -0,0 +1,59 @@
+#!/bin/sh
+
+set -x
+
+OS_PATH_NAME=$1
+GNS3_TEMPLATE_FILE=$2
+
+OS_NAME=$(basename ${OS_PATH_NAME})
+OS_MD5=$(md5sum ${OS_PATH_NAME} | awk '{print $1}')
+OS_SIZE=$(wc -c < ${OS_PATH_NAME})
+OS_VERSION=$(echo ${OS_NAME} | grep -Eo "[0-9]+\.[0-9]+")
+
+if [ -z "${OS_VERSION}" ]; then
+    OS_VERSION=1.1
+fi
+
+cat <<EOF > ${GNS3_TEMPLATE_FILE}
+{
+    "name": "TSG-X",
+    "category": "guest",
+    "description": "TSG-X 0906",
+    "vendor_name": "",
+    "vendor_url": "",
+    "documentation_url": "",
+    "product_name": "TSG-X",
+    "registry_version": 3,
+    "status": "stable",
+    "maintainer": "GNS3 Team",
+    "maintainer_email": "developers@gns3.net",
+    "qemu": {
+        "adapter_type": "virtio-net-pci",
+        "adapters": 4,
+        "ram": 32768,
+        "cpus": 4,
+        "hda_disk_interface": "virtio",
+        "arch": "x86_64",
+        "console_type": "vnc",
+        "kvm": "require",
+        "options": "-nographic -cpu host"
+    },
+    "images": [
+        {
+            "filename": "${OS_NAME}",
+            "version": "${OS_VERSION}",
+            "md5sum": "${OS_MD5}",
+            "filesize": ${OS_SIZE}
+        }
+    ],
+    "versions": [
+        {
+            "name": "${OS_VERSION}",
+            "images": {
+                "hda_disk_image": "${OS_NAME}"
+            }
+        }
+    ]
+}
+EOF
+
--- a/tools/mk-kvm.sh
+++ b/tools/mk-kvm.sh
@@ -0,0 +1,65 @@
+#!/bin/sh
+
+#  Copyright (C) 2014 Curt Brune <curt@cumulusnetworks.com>
+#
+#  SPDX-License-Identifier:     GPL-2.0
+set -xe
+
+MEM=16384
+DISK=$1
+
+# Path to ONIE installer .iso image
+CDROM="onie-uefi-kvm_x86_64.iso"
+
+# Path to OVMF firmware for qemu
+# Download OVMF from http://www.tianocore.org/ovmf/
+OVMF="OVMF.fd"
+
+# VM will listen on telnet port $KVM_PORT
+KVM_PORT=9000
+
+# VNC display to use
+VNC_PORT=0
+
+# set mode=disk to boot from hard disk
+mode=disk
+
+# set mode=cdrom to boot from the CDROM
+# mode=cdrom
+
+# set mode=net to boot from network adapters
+# mode=net
+
+# set firmware=uefi to boot with UEFI firmware, otherwise the system
+# will boot into legacy mode.
+# firmware=uefi
+
+boot=c
+if [ "$mode" = "cdrom" ] ; then
+    boot="order=cd,once=d"
+    cdrom="-cdrom $CDROM"
+elif [ "$mode" = "net" ] ; then
+    boot="order=cd,once=n,menu=on"
+fi
+
+if [ "$firmware" = "uefi" ] ; then
+    [ -r "$OVMF" ] || {
+        echo "ERROR:  Cannot find the OVMF firmware for UEFI: $OVMF"
+        echo "Please make sure to install the OVMF.fd in the expected directory"
+        exit 1
+    }
+    bios="-bios $OVMF"
+fi
+
+/usr/libexec/qemu-kvm -m $MEM \
+    -name "onie" \
+    -smp 4,cores=2,threads=1,sockets=2 \
+    $bios \
+    -boot $boot $cdrom \
+    -device e1000,netdev=onienet \
+    -netdev user,id=onienet,hostfwd=:0.0.0.0:3040-:22 \
+    -nographic \
+    -no-reboot \
+    -drive file=$DISK,media=disk,if=virtio,index=0
+
+exit 0
--- a/tools/onie-base-kvm.img
+++ b/tools/onie-base-kvm.img
--- a/tools/upload.sh
+++ b/tools/upload.sh
@@ -8,6 +8,9 @@ if [ -n "${UPLOAD_TO_FILE_REPO}" ]; then
 	#python3 ~/file_upload_tools.py ${PULP3_FILE_REPO_NAME} ${PULP3_FILE_DIST_NAME} ./images/*.sha256sum.txt
 	if [ "${PROFILE_LIST}" == "SERVER" ]; then
 		python3 ~/file_upload_tools_to_share_repo.py ${FILE_REPO_PATH} ./images/*.tar
+	elif [ "${PROFILE_LIST}" == "KVM" ]; then
+		python3 ~/file_upload_tools_to_share_repo.py ${FILE_REPO_PATH} ./images/*.img
+		python3 ~/file_upload_tools_to_share_repo.py ${FILE_REPO_PATH} ./images/*.gns3a
 	else
 		python3 ~/file_upload_tools_to_share_repo.py ${FILE_REPO_PATH} ./images/*.bin
 	fi