diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 7416014..413b5ff 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,108 +1,118 @@ variables: - BUILD_BASED_IMAGE_ROCKYLINUX8: "git.mesalab.cn:7443/mesa_platform/build-env:rockylinux" + JOB_IMAGE_ROCKYLINUX8_X86_64: "git.mesalab.cn:7443/mesa_platform/build-env:rockylinux-build-os" + JOB_IMAGE_ROCKYLINUX9_AARCH64: "git.mesalab.cn:7443/mesa_platform/build-env:rocky9-aarch64" -stages: -- prebuild -- build-rockylinux8 - -cache: - key: "$CI_PROJECT_NAME-cache" - paths: - - images - -.build_docker_images: - image: docker:stable - before_script: - - docker info - - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY - - chmod 0755 ./images_build/client/dign_client/bin/wait-for - - chmod 0755 ./images_build/server_web/certs/tool - tags: - - share - -.build_rpm: +.build_scripts: variables: GIT_STRATEGY: "clone" BUILD_PADDING_PREFIX: /tmp/padding_for_CPACK_RPM_BUILD_SOURCE_DIRS_PREFIX_PREFIX_PREFIX_PREFIX_PREFIX_PREFIX/ - TESTING_VERSION_BUILD: 0 + CLIENT_IMAGE_TAG: registry.gdnt-cloud.website/tsg/diagnose/client:latest + SERVER_WEB_IMAGE_TAG: registry.gdnt-cloud.website/tsg/diagnose/server-web:latest + SERVER_DNS_IMAGE_TAG: registry.gdnt-cloud.website/tsg/diagnose/server-dns:latest before_script: + - mkdir -p images;rm -rf images/* - mkdir -p $BUILD_PADDING_PREFIX/$CI_PROJECT_NAMESPACE/ - ln -s $CI_PROJECT_DIR $BUILD_PADDING_PREFIX/$CI_PROJECT_PATH - - cd $BUILD_PADDING_PREFIX/$CI_PROJECT_PATH + - chmod 0755 ./images_build/client/dign_client/bin/wait-for + - chmod 0755 ./images_build/server_web/certs/tool - chmod 0755 ./scripts/tsg-diagnose-oneshot - chmod 0755 ./scripts/tsg-diagnose-periodical - chmod 0755 ./scripts/tsg-diagnose.sh - chmod 0755 ./etc/certs_import/certs/tool - - chmod +x ./ci/travis.sh - tags: - - share - -images_build: - stage: prebuild - extends: .build_docker_images - variables: - CLIENT_IMAGE_TAG: dign-client:latest - SERVER_WEB_IMAGE_TAG: dign-server-web:latest - SERVER_DNS_IMAGE_TAG: dign-server-dns:latest + - chmod 0755 ./ci/travis.sh + - buildah login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY script: - - mkdir -p images - - rm -rf images/* - - docker build -t $CLIENT_IMAGE_TAG -f ./images_build/client/Dockerfile ./images_build/client/ - - docker save $CLIENT_IMAGE_TAG > images/tsg_diagnose_client.tar - - docker build -t $SERVER_WEB_IMAGE_TAG -f ./images_build/server_web/Dockerfile ./images_build/server_web/ - - docker save $SERVER_WEB_IMAGE_TAG > images/tsg_diagnose_server_web.tar - - docker build -t $SERVER_DNS_IMAGE_TAG -f ./images_build/server_dns/Dockerfile ./images_build/server_dns/ - - docker save $SERVER_DNS_IMAGE_TAG > images/tsg_diagnose_server_dns.tar + - buildah build -t $CLIENT_IMAGE_TAG -f ./images_build/client/Dockerfile ./images_build/client/ + - podman save -o images/tsg_diagnose_client.tar -m $CLIENT_IMAGE_TAG + - buildah build -t $SERVER_WEB_IMAGE_TAG -f ./images_build/server_web/Dockerfile ./images_build/server_web/ + - podman save -o images/tsg_diagnose_server_web.tar -m $SERVER_WEB_IMAGE_TAG + - buildah build -t $SERVER_DNS_IMAGE_TAG -f ./images_build/server_dns/Dockerfile ./images_build/server_dns/ + - podman save -o images/tsg_diagnose_server_dns.tar -m $SERVER_DNS_IMAGE_TAG + - cd $BUILD_PADDING_PREFIX/$CI_PROJECT_PATH + - ./ci/travis.sh -feature_branch_build_rockylinux8: - image: $BUILD_BASED_IMAGE_ROCKYLINUX8 - stage: build-rockylinux8 +feature_branch_build_rockylinux8_x86_64: + image: $JOB_IMAGE_ROCKYLINUX8_X86_64 variables: PACKAGE: 1 TESTING_VERSION_BUILD: 1 - extends: .build_rpm - script: - - ls -halt ./scripts - - ./ci/travis.sh - dependencies: - - images_build + extends: .build_scripts except: - tags - /^dev-.*$/i - /^rel-.*$/i - /^master.*$/i + tags: + - tsg-os-builder-el8 -develop_build_rockylinux8: - image: $BUILD_BASED_IMAGE_ROCKYLINUX8 - stage: build-rockylinux8 +feature_branch_build_rockylinux9_aarch64: + image: $JOB_IMAGE_ROCKYLINUX9_AARCH64 variables: PACKAGE: 1 - UPLOAD: 1 TESTING_VERSION_BUILD: 1 + extends: .build_scripts + except: + - tags + - /^dev-.*$/i + - /^rel-.*$/i + - /^master.*$/i + tags: + - tsg-os-builder-aarch64 + +develop_build_rockylinux8_x86_64: + image: $JOB_IMAGE_ROCKYLINUX8_X86_64 + variables: + PACKAGE: 1 + TESTING_VERSION_BUILD: 1 + UPLOAD: 1 PULP3_RPM_REPO_NAME: tsg-testing-x86_64.el8 PULP3_RPM_DIST_NAME: tsg-testing-x86_64.el8 - extends: .build_rpm - script: - - ./ci/travis.sh - dependencies: - - images_build + extends: .build_scripts only: - /^dev-.*$/i - /^rel-.*$/i - /^master.*$/i + tags: + - tsg-os-builder-el8 -release_build_rockylinux8: - image: $BUILD_BASED_IMAGE_ROCKYLINUX8 - stage: build-rockylinux8 +develop_build_rockylinux9_aarch64: + image: $JOB_IMAGE_ROCKYLINUX9_AARCH64 + variables: + PACKAGE: 1 + TESTING_VERSION_BUILD: 1 + UPLOAD: 1 + PULP3_RPM_REPO_NAME: tsg-testing-aarch64.el9 + PULP3_RPM_DIST_NAME: tsg-testing-aarch64.el9 + extends: .build_scripts + only: + - /^dev-.*$/i + - /^rel-.*$/i + - /^master.*$/i + tags: + - tsg-os-builder-aarch64 + +release_build_rockylinux8_x86_64: + image: $JOB_IMAGE_ROCKYLINUX8_X86_64 variables: PACKAGE: 1 UPLOAD: 1 PULP3_RPM_REPO_NAME: tsg-stable-x86_64.el8 PULP3_RPM_DIST_NAME: tsg-stable-x86_64.el8 - extends: .build_rpm - script: - - ./ci/travis.sh - dependencies: - - images_build + extends: .build_scripts only: - - tags \ No newline at end of file + - tags + tags: + - tsg-os-builder-el8 + +release_build_rockylinux9_aarch64: + image: $JOB_IMAGE_ROCKYLINUX9_AARCH64 + variables: + PACKAGE: 1 + UPLOAD: 1 + PULP3_RPM_REPO_NAME: tsg-stable-aarch64.el9 + PULP3_RPM_DIST_NAME: tsg-stable-aarch64.el9 + extends: .build_scripts + only: + - tags + tags: + - tsg-os-builder-aarch64 diff --git a/compose/docker-compose.yml b/compose/docker-compose.yml index 3b81c83..16bac37 100644 --- a/compose/docker-compose.yml +++ b/compose/docker-compose.yml @@ -3,7 +3,7 @@ version: '2.2' services: server_web: - image: "dign-server-web:latest" + image: "registry.gdnt-cloud.website/tsg/diagnose/server-web:latest" container_name: "dign-server-web" tty: true privileged: true @@ -26,7 +26,7 @@ services: server_dns: - image: "dign-server-dns:latest" + image: "registry.gdnt-cloud.website/tsg/diagnose/server-dns:latest" container_name: "dign-server-dns" tty: true privileged: true @@ -50,7 +50,7 @@ services: tail -f /dev/null client: - image: "dign-client:latest" + image: "registry.gdnt-cloud.website/tsg/diagnose/client:latest" container_name: "dign-client" depends_on: - server_web diff --git a/images_build/client/Dockerfile b/images_build/client/Dockerfile index e42b6b7..7965475 100644 --- a/images_build/client/Dockerfile +++ b/images_build/client/Dockerfile @@ -1,20 +1,19 @@ -# Start python3.6.9-alpine3.10 to build python:unitest -FROM python:3.6.9-alpine3.10 - +FROM python:3.9-alpine ADD dign_client /opt/dign_client RUN sed -i s@/dl-cdn.alpinelinux.org/@/mirrors.ustc.edu.cn/@g /etc/apk/repositories \ && apk update \ && apk add curl-dev gcc libc-dev curl gzip libpcap-dev\ - && pip3 install pycurl \ - && pip3 install httpstat \ - && pip3 install CIUnitTest \ - && pip3 install pytelegraf \ - && pip3 install dnspython \ - && pip3 install prettytable \ - && pip3 install pyyaml \ - && pip3 install scapy \ + && python3 -m pip config set global.index-url https://pypi.tuna.tsinghua.edu.cn/simple \ + && python3 -m pip install pycurl \ + && python3 -m pip install httpstat \ + && python3 -m pip install CIUnitTest \ + && python3 -m pip install pytelegraf \ + && python3 -m pip install dnspython \ + && python3 -m pip install prettytable \ + && python3 -m pip install pyyaml \ + && python3 -m pip install scapy \ && mv /opt/dign_client/etc/client.conf /opt/dign_client/etc/client.conf.sample WORKDIR /opt/dign_client diff --git a/images_build/client/dign_client/bin/client.py b/images_build/client/dign_client/bin/client.py index a214eb8..413ab82 100644 --- a/images_build/client/dign_client/bin/client.py +++ b/images_build/client/dign_client/bin/client.py @@ -18,7 +18,7 @@ import dns.resolver import sys import logging import copy -from prettytable import PrettyTable,NONE,HEADER +from prettytable import PrettyTable,VRuleStyle import yaml from urllib.parse import urlparse @@ -422,7 +422,7 @@ class DNSQueryBuilder: def _query(self, record_type): try: self._setup() - self._dns_answer = self._dns_resolver.query(self._domain, record_type) + self._dns_answer = self._dns_resolver.resolve(self._domain, record_type) except Exception as error_info: self._error_info = error_info @@ -1261,7 +1261,7 @@ class ResultExportBuilder: def _create_exporter(self): self._exporter = PrettyTable() - self._exporter.vrules = NONE + self._exporter.vrules = VRuleStyle.NONE self._exporter.field_names = [self.COLUMN_0, self.COLUMN_1, self.COLUMN_2] self._exporter.align[self.COLUMN_0] = "l" self._exporter.align[self.COLUMN_1] = "l" diff --git a/images_build/server_dns/Dockerfile b/images_build/server_dns/Dockerfile index 2bbea34..bfc7cea 100644 --- a/images_build/server_dns/Dockerfile +++ b/images_build/server_dns/Dockerfile @@ -1,19 +1,29 @@ FROM alpine RUN set -eux && sed -i 's/dl-cdn.alpinelinux.org/mirrors.tuna.tsinghua.edu.cn/g' /etc/apk/repositories LABEL maintainer="TSG" -ADD webproc_linux_amd64.gz /tmp/ + +COPY webproc_0.4.0_linux_amd64.gz /tmp/ +COPY webproc_0.4.0_linux_arm64.gz /tmp/ # fetch dnsmasq and webproc binary RUN apk update \ && apk upgrade alpine-keys \ && apk --no-cache add dnsmasq \ && apk add --no-cache --virtual .build-deps curl \ - && gzip -cd /tmp/webproc_linux_amd64.gz > /usr/local/bin/webproc \ - && rm /tmp/webproc_linux_amd64.gz \ - && chmod +x /usr/local/bin/webproc \ && apk del .build-deps + +RUN ARCH=$(uname -m) && \ + if [ "$ARCH" = "x86_64" ]; then \ + gzip -cd /tmp/webproc_0.4.0_linux_amd64.gz > /usr/local/bin/webproc; \ + elif [ "$ARCH" = "aarch64" ]; then \ + gzip -cd /tmp/webproc_0.4.0_linux_arm64.gz > /usr/local/bin/webproc; \ + fi \ + && chmod +x /usr/local/bin/webproc \ + && rm -rf /tmp/webproc_0.4.0_linux_amd64.gz \ + && rm -rf /tmp/webproc_0.4.0_linux_arm64.gz + #configure dnsmasq RUN mkdir -p /etc/default/ RUN echo -e "ENABLED=1\nIGNORE_RESOLVCONF=yes" > /etc/default/dnsmasq COPY dnsmasq.conf /etc/dnsmasq.conf #run! -ENTRYPOINT ["webproc","--config","/etc/dnsmasq.conf","--","dnsmasq","--no-daemon"] +ENTRYPOINT ["webproc","-c","/etc/dnsmasq.conf","--","dnsmasq","--no-daemon"] diff --git a/images_build/server_dns/webproc_0.4.0_linux_amd64.gz b/images_build/server_dns/webproc_0.4.0_linux_amd64.gz new file mode 100644 index 0000000..b3269e2 Binary files /dev/null and b/images_build/server_dns/webproc_0.4.0_linux_amd64.gz differ diff --git a/images_build/server_dns/webproc_0.4.0_linux_arm64.gz b/images_build/server_dns/webproc_0.4.0_linux_arm64.gz new file mode 100644 index 0000000..086d799 Binary files /dev/null and b/images_build/server_dns/webproc_0.4.0_linux_arm64.gz differ diff --git a/images_build/server_dns/webproc_linux_amd64.gz b/images_build/server_dns/webproc_linux_amd64.gz deleted file mode 100644 index ea2c478..0000000 Binary files a/images_build/server_dns/webproc_linux_amd64.gz and /dev/null differ diff --git a/images_build/server_web/Dockerfile b/images_build/server_web/Dockerfile index 7758112..334fbe8 100644 --- a/images_build/server_web/Dockerfile +++ b/images_build/server_web/Dockerfile @@ -14,15 +14,17 @@ RUN sed -i s@/archive.ubuntu.com/@/mirrors.tuna.tsinghua.edu.cn/@g /etc/apt/sour libffi-dev \ make \ nginx \ - ruby2.6 \ - ruby2.6-dev \ + ruby \ + ruby-dev \ net-tools \ iputils-ping \ faketime \ && gem update --system 3.1.6 \ && gem install ffi --version 1.15.5 \ - && gem install public_suffix --version 5.0.5 \ - && gem install jekyll --version 4.2.2 + && gem install public_suffix --version 4.0.7 \ + && gem install rb-inotify --version 0.10.1 \ + && gem install listen --version 3.5.0 \ + && gem install jekyll --version 3.6.0 # Install badssl.com ADD . badssl.com diff --git a/k8s-resource/tsg-diagnose-client.yaml b/k8s-resource/tsg-diagnose-client.yaml index f457c78..b93d156 100644 --- a/k8s-resource/tsg-diagnose-client.yaml +++ b/k8s-resource/tsg-diagnose-client.yaml @@ -43,7 +43,7 @@ spec: spec: containers: - name: dign-client - image: "dign-client:latest" + image: "registry.gdnt-cloud.website/tsg/diagnose/client:latest" imagePullPolicy: Never workingDir: /opt/dign_client command: ["/bin/sh", "-c", "update-ca-certificates; tail -f /dev/null"] @@ -79,6 +79,6 @@ spec: path: /opt/tsg/tsg-diagnose/etc - name: share-path hostPath: - path: /opt/tsg/tsg-diagnose/share + path: /opt/tsg/clixon/share --- diff --git a/k8s-resource/tsg-diagnose-server.yaml b/k8s-resource/tsg-diagnose-server.yaml index f65d1d6..df285bd 100644 --- a/k8s-resource/tsg-diagnose-server.yaml +++ b/k8s-resource/tsg-diagnose-server.yaml @@ -44,9 +44,9 @@ spec: spec: containers: - name: dign-server-dns - image: "dign-server-dns:latest" + image: "registry.gdnt-cloud.website/tsg/diagnose/server-dns:latest" imagePullPolicy: Never - command: ["/bin/sh", "-c", "echo 66.66.66.66 www.1testanswer-cname.com >> /etc/hosts; webproc --config /etc/dnsmasq.conf -- dnsmasq --no-daemon"] + command: ["/bin/sh", "-c", "echo 66.66.66.66 www.1testanswer-cname.com >> /etc/hosts; webproc -c /etc/dnsmasq.conf -- dnsmasq --no-daemon"] securityContext: privileged: true livenessProbe: @@ -66,7 +66,7 @@ spec: readOnly: true - name: dign-server-web - image: "dign-server-web:latest" + image: "registry.gdnt-cloud.website/tsg/diagnose/server-web:latest" imagePullPolicy: Never command: ["/bin/bash", "-c", "make inside-docker;nginx;tail -f /dev/null"] securityContext: @@ -91,7 +91,7 @@ spec: initContainers: - name: tsg-dign-init - image: "dign-server-dns:latest" + image: "registry.gdnt-cloud.website/tsg/diagnose/server-dns:latest" imagePullPolicy: Never command: ["/bin/sh","-c","i=1;while [ \"$i\" -le 64 ]; do ip_suffix=$(( i + 100 ));ifconfig net1:$i 192.0.2.$ip_suffix;i=$(( i + 1 ));done;"] securityContext: