From 7fd4db21b95b9bb30c8057907c7bcf14522e7843 Mon Sep 17 00:00:00 2001 From: linxin Date: Fri, 1 Mar 2024 18:41:55 +0800 Subject: [PATCH] =?UTF-8?q?=F0=9F=90=9E=20fix:TSG-18539=20add=20dynamic-ho?= =?UTF-8?q?st-port?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../k3s-install/files/dynamic-hostport.yaml | 96 +++++++++++++++++++ ansible/roles/k3s-install/tasks/main.yml | 14 ++- .../helm/templates/deployment-shaping.yaml | 24 ++--- .../files/helm/templates/service.yaml | 19 ---- 4 files changed, 121 insertions(+), 32 deletions(-) create mode 100644 ansible/roles/k3s-install/files/dynamic-hostport.yaml delete mode 100644 ansible/roles/traffic-engine/files/helm/templates/service.yaml diff --git a/ansible/roles/k3s-install/files/dynamic-hostport.yaml b/ansible/roles/k3s-install/files/dynamic-hostport.yaml new file mode 100644 index 00000000..a85da272 --- /dev/null +++ b/ansible/roles/k3s-install/files/dynamic-hostport.yaml @@ -0,0 +1,96 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: dynamic-hostports-account + namespace: tsg-os-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: dynamic-hostports-account-nodes +rules: + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: dynamic-hostports-account-pods +rules: +- apiGroups: [""] + resources: ["pods"] + verbs: ["list","watch","patch"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: dynamic-hostports-account-services +rules: +- apiGroups: [""] + resources: ["endpoints", "services"] + verbs: ["list","create","delete"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: dynamic-hostports-account-binding-nodes +subjects: + - kind: ServiceAccount + namespace: tsg-os-system + name: dynamic-hostports-account + apiGroup: "" +roleRef: + kind: ClusterRole + name: dynamic-hostports-account-nodes + apiGroup: "" +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: dynamic-hostports-account-binding-pods +subjects: + - kind: ServiceAccount + namespace: tsg-os-system + name: dynamic-hostports-account + apiGroup: "" +roleRef: + kind: ClusterRole + name: dynamic-hostports-account-pods + apiGroup: "" +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: dynamic-hostports-account-binding-services +subjects: +- kind: ServiceAccount + namespace: tsg-os-system + name: dynamic-hostports-account + apiGroup: "" +roleRef: + kind: ClusterRole + name: dynamic-hostports-account-services + apiGroup: "" +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: dynamic-hostports-deployment + namespace: tsg-os-system +spec: + replicas: 1 + selector: + matchLabels: + app: dynamic-hostports-app + template: + metadata: + labels: + app: dynamic-hostports-app + spec: + serviceAccountName: dynamic-hostports-account + containers: + - name: dynamic-hostports-container + image: 0blu/dynamic-hostport-manager:latest + imagePullPolicy: Never + restartPolicy: Always diff --git a/ansible/roles/k3s-install/tasks/main.yml b/ansible/roles/k3s-install/tasks/main.yml index a6461ae2..df41a7a2 100644 --- a/ansible/roles/k3s-install/tasks/main.yml +++ b/ansible/roles/k3s-install/tasks/main.yml @@ -130,4 +130,16 @@ - name: "copy mrzcpd-deviceplugin file to dest" copy: src: "{{ role_path }}/files/daemonset-mrzcpd-deviceplugin.yaml" - dest: /var/lib/rancher/k3s/server/manifests/ \ No newline at end of file + dest: /var/lib/rancher/k3s/server/manifests/ + +- name: "copy dynamic-hostport yaml file to k3s manifests directory" + copy: + src: "{{ role_path }}/files/dynamic-hostport.yaml" + dest: /var/lib/rancher/k3s/server/manifests/ + +- name: "Get dynamic-hostport chart images to dest" + get_url: + url: https://repo.geedge.net/filerepo/install/release/tsg-container-images/dynamic-host-port.tar + dest: /var/lib/rancher/k3s/agent/images/dynamic-host-port.tar + url_username: "{{ lookup('env', 'PULP_REPO_USERNAME') }}" + url_password: "{{ lookup('env', 'PULP_REPO_PASSWORD') }}" \ No newline at end of file diff --git a/ansible/roles/traffic-engine/files/helm/templates/deployment-shaping.yaml b/ansible/roles/traffic-engine/files/helm/templates/deployment-shaping.yaml index d78587f7..a7403dda 100644 --- a/ansible/roles/traffic-engine/files/helm/templates/deployment-shaping.yaml +++ b/ansible/roles/traffic-engine/files/helm/templates/deployment-shaping.yaml @@ -23,6 +23,7 @@ spec: vsysId: "{{ .Values.vsys_id }}" serviceFunction: {{ .Release.Name }} component: shaping + dynamic-hostports: '8551.8552' annotations: prometheus.io/port: "9007" prometheus.io/scrape: "true" @@ -51,9 +52,12 @@ spec: {{- include "traffic-engine.global.cm.read-server-ip" . | nindent 12 }} sed -Ei -c "s|CM_POLICY_LOCAL_CACHE_IP_LOCATION|${CM_POLICY_LOCAL_CACHE_IP?}|g" /opt/tsg/shaping_engine/conf/shaping.conf {{- end }} - curl --cacert ${CACERT} --header "Authorization: Bearer ${TOKEN}" -X GET ${APISERVER}/api/v1/namespaces/${NAMESPACE}/services/shaping-announce-port-{{ .Release.Name }} -o /tmp/service.txt - export CLUSTER_ANNOUNCE_PORT=$(cat /tmp/service.txt | jq '.spec.ports[] | select(.name=="cluster-announce-port") | .nodePort') - export HEALTH_CHECK_ANNOUNCE_PORT=$(cat /tmp/service.txt | jq '.spec.ports[] | select(.name=="healthcheck-announce-port") | .nodePort') + until nslookup ${MY_POD_NAME}-8551.default.svc; do echo waiting for kubernetes service; sleep 2; done + curl --cacert ${CACERT} --header "Authorization: Bearer ${TOKEN}" -X GET ${APISERVER}/api/v1/namespaces/${NAMESPACE}/services/${MY_POD_NAME}-8551 -o /tmp/service.txt + export CLUSTER_ANNOUNCE_PORT=$(cat /tmp/service.txt | jq '.spec.ports[] | .nodePort') + until nslookup ${MY_POD_NAME}-8552.default.svc; do echo waiting for kubernetes service; sleep 2; done + curl --cacert ${CACERT} --header "Authorization: Bearer ${TOKEN}" -X GET ${APISERVER}/api/v1/namespaces/${NAMESPACE}/services/${MY_POD_NAME}-8552 -o /tmp/service.txt + export HEALTH_CHECK_ANNOUNCE_PORT=$(cat /tmp/service.txt | jq '.spec.ports[] | .nodePort') echo "export CLUSTER_ANNOUNCE_PORT=${CLUSTER_ANNOUNCE_PORT}" > /etc/profile.d/announceinfo.sh echo "export HEALTH_CHECK_ANNOUNCE_PORT=${HEALTH_CHECK_ANNOUNCE_PORT}" >> /etc/profile.d/announceinfo.sh chmod 0755 /etc/profile.d/announceinfo.sh @@ -80,6 +84,10 @@ spec: - containerPort: 8552 - containerPort: 9007 env: + - name: MY_POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name - name: DEPLOYMENT_NAME value: {{ .Release.Name }}-shaping - name: NODE_IP @@ -182,14 +190,6 @@ spec: - | until nslookup kubernetes.default.svc; do echo waiting for kubernetes service; sleep 2; done - - name: init-announce-svc - image: "registry.gdnt-cloud.website/tsg-init:{{ .Chart.AppVersion }}" - imagePullPolicy: Never - command: - - "bash" - - "-ec" - - | - until nslookup shaping-announce-port-{{ .Release.Name }}.default.svc; do echo waiting for kubernetes service; sleep 2; done {{- if eq .Values.external_resources.cm.connectivity "local_cache" }} - name: init-cm-svc @@ -282,4 +282,4 @@ spec: hostPath: path: / {{- end }} -{{- end }} \ No newline at end of file +{{- end }} diff --git a/ansible/roles/traffic-engine/files/helm/templates/service.yaml b/ansible/roles/traffic-engine/files/helm/templates/service.yaml deleted file mode 100644 index 0c7f56de..00000000 --- a/ansible/roles/traffic-engine/files/helm/templates/service.yaml +++ /dev/null @@ -1,19 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - labels: - app: shaping-announce-port-{{ .Release.Name }} - name: shaping-announce-port-{{ .Release.Name }} - namespace: {{ .Release.Namespace }} - -spec: - ports: - - name: cluster-announce-port - port: 8551 - targetPort: 8551 - - name: healthcheck-announce-port - port: 8552 - targetPort: 8552 - selector: - app: {{ .Release.Name }}-shaping - type: NodePort