feature:TSG-12528:新增traffic-engine监控部署

2022-11-01 18:08:07 +08:00
parent 0bbf0a84ea
commit e48400e313
3 changed files with 66 additions and 186 deletions
--- a/ansible/roles/k3s-exporter/files/prometheus/prometheus-crd.yaml
+++ b/ansible/roles/k3s-exporter/files/prometheus/prometheus-crd.yaml
@@ -14,6 +14,8 @@ spec:
        servicePort: 8080
        nodePort: 30080
        type: NodePort
+      global:
+        scrape_interval: 30s
    configmapReload:
      prometheus:
        image:
@@ -38,32 +40,10 @@ spec:
          tsg/monitor: "true"


-
    serverFiles:
      prometheus.yml:
        scrape_configs:
-          - job_name: prometheus
-            static_configs:
-              - targets:
-                - localhost:9090
-
-          - job_name: 'kubernetes-apiservers'
-
-            kubernetes_sd_configs:
-              - role: endpoints
-            scheme: https
-            tls_config:
-              ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
-              insecure_skip_verify: true
-            bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
-
-            relabel_configs:
-              - source_labels: [__meta_kubernetes_namespace, __meta_kubernetes_service_name, __meta_kubernetes_endpoint_port_name]
-                action: keep
-                regex: default;kubernetes;https
-
-
-          - job_name: 'kubernetes-nodes'
+          - job_name: 'cadvisor'
            scheme: https
            tls_config:
              ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
@@ -74,36 +54,15 @@ spec:
              - role: node

            relabel_configs:
-              - action: labelmap
-                regex: __meta_kubernetes_node_label_(.+)
-              - target_label: __address__
-                replacement: kubernetes.default.svc:443
-              - source_labels: [__meta_kubernetes_node_name]
-                regex: (.+)
-                target_label: __metrics_path__
-                replacement: /api/v1/nodes/$1/proxy/metrics
-
-
-          - job_name: 'kubernetes-nodes-cadvisor'
-            scheme: https
-            tls_config:
-              ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
-              insecure_skip_verify: true
-            bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
-
-            kubernetes_sd_configs:
-              - role: node
-
-            relabel_configs:
-              - action: labelmap
-                regex: __meta_kubernetes_node_label_(.+)
              - target_label: __address__
                replacement: kubernetes.default.svc:443
              - source_labels: [__meta_kubernetes_node_name]
                regex: (.+)
                target_label: __metrics_path__
                replacement: /api/v1/nodes/$1/proxy/metrics/cadvisor
-
+            metric_relabel_configs:
+              - regex:  instance
+                action: labeldrop

          - job_name: 'exporter'
            honor_labels: true
@@ -134,90 +93,12 @@ spec:
                target_label: __address__
                regex: (.+?)(?::\d+)?;(\d+)
                replacement: $1:$2
-              - action: labelmap
-                regex: __meta_kubernetes_service_annotation_prometheus_io_param_(.+)
-                replacement: __param_$1
-              - action: labelmap
-                regex: __meta_kubernetes_service_label_(.+)
-              - source_labels: [__meta_kubernetes_namespace]
-                action: replace
-                target_label: namespace
              - source_labels: [__meta_kubernetes_service_name]
                action: replace
                target_label: service
-              - source_labels: [__meta_kubernetes_pod_node_name]
-                action: replace
-                target_label: node
-
-
-          - job_name: 'kubernetes-service-endpoints-slow'
-            honor_labels: true
-
-            scrape_interval: 5m
-            scrape_timeout: 30s
-
-            kubernetes_sd_configs:
-              - role: endpoints
-
-            relabel_configs:
-              - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scrape_slow]
-                action: keep
-                regex: true
-              - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scheme]
-                action: replace
-                target_label: __scheme__
-                regex: (https?)
-              - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_path]
-                action: replace
-                target_label: __metrics_path__
-                regex: (.+)
-              - source_labels: [__address__, __meta_kubernetes_service_annotation_prometheus_io_port]
-                action: replace
-                target_label: __address__
-                regex: (.+?)(?::\d+)?;(\d+)
-                replacement: $1:$2
-              - action: labelmap
-                regex: __meta_kubernetes_service_annotation_prometheus_io_param_(.+)
-                replacement: __param_$1
-              - action: labelmap
-                regex: __meta_kubernetes_service_label_(.+)
-              - source_labels: [__meta_kubernetes_namespace]
-                action: replace
-                target_label: namespace
-              - source_labels: [__meta_kubernetes_service_name]
-                action: replace
-                target_label: service
-              - source_labels: [__meta_kubernetes_pod_node_name]
-                action: replace
-                target_label: node
-
-
-          - job_name: 'kubernetes-services'
-            honor_labels: true
-
-            metrics_path: /probe
-            params:
-              module: [http_2xx]
-
-            kubernetes_sd_configs:
-              - role: service
-
-            relabel_configs:
-              - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_probe]
-                action: keep
-                regex: true
-              - source_labels: [__address__]
-                target_label: __param_target
-              - target_label: __address__
-                replacement: blackbox
-              - source_labels: [__param_target]
-                target_label: instance
-              - action: labelmap
-                regex: __meta_kubernetes_service_label_(.+)
-              - source_labels: [__meta_kubernetes_namespace]
-                target_label: namespace
-              - source_labels: [__meta_kubernetes_service_name]
-                target_label: service
+            metric_relabel_configs:
+              - regex:  instance
+                action: labeldrop


          - job_name: 'traffic-engine'
@@ -236,73 +117,29 @@ spec:
              - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape_slow]
                action: drop
                regex: true
-              - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scheme]
-                action: replace
-                regex: (https?)
-                target_label: __scheme__
-              - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_path]
-                action: replace
-                target_label: __metrics_path__
-                regex: (.+)
              - source_labels: [__address__, __meta_kubernetes_pod_annotation_prometheus_io_port]
                action: replace
                regex: (.+?)(?::\d+)?;(\d+)
                replacement: $1:$2
                target_label: __address__
-              - action: labelmap
-                regex: __meta_kubernetes_pod_annotation_prometheus_io_param_(.+)
-                replacement: __param_$1
-              - action: labelmap
-                regex: __meta_kubernetes_pod_label_(.+)
-              - source_labels: [__meta_kubernetes_namespace]
-                action: replace
-                target_label: namespace
              - source_labels: [__meta_kubernetes_pod_name]
                action: replace
-                target_label: pod
-              - source_labels: [__meta_kubernetes_pod_phase]
-                regex: Pending|Succeeded|Failed|Completed
-                action: drop
-
-
-          - job_name: 'kubernetes-pods-slow'
-            honor_labels: true
-
-            scrape_interval: 5m
-            scrape_timeout: 30s
-
-            kubernetes_sd_configs:
-              - role: pod
-
-            relabel_configs:
-              - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape_slow]
-                action: keep
-                regex: true
-              - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scheme]
+                target_label: function
+                regex: (.*)-([a-z0-9]+)-([a-z0-9]+)
+                replacement: $1
+              - source_labels: [__meta_kubernetes_pod_label_vsysId]
                action: replace
-                regex: (https?)
-                target_label: __scheme__
-              - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_path]
-                action: replace
-                target_label: __metrics_path__
+                target_label: vsysId
                regex: (.+)
-              - source_labels: [__address__, __meta_kubernetes_pod_annotation_prometheus_io_port]
+              - source_labels: [__meta_kubernetes_pod_label_serviceFunction]
                action: replace
-                regex: (.+?)(?::\d+)?;(\d+)
-                replacement: $1:$2
-                target_label: __address__
-              - action: labelmap
-                regex: __meta_kubernetes_pod_annotation_prometheus_io_param_(.+)
-                replacement: __param_$1
-              - action: labelmap
-                regex: __meta_kubernetes_pod_label_(.+)
-              - source_labels: [__meta_kubernetes_namespace]
-                action: replace
-                target_label: namespace
-              - source_labels: [__meta_kubernetes_pod_name]
-                action: replace
-                target_label: pod
+                target_label: serviceFunction
+                regex: (.+)
+
              - source_labels: [__meta_kubernetes_pod_phase]
                regex: Pending|Succeeded|Failed|Completed
                action: drop
+            metric_relabel_configs:
+              - regex:  instance
+                action: labeldrop

--- a/ansible/roles/traffic-engine/files/helm/templates/traffic-engine.yaml
+++ b/ansible/roles/traffic-engine/files/helm/templates/traffic-engine.yaml
@@ -16,8 +16,13 @@ spec:
    metadata:
      labels:
        app: traffic-engine
+        vsysId: vsys-{{ .Values.vsys_id }}
+        serviceFunction: {{ .Values.nic_raw_name }}
      annotations:
-        configPath: "{{ .Values.configHash }}"
+        configHash: "{{ .Values.configHash }}"
+  {{- with .Values.deployment.annotations }}
+        {{- toYaml . | nindent 8 }}
+  {{- end }}

    spec:
      tolerations:
@@ -33,6 +38,8 @@ spec:
        imagePullPolicy: {{ .Values.image.firewall.pullPolicy }}
        workingDir: /opt/tsg/sapp
        command: ["/opt/tsg/sapp/sapp"]
+        ports:
+          - containerPort: 9273
        env:
        - name: MRZCPD_CTRLMSG_LISTEN_ADDR
          valueFrom:
@@ -88,8 +95,8 @@ spec:
        - name: sapp
          mountPath: "/opt/tsg/sapp/etc/send_raw_pkt.conf"
          subPath: "send_raw_pkt.conf"
-        - name: sapp
 {{- if and (eq .Values.proxy.enable .Values.define_enable_val_yes) (ge (len .Values.tfe_affinity) 1) }}
+        - name: sapp
          mountPath: "/opt/tsg/sapp/etc/kni/kni.conf"
          subPath: "kni.conf"
 {{- end }}
@@ -125,6 +132,8 @@ spec:
        imagePullPolicy: {{ .Values.image.proxy.pullPolicy }}
        workingDir: /opt/tsg/tfe
        command: ["/opt/tsg/tfe/bin/tfe"]
+        ports:
+          - containerPort: 9001
        env:
        - name: MRZCPD_CTRLMSG_LISTEN_ADDR
          valueFrom:
@@ -175,6 +184,8 @@ spec:
        command: ["/opt/tsg/certstore/bin/certstore"]
        securityContext:
          privileged: true
+        ports:
+          - containerPort: 9002
        volumeMounts:
        - name: certstore
          mountPath: "/opt/tsg/certstore/conf/cert_store.ini"
@@ -224,6 +235,18 @@ spec:
          mountPath: /etc/localtime
          readOnly: true

+      - name: merge-exporter
+        image: "{{ .Values.mergeExporter.image.repository }}:{{ .Values.mergeExporter.image.tag }}"
+        imagePullPolicy: {{ .Values.mergeExporter.image.pullPolicy }}
+        ports:
+        - containerPort: {{ .Values.mergeExporter.port }}
+        env:
+        # space-separated list of URLs
+        - name: MERGER_URLS
+          value: {{ .Values.mergeExporter.mergeUrls }}
+        - name: MERGER_PORT
+          value: "{{ .Values.mergeExporter.mergePort }}"
+
      initContainers:
      - name: tsg-init
        image: "{{ .Values.image.tsgInit.repository }}:{{ .Values.image.tsgInit.tag | default .Chart.AppVersion }}"
--- a/ansible/roles/traffic-engine/files/helm/values.yaml
+++ b/ansible/roles/traffic-engine/files/helm/values.yaml
@@ -165,6 +165,8 @@ image:
    pullPolicy: Never
    tag: ""

+
+
 decoders:
  DNS: yes
  QUIC: yes
@@ -184,3 +186,21 @@ decoders:
  GTPC: yes

 configHash: "defaulthash"
+
+
+mergeExporter:
+  image:
+    repository: quay.io/rebuy/exporter-merger
+    pullPolicy: Never
+    tag: "v0.2.0"
+  port: "9004"
+  mergeUrls: "http://localhost:9273/metrics http://localhost:9001/metrics http://localhost:9002/metrics"
+  mergePort: "9004"
+
+deployment:
+  annotations:
+    prometheus.io/scrape: "true"
+    prometheus.io/port: "9004"
+    
+    
+