feature: add log collector

ansible/HAL_deploy.yml

@@ -196,3 +196,12 @@
   roles:
     - {role: framework, tags: framework}
     - {role: trex, tags: trex}
+
+- hosts: x86_64_COTS-log-collector
+  remote_user: root
+  vars_files:
+    - install_config/group_vars/HAL_x86_64_COTS.yml
+    - install_config/group_vars/rpm_version.yml
+  roles:
+    - {role: framework, tags: framework}
+    - {role: log-collector, tags: log-collector}

ansible/install_config/group_vars/rpm_version.yml

@@ -107,7 +107,10 @@ packet_adapter_rpm_version:
   packet_adapter: packet_adapter-2.0.2.4f48b59
 
 nic_uio_binder_rpm_version:
-  nic_uio_binder: nic-uio-binder-1.0.10-c67fa3b
+  nic_uio_binder: nic-uio-binder-1.0.11-08f054a
 
 dp_trace_rpm_version:
   dp_trace: dp_trace_telemetry-0.2.1.c48d240
+
+log_collector_rpm_version:
+  log_collector: log-collector-1.0.0-9bedb55

ansible/roles/dp-trace-telemetry/files/helm/templates/_config.tpl

@@ -2,7 +2,7 @@
 {{- if eq .Values.datapath_trace.debug.enable_interactive_startup .Values.define_enable_val_yes -}}
             while true; do sleep 10;done
 {{- else -}}
-           exec /opt/tsg/dp_trace_telemetry/bin/dp_trace_telemetry -c /opt/tsg/dp_trace_telemetry/etc/dp_trace.conf -d /opt/tsg/dp_trace_telemetry/etc/dp_trace_dy.conf 2>&1|tee >(systemd-cat -t $DEPLOYMENT_NAME)
+           exec /opt/tsg/dp_trace_telemetry/bin/dp_trace_telemetry -c /opt/tsg/dp_trace_telemetry/etc/dp_trace.conf -d /opt/tsg/dp_trace_telemetry/etc/dp_trace_dy.conf
 {{- end }}
 {{- end -}}
 

ansible/roles/k3s-install/files/log-collector.yaml

@@ -0,0 +1,40 @@
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: log-collector
+  labels:
+    app: log-collector
+  namespace: tsg-os-system
+spec:
+  selector:
+    matchLabels:
+      app: log-collector
+  template:
+    metadata:
+      labels:
+        app: log-collector
+    spec:
+      containers:
+      - name: log-collector
+        image: registry.gdnt-cloud.website/tsg-log-collector:%%LOG_COLLECTOR_IMAGE_TAR%%
+        command:
+          - "bash"
+          - "-ec"
+          - |
+            /opt/tsg/log-collector/bin/log-collector
+        volumeMounts:
+        - name: logs
+          mountPath: /var/log/pods
+        - name: journal-volume
+          mountPath: /run/systemd/journal
+        securityContext:
+          privileged: true
+      volumes:
+      - name: logs
+        hostPath:
+          path: /var/log/pods
+          type: DirectoryOrCreate
+      - name: journal-volume
+        hostPath:
+          path: /run/systemd/journal
+          type: Directory

ansible/roles/k3s-install/tasks/main.yml

@@ -122,6 +122,11 @@
     src: "{{ role_path }}/files/nic-uio-binder.yaml"
     dest: /var/lib/rancher/k3s/server/manifests/
 
+- name: "copy log-collector file to dest"
+  copy:
+    src: "{{ role_path }}/files/log-collector.yaml"
+    dest: /var/lib/rancher/k3s/server/manifests/
+
 - name: "copy dynamic-hostport yaml file to k3s manifests directory"
   copy:
     src: "{{ role_path }}/files/dynamic-hostport.yaml"

ansible/roles/log-collector/tasks/main.yml

@@ -0,0 +1,7 @@
+---
+- name: "install log-collector"
+  yum:
+    name:
+      - "{{ log_collector_rpm_version.log_collector }}"
+    conf_file: "{{ rpm_repo_config_path }}"
+    state: present

ansible/roles/mrzcpd/files/helm/templates/_config.tpl

@@ -10,7 +10,7 @@
 {{- if eq .Values.mrzcpd.debug.enable_interactive_startup .Values.define_enable_val_yes -}}
             while true; do sleep 10;done
 {{- else -}}
-           exec /opt/tsg/mrzcpd/bin/mrzcpd -c /opt/tsg/mrzcpd/etc/mrglobal.conf -s /opt/tsg/mrzcpd/etc/mrglobal.dynamic.conf 2>&1|tee >(systemd-cat -t $DEPLOYMENT_NAME)
+           exec /opt/tsg/mrzcpd/bin/mrzcpd -c /opt/tsg/mrzcpd/etc/mrglobal.conf -s /opt/tsg/mrzcpd/etc/mrglobal.dynamic.conf 
 {{- end }}
 {{- end -}}
 

ansible/roles/packet_adapter/files/helm/templates/_config.tpl

@@ -84,7 +84,7 @@
 {{- if eq .Values.debug.inject_adapter.enable_interactive_startup .Values.define_enable_val_yes -}}
             while true; do sleep 10;done
 {{- else -}}
-           exec /opt/tsg/packet_adapter/bin/packet_adapter 2>&1 | tee >(systemd-cat -t $DEPLOYMENT_NAME)
+           exec /opt/tsg/packet_adapter/bin/packet_adapter 
 {{- end }}
 {{- end -}}
 

ansible/roles/traffic-engine/files/helm/templates/deployment-firewall.yaml

@@ -90,7 +90,7 @@ spec:
             {{- if eq .Values.debug.firewall.enable_interactive_startup .Values.define_enable_val_yes }}
             while true; do sleep 10;done
             {{- else }}
-            exec /opt/tsg/sapp/sapp 2>&1|tee >(systemd-cat -t $DEPLOYMENT_NAME)
+            exec /opt/tsg/sapp/sapp 
             {{- end }}
         ports:
           - containerPort: 9273

ansible/roles/traffic-engine/files/helm/templates/deployment-proxy.yaml

@@ -63,7 +63,7 @@ spec:
             {{- if eq .Values.debug.proxy.enable_interactive_startup .Values.define_enable_val_yes }}
             while true; do sleep 10;done
             {{- else }}
-            exec /opt/tsg/tfe/bin/tfe 2>&1|tee >(systemd-cat -t $DEPLOYMENT_NAME)
+            exec /opt/tsg/tfe/bin/tfe 
             {{- end }}
         ports:
           - containerPort: 9001

ansible/roles/traffic-engine/files/helm/templates/deployment-sce.yaml

@@ -63,7 +63,7 @@ spec:
             {{- if eq .Values.debug.service_chaining.enable_interactive_startup .Values.define_enable_val_yes }}
             while true; do sleep 10;done
             {{- else }}
-            exec /opt/tsg/sce/bin/sce 2>&1|tee >(systemd-cat -t $DEPLOYMENT_NAME)
+            exec /opt/tsg/sce/bin/sce 
             {{- end }}
         ports:
           - containerPort: 9006

ansible/roles/traffic-engine/files/helm/templates/deployment-shaping.yaml

@@ -77,7 +77,7 @@ spec:
             {{- if eq .Values.debug.shaping.enable_interactive_startup .Values.define_enable_val_yes }}
             while true; do sleep 10;done
             {{- else }}
-            exec /opt/tsg/shaping_engine/bin/shaping_engine 2>&1|tee >(systemd-cat -t $DEPLOYMENT_NAME)
+            exec /opt/tsg/shaping_engine/bin/shaping_engine 
             {{- end }}
         ports:
           - containerPort: 8551

make/Makefile.x86_64_COTS

@@ -16,6 +16,8 @@ TARGET_BUILD_DIR 		:= $(BUILDDIR_BASE)/$(PROFILE_ID)
 TARGET_INSTALLER_DIR 	:= $(TARGET_BUILD_DIR)/installer
 NIC_UIO_BINDER_FILE_PATH  := /ansible/roles/k3s-install/files/nic-uio-binder.yaml
 NIC_UIO_BINDER_IMAGE_TAR  := ${OS_RELEASE_VER}
+LOG_COLLECTOR_FILE_PATH  := /ansible/roles/k3s-install/files/log-collector.yaml
+LOG_COLLECTOR_IMAGE_TAR  := ${OS_RELEASE_VER}
 TARGET_SYSROOT_DIR		:= $(TARGET_BUILD_DIR)/sysroot
 
 .PHONY: all builddir installer sysroot-base sysroot-cleanup sysroot-archive sysroot-binary add-images-into-installer-dir add-patch-into-installer-dir clean
@@ -43,6 +45,7 @@ installer: builddir
 	sed -i -e "s/%%SIZE_PART_UPDATE%%/$(SIZE_PART_UPDATE)/" $(TARGET_INSTALLER_DIR)/install.sh
 	sed -i -e "s/%%APP_BUNDLE_BIN%%/$(APP_BUNDLE_BIN)/" $(TARGET_INSTALLER_DIR)/install.sh
 	sed -i -e "s/%%NIC_UIO_BINDER_IMAGE_TAR%%/$(NIC_UIO_BINDER_IMAGE_TAR)/" $(PROJECTDIR)/$(NIC_UIO_BINDER_FILE_PATH)
+	sed -i -e "s/%%LOG_COLLECTOR_IMAGE_TAR%%/$(LOG_COLLECTOR_IMAGE_TAR)/" $(PROJECTDIR)/$(LOG_COLLECTOR_FILE_PATH)
 
 	sed -i '/sapp-pr:/d;/mrzcpd:/d' $(PROJECTDIR)/ansible/install_config/group_vars/rpm_version.yml
 ifeq ($(ENABLE_ALIYUN_INTERNAL_DOMAIN),1)

make/Makefile.x86_64_COTS_APP_BUNDLE

@@ -12,6 +12,7 @@ INJECT_HELM_CHART_TAR           := inject-adapter-$(HELM_CHART_VER).tgz
 CONTAINER_IMAGES_TAR_GLOB		:= traffic-engine-*-docker.tar
 HELM_CHART_TAR_GLOB				:= traffic-engine-*.tgz
 NIC_UIO_BINDER_IMAGE_TAR  := ${OS_RELEASE_VER}
+LOG_COLLECTOR_IMAGE_TAR  := ${OS_RELEASE_VER}
 
 TARGET_BUILD_DIR				:= $(BUILDDIR_BASE)/$(PROFILE_ID)/containers
 TARGET_INSTALLER_DIR			:= $(TARGET_BUILD_DIR)/installer
@@ -21,6 +22,7 @@ MRZCPD_HELM_FILE_PATH           := /ansible/roles/mrzcpd/files/helm
 TREX_HELM_FILE_PATH             :=/ansible/roles/trex/files/helm
 DP_TRACE_HELM_FILE_PATH         := /ansible/roles/dp-trace-telemetry/files/helm
 NIC_UIO_BINDER_FILE_PATH        := /ansible/roles/k3s-install/files/nic-uio-binder.yaml
+LOG_COLLECTOR_FILE_PATH        := /ansible/roles/k3s-install/files/log-collector.yaml
 
 
 .PHONY: all build_dir installer clean_installer_dir prebuild_container_images build_container_images build_helm_charts build_app_bundle_bin 
@@ -47,6 +49,7 @@ installer: build_dir
 	sed -i -e "s/%%CONTAINER_IMAGES_TAR_GLOB%%/$(CONTAINER_IMAGES_TAR_GLOB)/" $(TARGET_INSTALLER_DIR)/install.sh
 	sed -i -e "s/%%HELM_CHART_TAR_GLOB%%/$(HELM_CHART_TAR_GLOB)/" $(TARGET_INSTALLER_DIR)/install.sh
 	sed -i -e "s/%%NIC_UIO_BINDER_IMAGE_TAR%%/$(NIC_UIO_BINDER_IMAGE_TAR)/" $(PROJECTDIR)/$(NIC_UIO_BINDER_FILE_PATH)
+	sed -i -e "s/%%LOG_COLLECTOR_IMAGE_TAR%%/$(LOG_COLLECTOR_IMAGE_TAR)/" $(PROJECTDIR)/$(LOG_COLLECTOR_FILE_PATH)
 
 prebuild_container_images:
 ifeq ($(ENABLE_LICENSE_PROTECTED),1)

tools/build_container_images.sh

@@ -69,6 +69,7 @@ function export_images()
             ${IMAGE_REPO_PREFIX}/trex:${OS_RELEASE_VER} \
             ${IMAGE_REPO_PREFIX}/tsg-shaping:${OS_RELEASE_VER} \
             ${IMAGE_REPO_PREFIX}/tsg-nic-uio-binder:${OS_RELEASE_VER} \
+             ${IMAGE_REPO_PREFIX}/tsg-log-collector:${OS_RELEASE_VER} \
             ${IMAGE_REPO_PREFIX}/tsg-dp-trace-telemetry:${OS_RELEASE_VER}
 }
 ##function define end
@@ -90,6 +91,7 @@ build_image "packet-io-engine"    "${BASE_IMAGE_NAME}" "${IMAGE_REPO_PREFIX}/tsg
 build_image "trex"      "${BASE_IMAGE_NAME}" "${IMAGE_REPO_PREFIX}/trex:${OS_RELEASE_VER}"
 build_image "nic-uio-binder"    "${BASE_IMAGE_NAME}" "${IMAGE_REPO_PREFIX}/tsg-nic-uio-binder:${OS_RELEASE_VER}"
 build_image "dp-trace-telemetry"    "${BASE_IMAGE_NAME}" "${IMAGE_REPO_PREFIX}/tsg-dp-trace-telemetry:${OS_RELEASE_VER}"
+build_image "log-collector"    "${BASE_IMAGE_NAME}" "${IMAGE_REPO_PREFIX}/tsg-log-collector:${OS_RELEASE_VER}"
 
 export_images