diff --git a/ansible/HAL_deploy.yml b/ansible/HAL_deploy.yml index 398d8129..98cdca69 100644 --- a/ansible/HAL_deploy.yml +++ b/ansible/HAL_deploy.yml @@ -210,6 +210,7 @@ roles: - {role: framework, tags: framework} - {role: mrzcpd, tags: mrzcpd} + - {role: confd, tags: confd} - hosts: server remote_user: root diff --git a/ansible/roles/confd/files/confd b/ansible/roles/confd/files/confd new file mode 100644 index 00000000..e6976a0f Binary files /dev/null and b/ansible/roles/confd/files/confd differ diff --git a/ansible/roles/confd/files/packet-io-conf.toml b/ansible/roles/confd/files/packet-io-conf.toml new file mode 100644 index 00000000..8144c608 --- /dev/null +++ b/ansible/roles/confd/files/packet-io-conf.toml @@ -0,0 +1,7 @@ +[template] +src = "packet-io.conf.tmpl" +dest = "/opt/tsg/config/shared/mrglobal.dynamic.conf" +keys = [ + "/obp_devices/" +] +reload_cmd = "/opt/tsg/mrzcpd/bin/signal.sh" \ No newline at end of file diff --git a/ansible/roles/confd/files/packet-io.conf.tmpl b/ansible/roles/confd/files/packet-io.conf.tmpl new file mode 100644 index 00000000..5fe56137 --- /dev/null +++ b/ansible/roles/confd/files/packet-io.conf.tmpl @@ -0,0 +1,17 @@ +{{range $key, $dir := lsdir "/obp_devices"}}{{$name_dir := printf "/obp_devices/%s/name" $dir}}{{$type_dir := printf "/obp_devices/%s/type" $dir}}{{$connect_dir := printf "/obp_devices/%s/connect" $dir}}{{$address_dir := printf "/obp_devices/%s/address" $dir}}{{$port_dir := printf "/obp_devices/%s/port" $dir}}{{$segment_dir := printf "/obp_devices/%s/segment" $dir}}[olp_device:{{$key}}] +name = {{getv $name_dir}} +type = {{getv $type_dir}} +connect = {{getv $connect_dir}} +in_addr = {{getv $address_dir}} +port = {{getv $port_dir}} +{{$num := ""}} +{{range $segkey, $segdir := lsdir $segment_dir}}{{$index:=len $num}}{{$id_dir := printf "/obp_devices/%s/segment/%d/id" $dir $index}}{{$workline_dir := printf "/obp_devices/%s/segment/%d/workline" $dir $index}}{{$heartbeat_modeline_dir := printf "/obp_devices/%s/segment/%d/heartbeat_mode" $dir $index}}{{$hb_send_interval_in_ms_dir := printf "/obp_devices/%s/segment/%d/hb_send_interval_in_ms" $dir $index}}{{$hb_timeout_interval_in_ms_dir := printf "/obp_devices/%s/segment/%d/hb_timeout_interval_in_ms" $dir $index}}{{$hb_lost_threshold_dir := printf "/obp_devices/%s/segment/%d/hb_lost_threshold" $dir $index}}{{$nonrevertive_mode_dir := printf "/obp_devices/%s/segment/%d/nonrevertive_mode" $dir $index}}{{$num = printf "%s1" $num}} +[{{getv $name_dir}}:{{getv $id_dir}}] +state = {{getv $workline_dir}} +heartbeat = {{getv $heartbeat_modeline_dir}} +heartbeat_send_interval_in_ms= {{getv $hb_send_interval_in_ms_dir}} +heartbeat_timeout_interval_in_ms= {{getv $hb_timeout_interval_in_ms_dir}} +heartbeat_lost_threshold = {{getv $hb_lost_threshold_dir}} +nonrevertive_mode = {{getv $nonrevertive_mode_dir}} +{{end}} +{{end}} \ No newline at end of file diff --git a/ansible/roles/confd/tasks/main.yml b/ansible/roles/confd/tasks/main.yml new file mode 100644 index 00000000..a61f8d60 --- /dev/null +++ b/ansible/roles/confd/tasks/main.yml @@ -0,0 +1,47 @@ +- name: "Create /opt/confd/bin directory if it does not exist" + file: + path: "{{ item }}" + state: directory + mode: '0755' + with_items: + - /opt/confd/bin + when: runtime_env == 'TSG-X-P0906' + +- name: "Create /etc/confd/conf.d directory if it does not exist" + file: + path: "{{ item }}" + state: directory + mode: '0755' + with_items: + - /etc/confd/conf.d + when: runtime_env == 'TSG-X-P0906' + +- name: "Create /etc/confd/templates directory if it does not exist" + file: + path: "{{ item }}" + state: directory + mode: '0755' + with_items: + - /etc/confd/templates + when: runtime_env == 'TSG-X-P0906' + +- name: "copy confd to dest" + copy: + src: "{{ role_path }}/files/confd" + dest: /opt/confd/bin/confd + mode: 0755 + when: runtime_env == 'TSG-X-P0906' + +- name: "copy packet-io-conf.toml to dest" + copy: + src: "{{ role_path }}/files/packet-io-conf.toml" + dest: /etc/confd/conf.d/packet-io-conf.toml + mode: 0755 + when: runtime_env == 'TSG-X-P0906' + +- name: "copy packet-io.conf.tmpl to dest" + copy: + src: "{{ role_path }}/files/packet-io.conf.tmpl" + dest: /etc/confd/templates/packet-io.conf.tmpl + mode: 0755 + when: runtime_env == 'TSG-X-P0906' \ No newline at end of file diff --git a/ansible/roles/mrzcpd/files/helm/templates/_config.tpl b/ansible/roles/mrzcpd/files/helm/templates/_config.tpl index e0f773af..a7dbe6bb 100644 --- a/ansible/roles/mrzcpd/files/helm/templates/_config.tpl +++ b/ansible/roles/mrzcpd/files/helm/templates/_config.tpl @@ -1,27 +1,3 @@ -{{- define "mrzcpd.mount.hostpath" -}} -{{- if eq .Values.mrzcpd.debug.enable_mount_host_filesystem .Values.define_enable_val_yes }} - - name: host-root - mountPath: /host -{{- end }} -{{- end -}} - -{{- define "mrzcpd.volume.hostpath" -}} - - name: host-root - hostPath: - path: / -{{- end -}} - -{{- define "mrzcpd.mount.localtime" -}} - - name: localtime-node - mountPath: /etc/localtime - readOnly: true -{{- end -}} - -{{- define "mrzcpd.volume.localtime" -}} - - name: localtime-node - hostPath: - path: /etc/localtime -{{- end -}} {{- define "mrzcpd.resources" -}} {{- if .Values.mrzcpd.interfaces }} @@ -35,7 +11,7 @@ {{- if eq .Values.mrzcpd.debug.enable_interactive_startup .Values.define_enable_val_yes -}} while true; do sleep 10;done {{- else -}} - /opt/tsg/mrzcpd/bin/mrzcpd -c /opt/tsg/mrzcpd/etc/mrglobal.conf + /opt/tsg/mrzcpd/bin/mrzcpd -c /opt/tsg/mrzcpd/etc/mrglobal.conf -s /opt/tsg/mrzcpd/etc/mrglobal.dynamic.conf {{- end }} {{- end -}} @@ -329,6 +305,12 @@ interface_ext=virtio_dign_s [vwire:{{ $virtual_wire.id }}] interface_int={{ $virtual_wire.internal_interface }} interface_ext={{ $virtual_wire.external_interface }} +{{- if $virtual_wire.obp_device }} +obp_device={{ $virtual_wire.obp_device }} +{{- end }} +{{- if $virtual_wire.obp_segment }} +obp_segment={{ $virtual_wire.obp_segment }} +{{- end }} {{$enter}} {{- end }} # classifier rule diff --git a/ansible/roles/mrzcpd/files/helm/templates/deployment-mrzcpd.yaml b/ansible/roles/mrzcpd/files/helm/templates/deployment-mrzcpd.yaml index ce7f4942..bafbbe2c 100644 --- a/ansible/roles/mrzcpd/files/helm/templates/deployment-mrzcpd.yaml +++ b/ansible/roles/mrzcpd/files/helm/templates/deployment-mrzcpd.yaml @@ -6,7 +6,7 @@ metadata: app: packet-io-engine component: packet-io-engine annotations: - reloader.stakater.com/auto: "true" + configmap.reloader.stakater.com/reload: mrzcpd-static-conf spec: replicas: 1 @@ -25,7 +25,8 @@ spec: prometheus.io/port: "9005" prometheus.io/scrape: "true" - spec: + spec: + shareProcessNamespace: true tolerations: - key: node-role.kubernetes.io/control-plane operator: Exists @@ -49,13 +50,16 @@ spec: cp -f /etc/profile.d/mrzcpd.sh /mnt/packet-io-engine/profile.d/mrzcpd.sh ldconfig -C /etc/ld.so.cache -r /host - cp -f /opt/tsg/config/mrzcpd.conf /opt/tsg/mrzcpd/etc/mrglobal.conf + cp -f /opt/tsg/config/mrzcpd-static-conf/static.conf /opt/tsg/mrzcpd/etc/mrglobal.conf chmod +x /opt/tsg/mrzcpd/bin/replaceconf.sh /opt/tsg/mrzcpd/bin/replaceconf.sh - + ln -s /opt/tsg/mrzcpd/shared/mrglobal.dynamic.conf /opt/tsg/mrzcpd/etc/mrglobal.dynamic.conf + {{ template "mrzcpd.prestart" . }} {{ template "mrzcpd.start" . }} volumeMounts: + - name: shared-dir + mountPath: /opt/tsg/mrzcpd/shared - name: mrzcpd mountPath: /mnt/packet-io-engine/mrzcpd mountPropagation: Bidirectional @@ -63,9 +67,8 @@ spec: mountPath: /mnt/packet-io-engine/ld.so.conf.d - name: etc-profile mountPath: /mnt/packet-io-engine/profile.d - - name: mrzcpd-conf - mountPath: "/opt/tsg/config/mrzcpd.conf" - subPath: mrzcpd.conf + - name: mrzcpd-static-conf + mountPath: /opt/tsg/config/mrzcpd-static-conf/ - name: run-mrzcpd mountPath: /var/run/mrzcpd - name: hugepages @@ -79,12 +82,18 @@ spec: - name: host-root mountPath: /host mountPropagation: HostToContainer - {{ template "mrzcpd.mount.localtime" . }} + - name: localtime-node + mountPath: /etc/localtime + readOnly: true {{ template "mrzcpd.mount.prestart" . }} resources: limits: mrzcpd/mrzcpd: 1 - {{ template "mrzcpd.resources" . }} +{{- if .Values.mrzcpd.interfaces }} +{{- range $interface := .Values.mrzcpd.interfaces }} + mrzcpd/{{ $interface.name }}: 1 +{{- end }} +{{- end }} hugepages-1Gi: {{.Values.mrzcpd.hugepages }} requests: memory: 100Mi @@ -131,6 +140,26 @@ spec: securityContext: privileged: true + + - name: confd + image: "registry.gdnt-cloud.website/tsg-packet-io-engine:{{ .Chart.AppVersion }}" + imagePullPolicy: Never + lifecycle: + type: Sidecar + command: + - "bash" + - "-ec" + - | + chmod +x /opt/tsg/mrzcpd/bin/signal.sh + /opt/confd/bin/confd -watch -backend file -file /opt/tsg/config/mrzcpd-reload-conf/reload-conf.yaml -log-level debug + volumeMounts: + - name: shared-dir + mountPath: /opt/tsg/config/shared + - name: mrzcpd-reload-conf + mountPath: /opt/tsg/config/mrzcpd-reload-conf/ + securityContext: + privileged: true + - command: - /opt/tsg/mrzcpd/bin/monit_device - --prometheus-client @@ -163,6 +192,22 @@ spec: - mountPath: /etc/localtime name: localtime-node readOnly: true + - command: + - /opt/tsg/mrzcpd/bin/monit_obp + - --prometheus-client + image: "registry.gdnt-cloud.website/tsg-packet-io-engine:{{ .Chart.AppVersion }}" + imagePullPolicy: Never + name: monit-obp + volumeMounts: + - mountPath: /var/run/mrzcpd + name: run-mrzcpd + readOnly: true + - mountPath: /var/run/dpdk + name: dpdk + readOnly: true + - mountPath: /etc/localtime + name: localtime-node + readOnly: true - env: - name: MERGER_URLS value: {{ .Values.mergeExporter.mergeUrls }} @@ -176,13 +221,18 @@ spec: containerPort: {{ .Values.mergeExporter.mergePort }} protocol: TCP volumes: + - name: shared-dir + emptyDir: {} - name: prestart-dir hostPath: path: /etc/tsg-os/{{ .Release.Name }}/ type: DirectoryOrCreate - - name: mrzcpd-conf + - name: mrzcpd-static-conf configMap: - name: mrzcpd + name: mrzcpd-static-conf + - name: mrzcpd-reload-conf + configMap: + name: mrzcpd-reload-conf - name: mrzcpd hostPath: path: /opt/tsg/mrzcpd @@ -214,7 +264,11 @@ spec: - name: dev-vfio hostPath: path: /dev/vfio - {{ template "mrzcpd.volume.localtime" . }} + - name: localtime-node + hostPath: + path: /etc/localtime + - name: host-root + hostPath: + path: / {{ template "mrzcpd.volume.prestart" . }} - {{ template "mrzcpd.volume.hostpath" . }} hostNetwork: true diff --git a/ansible/roles/mrzcpd/files/helm/templates/mrzcpd.yaml b/ansible/roles/mrzcpd/files/helm/templates/mrzcpd.yaml index 75f5919e..d590cc90 100644 --- a/ansible/roles/mrzcpd/files/helm/templates/mrzcpd.yaml +++ b/ansible/roles/mrzcpd/files/helm/templates/mrzcpd.yaml @@ -1,6 +1,15 @@ apiVersion: v1 kind: ConfigMap metadata: - name: mrzcpd + name: mrzcpd-static-conf data: - mrzcpd.conf: {{ tpl (.Files.Get "conf/mrzcpd.conf") . | quote }} \ No newline at end of file + static.conf: {{ tpl (.Files.Get "conf/mrzcpd.conf") . | quote }} + +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: mrzcpd-reload-conf +data: + reload-conf.yaml: | + obp_devices: {{ .Values.mrzcpd.obp_devices | toJson }} \ No newline at end of file diff --git a/ansible/roles/mrzcpd/files/helm/values.yaml b/ansible/roles/mrzcpd/files/helm/values.yaml index f14e69e5..ff53f15c 100644 --- a/ansible/roles/mrzcpd/files/helm/values.yaml +++ b/ansible/roles/mrzcpd/files/helm/values.yaml @@ -85,7 +85,7 @@ mergeExporter: pullPolicy: Never tag: "v0.2.0" mergePort: "9005" - mergeUrls: "http://0.0.0.0:8901/metrics http://0.0.0.0:8902/metrics" + mergeUrls: "http://0.0.0.0:8901/metrics http://0.0.0.0:8902/metrics http://0.0.0.0:8903/metrics" service: type: ClusterIP @@ -105,6 +105,7 @@ mrzcpd: traffic_mirror: {} distmode: 0 hugepages: 32Gi + obp_devices: [{'name': 'obp1', 'type': '2', 'connect': '3', 'address': '2.2.2.2', 'port': '6800', 'segment': [{'id': '1', 'workline':'0', 'heartbeat_mode': '1', 'hb_send_interval_in_ms': '100', 'hb_timeout_interval_in_ms': '100', 'hb_lost_threshold': '5', 'nonrevertive_mode': '1'}, {'id': '1', 'workline':'0', 'heartbeat_mode': '1', 'hb_send_interval_in_ms': '100', 'hb_timeout_interval_in_ms': '100', 'hb_lost_threshold': '5', 'nonrevertive_mode': '1'}]}] debug: enable_liveness_probe: yes define_enable_val_yes: yes diff --git a/ansible/roles/mrzcpd/files/signal.sh b/ansible/roles/mrzcpd/files/signal.sh new file mode 100644 index 00000000..57a28196 --- /dev/null +++ b/ansible/roles/mrzcpd/files/signal.sh @@ -0,0 +1,10 @@ +#!/bin/sh +# Find the process PID +pid=$(pidof mrzcpd) +if [ -z "$pid" ]; then + echo "No process found that matches the criteria" + exit 0 +fi +# Send SIGHUP signal +kill -HUP "$pid" +echo "SIGHUP signal sent to process with PID: $pid" \ No newline at end of file diff --git a/ansible/roles/mrzcpd/tasks/main.yml b/ansible/roles/mrzcpd/tasks/main.yml index 630abb20..e55a414d 100644 --- a/ansible/roles/mrzcpd/tasks/main.yml +++ b/ansible/roles/mrzcpd/tasks/main.yml @@ -77,6 +77,14 @@ mode: 0644 when: runtime_env == 'TSG-X-P0906' +- name: "copy signal.sh to dest" + copy: + src: "{{ role_path }}/files/signal.sh" + dest: /opt/tsg/mrzcpd/bin/signal.sh + mode: 0644 + when: runtime_env == 'TSG-X-P0906' + + - name: "replace action: replace Requires to Wants" replace: path: "{{ item }}" diff --git a/ansible/roles/tsg-os-HAL/files/script/tsg-os-HAL.sh b/ansible/roles/tsg-os-HAL/files/script/tsg-os-HAL.sh index 23b2f731..c009382e 100644 --- a/ansible/roles/tsg-os-HAL/files/script/tsg-os-HAL.sh +++ b/ansible/roles/tsg-os-HAL/files/script/tsg-os-HAL.sh @@ -100,6 +100,7 @@ cat < ${k3s_cfg_file} kubelet-arg: - "kube-reserved=memory=${KUBE_RESERVED}Gi" - "system-reserved=memory=${system_mem}Gi" + - "sync-frequency=1s" EOF }