修改获取 dockerenv 环境包是url错误问题

2、同步修改deploy.yml
3、为tsgconf/main.conf增加一个kafka的动态配置
4、修改tfe.conf中mc_default_eth的动态配置变量
5、原certstore中的r2、r3文件含有dos字符，修复该问题

.gitignore

@@ -1,2 +0,0 @@
-.vscode
-*.retry

.gitlab-ci.yml

@@ -0,0 +1,35 @@
+stages:
+- build
+    
+.build_tar:
+  image: "git.mesalab.cn:7443/mesa_platform/build-env:self-test-env"
+  variables:
+    GIT_STRATEGY: "clone"
+    BUILD_PADDING_PREFIX: /tmp/padding_for_PACK_TAR_BUILD_DIRS_PREFIX/
+    TESTING_VERSION_BUILD: 0
+  before_script:
+    - dockerd > /dev/null &
+    - docker info
+    - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
+    - mkdir -p $BUILD_PADDING_PREFIX/$CI_PROJECT_NAMESPACE/
+    - ln -s $CI_PROJECT_DIR $BUILD_PADDING_PREFIX/$CI_PROJECT_PATH
+    - cd $BUILD_PADDING_PREFIX/$CI_PROJECT_PATH
+    - pwd
+    - chmod +x ./ci/travis.sh
+  script:
+    - yum makecache
+    - ./ci/travis.sh
+  tags:
+  - share
+    
+file_build:
+  stage: build
+  variables:
+    VER_NAME: $CI_COMMIT_REF_NAME
+    PULP3_REPO_NAME: install-package-stable
+    PULP3_DIST_NAME: install-package-stable
+
+  extends: .build_tar
+  only:
+    - tags
+

adc_inline_device_access/group_vars/all.yml

@@ -1,75 +0,0 @@
-maat_redis_server:
-    address: "192.168.41.206"
-    port: 7002
-    db: 0
-
-dynamic_maat_redis_server:
-    address: "192.168.41.206"
-    port: 7002
-    db: 1
-
-cert_store_server:
-    address: "192.168.100.1"
-    port: 9991
-
-log_kafkabrokers:
-    address: "192.168.41.204:9092"
-
-log_minio:
-    address: "192.168.41.206"
-    port: 9090
-
-fs_remote:
-    switch: 1
-    address: "192.168.100.1"
-    port: 58125
-
-nic_transparent_mode:
-    enable: 0
-
-run_as_tun_mode: 0
-package_source: "local"
-
-install_dns_debug: "yes"
-install_ftp_debug: "yes"
-install_http_debug: "yes"
-install_mail_debug: "yes"
-install_ssl_debug: "yes"
-install_fw_dns_plug_debug: "yes"
-install_fw_ftp_plug_debug: "yes"
-install_fw_http_plug_debug: "yes"
-install_fw_mail_plug_debug: "yes"
-install_tsg_master: "yes"
-
-kni:
-    global:
-        log_level: 10
-        tfe_node_count: 3
-    watch_dog:
-        switch: 1
-    maat:
-        readconf_mode: 2
-    send_logger:
-        switch: 1
-    tfe_nodes:
-        - tfe0:
-              enabled: 1
-        - tfe1:
-              enabled: 1
-        - tfe2:
-              enabled: 1
-tfe:
-    nr_threads: 16
-    kni_ip: 192.168.100.1
-    mc_cache_eth: ens1.100
-    keykeeper:
-        mode: "normal"
-        no_cache: 0
-
-mrzcpd:
-    iocore: 47
-
-mrtunnat:
-    lcore_id: 46
-
-

adc_inline_device_access/hosts

@@ -1,24 +0,0 @@
-[all:vars]
-ansible_user=root
-package_source=local
-
-[blade-mxn]
-192.168.40.170
-
-[blade-00]
-192.168.40.166
-
-[blade-01]
-192.168.40.167
-
-[blade-02]
-192.168.40.168
-
-[blade-03]
-192.168.40.169
-
-[Functional_Host:children]
-blade-00
-blade-01
-blade-02
-blade-03

adc_tera_access/group_vars/all.yml

@@ -1,73 +0,0 @@
-maat_redis_server:
-    address: "192.168.41.206"
-    port: 7002
-    db: 0
-
-dynamic_maat_redis_server:
-    address: "192.168.41.206"
-    port: 7002
-    db: 1
-
-cert_store_server:
-    address: "192.168.100.1"
-    port: 9991
-
-log_kafkabrokers:
-    address: "192.168.41.204:9092"
-
-log_minio:
-    address: "192.168.41.206"
-    port: 9090
-
-fs_remote:
-    switch: 1
-    address: "192.168.100.1"
-    port: 58125
-
-nic_transparent_mode:
-    enable: 0
-
-run_as_tun_mode: 0
-package_source: "local"
-
-install_dns_debug: "yes"
-install_ftp_debug: "yes"
-install_http_debug: "yes"
-install_mail_debug: "yes"
-install_ssl_debug: "yes"
-install_fw_dns_plug_debug: "yes"
-install_fw_ftp_plug_debug: "yes"
-install_fw_http_plug_debug: "yes"
-install_fw_mail_plug_debug: "yes"
-install_tsg_master: "yes"
-
-kni:
-    global:
-        log_level: 10
-        tfe_node_count: 3
-    watch_dog:
-        switch: 1
-    maat:
-        readconf_mode: 2
-    send_logger:
-        switch: 1
-    tfe_nodes:
-        - tfe0:
-              enabled: 1
-        - tfe1:
-              enabled: 1
-        - tfe2:
-              enabled: 1
-tfe:
-    nr_threads: 16
-    keykeeper:
-        mode: "normal"
-        no_cache: 0
-
-mrzcpd:
-    iocore: 47
-
-mrtunnat:
-    lcore_id: 46
-
-

adc_tera_access/group_vars/blade-00.yml

@@ -1,14 +0,0 @@
-nic_mgr:
-    name: enp6s0
-nic_data_incoming:
-    name: ens1f4
-    address: 127.0.0.1
-nic_inner_ctrl:
-    name: ens1.100
-nic_to_tfe:
-    tfe0:
-        name: ens1f5
-    tfe1:
-        name: ens1f6
-    tfe2:
-        name: ens1f7

adc_tera_access/group_vars/blade-01.yml

@@ -1,11 +0,0 @@
-nic_mgr:
-    name: enp6s0
-nic_data_incoming:
-    name: ens1f1
-    mac: AA:BB:CC:DD:EE:FF
-    address: 127.0.0.1
-nic_inner_ctrl:
-    name: ens1.100
-nic_traffic_mirror:
-    name: ens1f2
-    use_mrzcpd: 1

adc_tera_access/group_vars/blade-02.yml

@@ -1,10 +0,0 @@
-nic_mgr:
-    name: enp6s0
-nic_data_incoming:
-    name: ens8f1
-    mac: AA:BB:CC:DD:EE:FF
-nic_inner_ctrl:
-    name: ens8.100
-nic_traffic_mirror:
-    name: ens8f2
-    use_mrzcpd: 1

adc_tera_access/group_vars/blade-03.yml

@@ -1,10 +0,0 @@
-nic_mgr:
-    name: enp6s0
-nic_data_incoming:
-    name: ens8f1
-    mac: AA:BB:CC:DD:EE:FF
-nic_inner_ctrl:
-    name: ens8.100
-nic_traffic_mirror:
-    name: ens8f2
-    use_mrzcpd: 1

buildPackage.yml

@@ -0,0 +1,3 @@
+- hosts: local
+  roles:
+    - package-build

build_config/group_vars/local.yml

@@ -0,0 +1,10 @@
+tarpath:
+  src:
+    - /tmp/padding_for_PACK_TAR_BUILD_DIRS_PREFIX/tsg/tsg-scripts/install_config
+    - /tmp/padding_for_PACK_TAR_BUILD_DIRS_PREFIX/tsg/tsg-scripts/deploy.yml
+    - /tmp/padding_for_PACK_TAR_BUILD_DIRS_PREFIX/tsg/tsg-scripts/roles/tsg-diagnose
+  destdict: /tmp/padding_for_PACK_TAR_BUILD_DIRS_PREFIX/tsg/tsg-scripts/build/
+
+tsgDiagnoseDockerFile: 
+  unarchiveUrl: http://repo.internal.geedge.net/pulp/content/install/stable/package/docker-rpm-test-docker-ce-7.tar.gz
+  unarchiveDest: /tmp/padding_for_PACK_TAR_BUILD_DIRS_PREFIX/tsg/tsg-scripts/roles/tsg-diagnose/files

build_config/hosts

@@ -0,0 +1,2 @@
+[local]
+localhost ansible_connection=local

ci/travis.sh

@@ -0,0 +1,14 @@
+#!/usr/bin/env sh
+mkdir build || true
+
+cat ./customize.yml >> ./build_config/group_vars/local.yml
+cat ./customize.yml >> ./install_config/group_vars/all.yml
+
+ansible-playbook -i ./build_config -e tarname=tsg-scripts-${VER_NAME}.tar.gz  buildPackage.yml
+
+ls -halt ./build/tsg-scripts-${VER_NAME}.tar.gz
+
+cd build
+cp ~/file_upload_tools.py ./
+
+python3 file_upload_tools.py ${PULP3_REPO_NAME} ${PULP3_DIST_NAME} *.tar.gz

clear_redis_cache.yml

@@ -1,6 +0,0 @@
-- hosts: blade-00
-  tasks:
-  - name: "killall certstore"
-    command: "killall certstore"
-  - name: "clear redis cache"
-    command: "redis-cli flushdb"

customize.yml

@@ -0,0 +1,52 @@
+rpmdict:
+  tsgDiagnose:
+    fullname: "tsg-diagnose-test_edit_name-1.el7.x86_64.rpm"
+    name: "tsg-diagnose"
+    downpath: "/tmp/padding_for_PACK_TAR_BUILD_DIRS_PREFIX/tsg/tsg-scripts/roles/tsg-diagnose/files/rpms"
+
+dockerEnvRpm:
+  dockerCe:
+    - container-selinux-2.119.2-1.911c772.el7_8.noarch.rpm
+    - selinux-policy-targeted-3.13.1-266.el7_8.1.noarch.rpm
+    - selinux-policy-3.13.1-266.el7_8.1.noarch.rpm
+    - containerd.io-1.2.13-3.2.el7.x86_64.rpm
+    - policycoreutils-python-2.5-34.el7.x86_64.rpm
+    - policycoreutils-2.5-34.el7.x86_64.rpm
+    - libselinux-utils-2.5-15.el7.x86_64.rpm
+    - libselinux-python-2.5-15.el7.x86_64.rpm
+    - libseccomp-2.3.1-4.el7.x86_64.rpm
+    - iptables-1.4.21-34.el7.x86_64.rpm
+    - libcgroup-0.41-21.el7.x86_64.rpm
+    - audit-libs-python-2.8.5-4.el7.x86_64.rpm
+    - setools-libs-3.3.8-4.el7.x86_64.rpm
+    - libsemanage-python-2.5-14.el7.x86_64.rpm
+    - checkpolicy-2.5-8.el7.x86_64.rpm
+    - libnetfilter_conntrack-1.0.6-1.el7_3.x86_64.rpm
+    - python-IPy-0.75-6.el7.noarch.rpm
+    - libnfnetlink-1.0.1-4.el7.x86_64.rpm
+    - libmnl-1.0.3-7.el7.x86_64.rpm
+    - docker-ce-cli-19.03.12-3.el7.x86_64.rpm
+    - docker-ce-19.03.12-3.el7.x86_64.rpm
+  dockerCompose:
+    - libtirpc-0.2.4-0.16.el7.x86_64.rpm
+    - libyaml-0.1.4-11.el7_0.x86_64.rpm
+    - python3-3.6.8-13.el7.x86_64.rpm
+    - python36-cached_property-1.5.1-2.el7.noarch.rpm
+    - python36-chardet-3.0.4-1.el7.noarch.rpm
+    - python36-docker-2.6.1-3.el7.noarch.rpm
+    - python36-dockerpty-0.4.1-18.el7.noarch.rpm
+    - python36-docker-pycreds-0.2.1-2.el7.noarch.rpm
+    - python36-docopt-0.6.2-8.el7.noarch.rpm
+    - python36-idna-2.7-2.el7.noarch.rpm
+    - python36-jsonschema-2.5.1-4.el7.noarch.rpm
+    - python36-pysocks-1.6.8-7.el7.noarch.rpm
+    - python36-PyYAML-3.13-1.el7.x86_64.rpm
+    - python36-requests-2.14.2-2.el7.noarch.rpm
+    - python36-six-1.14.0-2.el7.noarch.rpm
+    - python36-texttable-1.6.2-1.el7.noarch.rpm
+    - python36-urllib3-1.25.6-1.el7.noarch.rpm
+    - python36-websocket-client-0.47.0-2.el7.noarch.rpm
+    - python3-libs-3.6.8-13.el7.x86_64.rpm
+    - python3-pip-9.0.3-7.el7_7.noarch.rpm
+    - python3-setuptools-39.2.0-10.el7.noarch.rpm
+    - docker-compose-1.18.0-4.el7.noarch.rpm

deploy.yml

@@ -5,37 +5,55 @@
 
 - hosts: blade-00
   roles:
-    - tsg-env-mcn0
+#    - tsg-env-mcn0
     - mrzcpd
+    - sapp
+    - tsg_master
     - kni
+    - firewall
+    - http_healthcheck
+    - clotho
     - certstore
+    - cert-redis
+    - telegraf_statistic
+    - tsg-diagnose
 
 - hosts: blade-01
   roles:
-    - tsg-env-mcn1
+#    - tsg-env-mcn1
     - mrzcpd
     - tfe
 
 - hosts: blade-02
   roles:
-    - tsg-env-mcn2
+#    - tsg-env-mcn2
     - mrzcpd
     - tfe
 
 - hosts: blade-03
   roles:
-    - tsg-env-mcn3
+#    - tsg-env-mcn3
     - mrzcpd
     - tfe
 
 - hosts: blade-mxn
   roles:
-    - tsg-env-mxn
+#    - tsg-env-mxn
 
 - hosts: pc-as-tun-mode
   roles:
-    - mrzcpd
+    - kernel-ml
     - framework
+    - mrzcpd
+    - tsg-env-tun-mode
+    - sapp
+    - tsg_master
     - kni
+    - firewall
+    - http_healthcheck
+    - clotho
     - certstore
+    - cert-redis
     - tfe
+    - telegraf_statistic
+    - proxy_status

env-prod-astana/group_vars/all.yml

@@ -1,60 +0,0 @@
-maat_redis_server:
-    address: "10.4.35.1"
-    port: 6379
-    db: 0
-
-dynamic_maat_redis_server:
-    address: "10.4.35.1"
-    port: 6379
-    db: 1
-
-cert_store_server:
-    address: "192.168.100.1"
-    port: 9991
-
-log_kafkabrokers:
-    address: "10.4.35.7:9092,10.4.35.8:9092,10.4.35.9:9092,10.4.35.10:9092,10.4.35.11:9092"
-
-log_minio:
-    address: "10.4.35.1;"
-    port: 9000
-
-fs_remote:
-    switch: 1
-    address: "192.168.100.1"
-    port: 58125
-
-nic_transparent_mode:
-    enable: 0
-
-run_as_tun_mode: 0
-package_source: "local"
-
-kni:
-    global:
-        log_level: 10
-        tfe_node_count: 3
-    watch_dog:
-        switch: 1
-    send_logger:
-        switch: 1
-    tfe_nodes:
-        - tfe0:
-              enabled: 1
-        - tfe1:
-              enabled: 1
-        - tfe2:
-              enabled: 1
-tfe:
-    nr_threads: 16
-    keykeeper:
-        mode: "normal"
-        no_cache: 0
-
-mrzcpd:
-    iocore: 55
-
-mrtunnat:
-    lcore_id: 54
-
-

env-prod-astana/group_vars/blade-00.yml

@@ -1,14 +0,0 @@
-nic_mgr:
-    name: enp7s0
-nic_data_incoming:
-    name: ens1f4
-    address: 127.0.0.1
-nic_inner_ctrl:
-    name: ens1.100
-nic_to_tfe:
-    tfe0:
-        name: ens1f5
-    tfe1:
-        name: ens1f6
-    tfe2:
-        name: ens1f7

env-prod-astana/group_vars/blade-01.yml

@@ -1,11 +0,0 @@
-nic_mgr:
-    name: enp7s0
-nic_data_incoming:
-    name: ens1f1
-    mac: AA:BB:CC:DD:EE:FF
-    address: 127.0.0.1
-nic_inner_ctrl:
-    name: ens1.100
-nic_traffic_mirror:
-    name: ens1f2
-    use_mrzcpd: 1

env-prod-astana/group_vars/blade-02.yml

@@ -1,10 +0,0 @@
-nic_mgr:
-    name: enp7s0
-nic_data_incoming:
-    name: ens8f1
-    mac: AA:BB:CC:DD:EE:FF
-nic_inner_ctrl:
-    name: ens8.100
-nic_traffic_mirror:
-    name: ens8f2
-    use_mrzcpd: 1

env-prod-astana/group_vars/blade-03.yml

@@ -1,10 +0,0 @@
-nic_mgr:
-    name: enp7s0
-nic_data_incoming:
-    name: ens8f1
-    mac: AA:BB:CC:DD:EE:FF
-nic_inner_ctrl:
-    name: ens8.100
-nic_traffic_mirror:
-    name: ens8f2
-    use_mrzcpd: 1

env-prod-astana/hosts

@@ -1,100 +0,0 @@
-[all:vars]
-ansible_user=root
-
-[blade-mxn]
-10.4.164.23
-#10.4.164.24
-10.4.164.25
-10.4.164.26
-10.4.164.27
-10.4.164.28
-10.4.164.29
-
-[blade-00]
-10.4.39.9
-#10.4.39.13
-10.4.39.17
-10.4.39.21
-10.4.39.25
-10.4.39.29
-10.4.39.33
-
-[blade-01]
-10.4.39.10
-#10.4.39.14
-10.4.39.18
-10.4.39.22
-10.4.39.26
-10.4.39.30
-10.4.39.34
-
-[blade-02]
-10.4.39.11
-#10.4.39.15
-10.4.39.19
-10.4.39.23
-10.4.39.27
-10.4.39.31
-10.4.39.35
-
-[blade-03]
-10.4.39.12
-#10.4.39.16
-10.4.39.20
-10.4.39.24
-10.4.39.28
-10.4.39.32
-10.4.39.36
-
-[astana-adc-3]
-10.4.164.23
-10.4.39.9
-10.4.39.10
-10.4.39.11
-10.4.39.12
-
-[astana-adc-5]
-10.4.164.25
-10.4.39.17
-10.4.39.18
-10.4.39.19
-10.4.39.20
-
-[astana-adc-6]
-10.4.164.26
-10.4.39.21
-10.4.39.22
-10.4.39.23
-10.4.39.24
-
-[astana-adc-7]
-10.4.164.27
-10.4.39.25
-10.4.39.26
-10.4.39.27
-10.4.39.28
-
-[astana-adc-8]
-10.4.164.28
-10.4.39.29
-10.4.39.30
-10.4.39.31
-10.4.39.32
-
-[astana-adc-9]
-10.4.164.29
-10.4.39.33
-10.4.39.34
-10.4.39.35
-10.4.39.36
-
-[Functional_Host:children]
-blade-00
-blade-01
-blade-02
-blade-03
-
-[Slave_Host:children]
-blade-01
-blade-02
-blade-03

env-stage-hy/group_vars/all.yml

@@ -1,45 +0,0 @@
-maat_redis_server:
-    address: 192.168.100.3
-    port: 7002
-    db: 0
-
-dynamic_maat_redis_server:
-    address: 192.168.100.3
-    port: 7002
-    db: 0
-    
-cert_store_server:
-    address: 192.168.100.1
-    port: 9991
-
-log_kafkabrokers:
-    address: "192.168.100.4:9092"
-
-log_minio:
-    address: "192.168.100.4;"
-    port: 9000
-
-fs_remote:
-    switch: 0
-    address: "192.168.10.152"
-    port: 8125
-
-kni:
-    global:
-        log_level: 30
-        tfe_node_count: 3
-    watch_dog:
-        switch: 1
-    tfe_nodes:
-        - tfe0:
-              enabled: 1
-        - tfe1:
-              enabled: 1
-        - tfe2:
-              enabled: 1
-              
-tfe:
-    nr_threads: 32
-    keykeeper:
-        mode: "debug"
-        no_cache: 0

env-stage-hy/group_vars/blade-00.yml

@@ -1,13 +0,0 @@
-nic_mgr:
-    name: enp7s0
-nic_data_incoming:
-    name: ens1f4
-nic_inner_ctrl:
-    name: ens1.100
-nic_to_tfe:
-    tfe0:
-        name: ens1f5
-    tfe1:
-        name: ens1f6
-    tfe2:
-        name: ens1f7

env-stage-hy/group_vars/blade-01.yml

@@ -1,10 +0,0 @@
-nic_mgr:
-    name: enp7s0
-nic_data_incoming:
-    name: ens1f1
-    mac: AA:BB:CC:DD:EE:FF
-nic_inner_ctrl:
-    name: ens1.100
-nic_traffic_mirror:
-    name: ens1f2
-    use_mrzcpd: 1

env-stage-hy/group_vars/blade-02.yml

@@ -1,10 +0,0 @@
-nic_mgr:
-    name: enp7s0
-nic_data_incoming:
-    name: ens8f1
-    mac: AA:BB:CC:DD:EE:FF
-nic_inner_ctrl:
-    name: ens1.100
-nic_traffic_mirror:
-    name: ens8f2
-    use_mrzcpd: 1

env-stage-hy/group_vars/blade-03.yml

@@ -1,10 +0,0 @@
-nic_mgr:
-    name: enp7s0
-nic_data_incoming:
-    name: ens8f1
-    mac: AA:BB:CC:DD:EE:FF
-nic_inner_ctrl:
-    name: ens1.100
-nic_traffic_mirror:
-    name: ens8f2
-    use_mrzcpd: 1

env-stage-hy/hosts

@@ -1,12 +0,0 @@
-[all:vars]
-ansible_user=root
-
-[blade-00]
-192.168.10.41
-
-[blade-01]
-192.168.10.42
-
-[Functional_Host:children]
-blade-00
-blade-01

env-stage-pc/group_vars/all.yml

@@ -1,76 +0,0 @@
-maat_redis_server:
-  address: "192.168.40.83"
-  port: 7002
-  db: 0
-
-dynamic_maat_redis_server:
-  address: "192.168.40.83"
-  port: 7002
-  db: 0
-
-cert_store_server:
-  address: "127.0.0.1"
-  port: 9991
-
-log_kafkabrokers:
-  address: "192.168.40.85:9092"
-
-log_minio:
-  address: "192.168.40.85;"
-  port: 9000
-
-fs_remote:
-  switch: 1
-  address: "127.0.0.1"
-  port: 8125
-
-kni:
-  global:
-    log_level: 30
-    tfe_node_count: 3
-  watch_dog:
-    switch: 1
-  tfe_nodes:
-    - tfe0:
-        enabled: 1
-    - tfe1:
-        enabled: 1
-    - tfe2:
-        enabled: 1
-tfe:
-  nr_threads: 32
-  keykeeper:
-    mode: "normal"
-    no_cache: 0
-
-mrzcpd:
-  iocore: 47
-
-mrtunnat:
-  lcore_id: 46
-
-nic_mgr:
-  name: eth0
-nic_data_incoming:
-  name: tun_kni
-  address: 127.0.0.1
-nic_inner_ctrl:
-  name: lo
-nic_to_tfe:
-  tfe0:
-    name: lo
-  tfe1:
-    name: lo
-  tfe2:
-    name: lo
-nic_traffic_mirror:
-  name: lo
-  use_mrzcpd: 0
-
-nic_transparent_mode:
-  enable: 1
-  mode: pcap
-  internel_interface: "enp0s20f0u3"
-  external_interface: "enp0s20f0u4"
-
-run_as_tun_mode: 1

env-stage-pc/hosts

@@ -1,6 +0,0 @@
-[all:vars]
-ansible_user=root
-package_source=local
-
-[pc-as-tun-mode]
-192.168.40.85

env-stage-xxg/group_vars/all.yml

@@ -1,55 +0,0 @@
-maat_redis_server:
-    address: "192.168.40.120"
-    port: 7002
-    db: 0
-
-dynamic_maat_redis_server:
-    address: "192.168.40.120"
-    port: 7002
-    db: 1
-
-cert_store_server:
-    address: "192.168.40.161"
-    port: 9991
-
-log_kafkabrokers:
-    address: "192.168.40.119:9092"
-
-log_minio:
-    address: "192.168.40.223;"
-    port: 9000
-
-fs_remote:
-    switch: 1
-    address: "192.168.100.1"
-    port: 8125
-
-nic_transparent_mode:
-    enable: 0
-
-kni:
-    global:
-        log_level: 30
-        tfe_node_count: 3
-    watch_dog:
-        switch: 1
-    tfe_nodes:
-        - tfe0:
-              enabled: 1
-        - tfe1:
-              enabled: 1
-        - tfe2:
-              enabled: 1
-tfe:
-    nr_threads: 32
-    keykeeper:
-        mode: "normal"
-        no_cache: 0
-
-mrzcpd:
-    iocore: 47
-
-mrtunnat:
-    lcore_id: 46
-
-run_as_tun_mode: 1

env-stage-xxg/group_vars/blade-00.yml

@@ -1,14 +0,0 @@
-nic_mgr:
-    name: enp6s0
-nic_data_incoming:
-    name: ens1f4
-    address: 127.0.0.1
-nic_inner_ctrl:
-    name: ens1.100
-nic_to_tfe:
-    tfe0:
-        name: ens1f5
-    tfe1:
-        name: ens1f6
-    tfe2:
-        name: ens1f7

env-stage-xxg/group_vars/blade-01.yml

@@ -1,11 +0,0 @@
-nic_mgr:
-    name: enp6s0
-nic_data_incoming:
-    name: ens1f1
-    mac: AA:BB:CC:DD:EE:FF
-    address: 127.0.0.1
-nic_inner_ctrl:
-    name: ens1.100
-nic_traffic_mirror:
-    name: ens1f2
-    use_mrzcpd: 1

env-stage-xxg/group_vars/blade-02.yml

@@ -1,10 +0,0 @@
-nic_mgr:
-    name: enp6s0
-nic_data_incoming:
-    name: ens8f1
-    mac: AA:BB:CC:DD:EE:FF
-nic_inner_ctrl:
-    name: ens8.100
-nic_traffic_mirror:
-    name: ens8f2
-    use_mrzcpd: 1

env-stage-xxg/group_vars/blade-03.yml

@@ -1,10 +0,0 @@
-nic_mgr:
-    name: enp6s0
-nic_data_incoming:
-    name: ens8f1
-    mac: AA:BB:CC:DD:EE:FF
-nic_inner_ctrl:
-    name: ens8.100
-nic_traffic_mirror:
-    name: ens8f2
-    use_mrzcpd: 1

env-stage-xxg/hosts

@@ -1,24 +0,0 @@
-[all:vars]
-ansible_user=root
-package_source=pulp
-
-[blade-mxn]
-192.168.40.25
-
-[blade-00]
-192.168.40.21
-
-[blade-01]
-192.168.40.22
-
-[blade-02]
-192.168.40.23
-
-[blade-03]
-192.168.40.24
-
-[Functional_Host:children]
-blade-00
-blade-01
-blade-02
-blade-03

install_config/group_vars/all.yml

@@ -0,0 +1,90 @@
+#########################################
+#####0: Pcap; 1: Inline_device;  2: Allot;  3: ADC_Tun_mode;  4: ATCA;
+tsg_access_type: 4
+
+#####0: Tun_mode;  1: normal; 2: ADC;
+tsg_running_type: 1 
+
+########################################
+maat_redis_server:
+  address: "192.168.40.168"
+  port: 7002
+  db: 0
+
+dynamic_maat_redis_server:
+  address: "192.168.40.168"
+  port: 7002
+  db: 0
+
+cert_store_server:
+  address: "192.168.100.1"
+  port: 9991
+
+log_kafkabrokers:
+  address: "1.1.1.1:9092,2.2.2.2:9092"
+
+log_minio:
+  address: "192.168.40.168;"
+  port: 9090
+
+fs_remote:
+  switch: 1
+  address: "192.168.100.1"
+  port: 58125
+  
+########################################
+sapp:
+  worker_threads: 16
+  send_only_threads_max: 8
+  bind_mask: 1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16
+  inbound_route_dir: 1
+
+########################################
+kni:
+  global:
+    log_level: 30
+    tfe_node_count: 3
+  watch_dog:
+    switch: 1
+  maat:
+    readconf_mode: 2
+  send_logger:
+    switch: 1
+  tfe_nodes:
+    tfe0_enabled: 1
+    tfe1_enabled: 1
+    tfe2_enabled: 1
+
+########################################
+tfe:
+  nr_threads: 32
+  mc_cache_eth: lo 
+  keykeeper:
+    mode: "normal"
+    no_cache: 0
+
+########################################
+mrzcpd:
+  iocore: 39
+
+mrtunnat:
+  lcore_id: 38
+
+nic_data_incoming:
+  ethname: enp1s0
+  vf0_name: enp1s2
+  vf1_name: enp1s2f1
+  vf2_name: enp1s2f2
+
+VlanFlipping:
+  vlanID_1: 100
+  vlanID_2: 101
+  vlanID_3: 103
+  vlanID_4: 104
+########################################
+server:
+  ethname: eth0
+  tun_name: eth0.100
+  internal_interface: "eth2"
+  external_interface: "eth3"
+

install_config/group_vars/blade-00.yml

@@ -13,3 +13,11 @@ nic_to_tfe:
         name: ens1f6
     tfe2:
         name: ens1f7
+
+AllotAccess:
+    virturlInterface_1: ens1f2.103
+    virturlInterface_2: ens1f2.104
+    virturlID_1: 103
+    virturlID_2: 104
+    vvipv4_mask: 24
+    vvipv6_mask: 64

install_config/hosts

@@ -2,11 +2,13 @@
 ansible_user=root
 package_source=local
 
+[pc-as-tun-mode]
+
 [blade-mxn]
 192.168.40.170
 
 [blade-00]
-192.168.40.166
+192.168.40.166 vvipv4_1= vvipv4_2= vvipv6_1= vvipv6_2=
 
 [blade-01]
 192.168.40.167
@@ -17,14 +19,8 @@ package_source=local
 [blade-03]
 192.168.40.169
 
-
 [Functional_Host:children]
 blade-00
 blade-01
 blade-02
 blade-03
-
-[Slave_Host:children]
-blade-01
-blade-02
-blade-03

pc_double_arm_access/group_vars/all.yml

@@ -1,91 +0,0 @@
-maat_redis_server:
-  address: "192.168.40.83"
-  port: 7002
-  db: 0
-
-dynamic_maat_redis_server:
-  address: "192.168.40.83"
-  port: 7002
-  db: 0
-
-cert_store_server:
-  address: "127.0.0.1"
-  port: 9991
-
-log_kafkabrokers:
-  address: "192.168.40.85:9092"
-
-log_minio:
-  address: "192.168.40.85;"
-  port: 9090
-
-fs_remote:
-  switch: 1
-  address: "127.0.0.1"
-  port: 8125
-  
-install_dns_debug: "yes"
-install_ftp_debug: "yes"
-install_http_debug: "yes"
-install_mail_debug: "yes"
-install_ssl_debug: "yes"
-install_fw_dns_plug_debug: "yes"
-install_fw_ftp_plug_debug: "yes"
-install_fw_http_plug_debug: "yes"
-install_fw_mail_plug_debug: "yes"
-install_tsg_master: "yes"
-
-kni:
-  global:
-    log_level: 30
-    tfe_node_count: 3
-  watch_dog:
-    switch: 1
-  maat:
-    readconf_mode: 2
-  send_logger:
-    switch: 1
-  tfe_nodes:
-    - tfe0:
-        enabled: 1
-    - tfe1:
-        enabled: 1
-    - tfe2:
-        enabled: 1
-tfe:
-  nr_threads: 32
-  keykeeper:
-    mode: "normal"
-    no_cache: 0
-
-mrzcpd:
-  iocore: 39
-
-mrtunnat:
-  lcore_id: 38
-
-nic_mgr:
-  name: eth0
-nic_data_incoming:
-  name: tun_kni
-  address: 127.0.0.1
-nic_inner_ctrl:
-  name: lo
-nic_to_tfe:
-  tfe0:
-    name: lo
-  tfe1:
-    name: lo
-  tfe2:
-    name: lo
-nic_traffic_mirror:
-  name: lo
-  use_mrzcpd: 0
-
-nic_transparent_mode:
-  enable: 1
-  mode: pcap
-  internel_interface: "eth2"
-  external_interface: "eth3"
-
-run_as_tun_mode: 1

pc_double_arm_access/hosts

@@ -1,6 +0,0 @@
-[all:vars]
-ansible_user=root
-package_source=local
-
-[pc-as-tun-mode]
-192.168.40.139

pulp-install.yml

@@ -1,3 +0,0 @@
-- hosts: blade-0*
-  roles:
-    - pulp-consumer

rc.local

@@ -1,13 +0,0 @@
-#!/bin/bash
-# THIS FILE IS ADDED FOR COMPATIBILITY PURPOSES
-#
-# It is highly advisable to create own systemd services or udev rules
-# to run scripts during boot instead of using this file.
-#
-# In contrast to previous versions due to parallel execution during boot
-# this script will NOT be run after all other services.
-#
-# Please note that you must run 'chmod +x /etc/rc.d/rc.local' to ensure
-# that this script will be executed during boot.
-
-touch /var/lock/subsys/local

roles/cert-redis/files/cert-redis/cert-redis.service

@@ -0,0 +1,16 @@
+[Unit]
+Description=Redis persistent key-value database
+After=network.target
+After=network-online.target
+Wants=network-online.target
+
+[Service]
+ExecStart=/usr/local/bin/start-cert-redis
+ExecStop=killall redis-server
+Type=forking
+RuntimeDirectory=redis
+RuntimeDirectoryMode=0755
+
+[Install]
+WantedBy=multi-user.target
+

roles/cert-redis/files/cert-redis/install.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+#
+cp -rf redis-server /usr/local/bin/
+cp -rf redis-cli /usr/local/bin
+cp -rf cert-redis.service /usr/lib/systemd/system/
+cp -rf start-cert-redis /usr/local/bin

roles/cert-redis/files/cert-redis/start-cert-redis

@@ -0,0 +1,4 @@
+#!/bin/bash
+#
+
+/usr/local/bin/redis-server /home/tsg/cert-redis/6379/6379.conf

roles/cert-redis/tasks/main.yml

@@ -0,0 +1,15 @@
+- name: "copy cert-redis to destination server"
+  copy:
+    src: "{{ role_path }}/files/"
+    dest: /home/tsg
+    mode: 0755
+
+- name: "install cert-redis"
+  shell: cd /home/tsg/cert-redis;sh install.sh
+
+- name: "start cert-redis"
+  systemd:
+    name: cert-redis.service
+    state: started
+    daemon_reload: yes
+    enabled: yes

roles/certstore/tasks/main.yml

@@ -1,39 +1,26 @@
----
-- name: "copy redis and dependency to destination"
+- name: "copy certstore rpm to destination"
   synchronize:
     src: "{{ role_path }}/files/"
     dest: "/tmp/ansible_deploy/"
 
-#- name: "install redis"
-#  yum:
-#    name:
-#      - /tmp/ansible_deploy/jemalloc-3.6.0-1.el7.x86_64.rpm
-#      - /tmp/ansible_deploy/redis-3.2.12-2.el7.x86_64.rpm
-#    state: present
-
-#- name: "enable redis"
-#  systemd:
-#    name: redis
-#    enabled: yes
-#    state: started
-
 - name: Ensures /home/tsg exists
   file: path=/home/tsg state=directory
   tags: mkdir
 
 - name: install certstore
-  unarchive:
-    src: "{{ role_path }}/files/certstore-base-online-20200108.tar.gz"
-    dest: /home/tsg
+  yum:
+    name:
+      - /tmp/ansible_deploy/certstore-v20.05.0f61dde-1.el7.centos.x86_64.rpm
+    state: present
 
 - name: template certstore configure file
   template:
     src: "{{ role_path }}/templates/cert_store.ini.j2"
-    dest: /home/tsg/certstore-base/conf/cert_store.ini
+    dest: /home/tsg/certstore/conf/cert_store.ini
 
-- name: bootup certstore
-  blockinfile:
-    marker: "## {mark} bootstrap certstore"
-    path: /etc/rc.d/rc.local
-    block: |
-      cd /home/tsg/certstore-base; ./r2_certstore
+- name: "start certstore"
+  systemd:
+    name: certstore.service
+    state: started
+    enabled: yes
+    daemon_reload: yes

roles/certstore/templates/cert_store.ini.j2

@@ -2,7 +2,7 @@
 #1:print on screen, 0:don't
 DEBUG_SWITCH = 1
 #10:DEBUG, 20:INFO, 30:FATAL
-RUN_LOG_LEVEL = 30
+RUN_LOG_LEVEL = 10
 RUN_LOG_PATH = ./logs
 [CONFIG]
 #Number of running threads
@@ -15,7 +15,7 @@ expire_after = 30
 local_debug = 1
 ca_path = ./cert/tango-ca-v3-trust-ca.pem
 untrusted_ca_path = ./cert/mesalab-ca-untrust.pem
-[NTC_MAAT]
+[MAAT]
 #Configure the load mode,
 #0: using the configuration distribution network
 #1: using local json
@@ -42,4 +42,7 @@ port = 6379
 #Maat monitors the Redsi server IP address and port number
 ip = {{ maat_redis_server.address }}
 port = {{ maat_redis_server.port }}
-dbindex =  {{ maat_redis_server.db }}
+dbindex =  {{ maat_redis_server.db }}
+[stat]
+statsd_server=192.168.100.1
+statsd_port=8126

roles/clotho/files/clotho.service

@@ -0,0 +1,13 @@
+[Unit]
+Description=clotho
+After=network.target
+After=network-online.target
+Wants=network-online.target
+
+[Service]
+ExecStart=/home/mesasoft/clotho/clotho
+ExecStop=killall clotho
+Type=forking
+
+[Install]
+WantedBy=multi-user.target

roles/clotho/tasks/main.yml

@@ -0,0 +1,29 @@
+- name: "copy clotho rpm to destination server"
+  copy:
+    src: "{{ role_path }}/files/clotho-debug-1.0.0.-1.el7.x86_64.rpm"
+    dest: /tmp/ansible_deploy/
+
+- name: "copy  clotho.service to destination server"
+  copy:
+    src: "{{ role_path }}/files/clotho.service"
+    dest: /usr/lib/systemd/system
+    mode: 0755
+
+- name: "install clotho rpm from localhost"
+  yum:
+    name:
+      - /tmp/ansible_deploy/clotho-debug-1.0.0.-1.el7.x86_64.rpm
+    state: present
+
+- name: "Template the clotho.conf"
+  template:
+    src: "{{ role_path }}/templates/clotho.conf.j2"
+    dest: /home/mesasoft/clotho/conf/clotho.conf 
+  tags: template
+ 
+- name: "start clotho"
+  systemd:
+    name: clotho.service
+    enabled: yes
+    daemon_reload: yes
+

roles/clotho/templates/clotho.conf.j2

@@ -0,0 +1,11 @@
+[KAFKA]
+BROKER_LIST={{ log_kafkabrokers.address }}
+
+[SYSTEM]
+{% if tsg_running_type == 0 or 1 %}
+NIC_NAME={{ server.ethname }}
+{% else %}
+NIC_NAME={{ nic_mgr.name }}
+{% endif %}
+LOG_LEVEL=10
+LOG_PATH=log/clotho

roles/firewall/tasks/main.yml

@@ -0,0 +1,47 @@
+---
+- name: "copy firewall rpms to destination server"
+  copy:
+    src: "{{ role_path }}/files/"
+    dest: /tmp/ansible_deploy/
+
+- name: "install firewall packages"
+  yum:
+    name: "{{ fw_packages }}"
+    state: present
+    skip_broken: yes
+  vars:
+    fw_packages:
+      - /tmp/ansible_deploy/dns-2.0.2.5effe72-2.el7.x86_64.rpm
+      - /tmp/ansible_deploy/ftp-1.0.4.5d3a283-2.el7.x86_64.rpm
+      - /tmp/ansible_deploy/http-2.0.1.e8f12ee-2.el7.x86_64.rpm
+      - /tmp/ansible_deploy/mail-1.0.3.cbc6034-2.el7.x86_64.rpm
+      - /tmp/ansible_deploy/ssl-1.0.0.73e5273-2.el7.x86_64.rpm
+      - /tmp/ansible_deploy/tsg_conn_record-1.0.0.2155660-1.el7.centos.x86_64.rpm
+      - /tmp/ansible_deploy/fw_dns_plug-debug-1.0.3.ea8e0f6-1.el7.centos.x86_64.rpm
+      - /tmp/ansible_deploy/fw_ftp_plug-1.1.0.74c9a05-2.el7.x86_64.rpm
+      - /tmp/ansible_deploy/fw_ssl_plug-1.0.3.30fcf35-2.el7.x86_64.rpm
+      - /tmp/ansible_deploy/fw_mail_plug-1.1.0.a42c5a0-2.el7.x86_64.rpm
+      - /tmp/ansible_deploy/fw_http_plug-1.2.0.a7e63c0-2.el7.x86_64.rpm
+      - /tmp/ansible_deploy/capture_packet_plug-debug-1.0.0.-1.el7.x86_64.rpm
+      - /tmp/ansible_deploy/clotho-debug-1.0.0.-1.el7.x86_64.rpm
+      - /tmp/ansible_deploy/quic-1.1.4.9c2e0ba-2.el7.x86_64.rpm
+      - /tmp/ansible_deploy/fw_quic_plug-1.0.1.e8cded4-2.el7.x86_64.rpm
+
+- name: "Template the tsgconf/main.conf"
+  template:
+    src: "{{ role_path }}/templates/main.conf.j2"
+    dest: /home/mesasoft/sapp_run/tsgconf/main.conf
+  tags: template
+
+
+- name: "Template the tsgconf/maat.conf"
+  template:
+    src: "{{ role_path }}/templates/maat.conf.j2"
+    dest: /home/mesasoft/sapp_run/tsgconf/maat.conf
+  tags: template
+
+- name: "Template the conf/capture_packet_plug.conf.j2"
+  template:
+    src: "{{ role_path }}/templates/capture_packet_plug.conf.j2"
+    dest: /home/mesasoft/sapp_run/conf/capture_packet_plug.conf
+  tags: template

roles/firewall/templates/capture_packet_plug.conf.j2

@@ -0,0 +1,29 @@
+[MAAT]
+MAAT_MODE=2
+#EFFECTIVE_FLAG=
+STAT_SWITCH=1
+PERF_SWITCH=1
+TABLE_INFO=conf/capture_packet_tableinfo.conf
+STAT_FILE=capture_packet_maat.status
+EFFECT_INTERVAL_S=1
+REDIS_IP={{ maat_redis_server.address }}
+REDIS_PORT_NUM=1
+REDIS_PORT={{ maat_redis_server.port }}
+REDIS_INDEX=0
+JSON_CFG_FILE=conf/capture_packet_maat.json
+INC_CFG_DIR=capture_packet_rule/inc/index/
+FULL_CFG_DIR=capture_packet_rule/full/index/
+
+[LOG]
+{% if tsg_running_type == 0 or 1 %}
+NIC_NAME={{ server.ethname }}
+{% else %}
+NIC_NAME={{ nic_mgr.name }}
+{% endif %}
+BROKER_LIST={{ log_kafkabrokers.address }}
+FIELD_FILE=conf/capture_packet_log_field.conf
+
+[SYSTEM]
+LOG_LEVEL=10
+LOG_PATH=./tsglog/capture_packet_plug/capture_packet
+

roles/firewall/templates/maat.conf.j2

@@ -0,0 +1,32 @@
+[STATIC]
+###0:location 1:json 2:redis
+MAAT_MODE=2
+STAT_SWITCH=1
+PERF_SWITCH=1
+TABLE_INFO=tsgconf/tsg_static_tableinfo.conf
+STAT_FILE=tsg_static_maat.status
+EFFECT_INTERVAL_S=1
+REDIS_IP={{ maat_redis_server.address }}
+REDIS_PORT_NUM=1
+REDIS_PORT=7002
+REDIS_INDEX=0
+JSON_CFG_FILE=tsgconf/tsg_maat.json
+INC_CFG_DIR=tsgrule/inc/index/
+FULL_CFG_DIR=tsgrule/full/index/
+
+[DYNAMIC]
+###0:location 1:json 2:redis
+MAAT_MODE=2
+STAT_SWITCH=1
+PERF_SWITCH=1
+TABLE_INFO=tsgconf/tsg_dynamic_tableinfo.conf
+STAT_FILE=tsg_dynamic_maat.status
+EFFECT_INTERVAL_S=1
+REDIS_IP={{ dynamic_maat_redis_server.address }}
+REDIS_PORT_NUM=1
+REDIS_PORT=7002
+REDIS_INDEX=1
+JSON_CFG_FILE=tsgconf/tsg_maat.json
+INC_CFG_DIR=tsgrule/inc/index/
+FULL_CFG_DIR=tsgrule/full/index/
+

roles/firewall/templates/main.conf.j2

@@ -0,0 +1,55 @@
+[FTP_PLUG]
+LOG_PATH=./tsglog/fw_ftp_plug/fw_ftp_plug
+LOG_LEVEL=10
+TIMEOUT=600
+
+[MAIL_PLUG]
+LOG_PATH=./tsglog/fw_mail_plug/fw_mail_plug
+LOG_LEVEL=10
+TIMEOUT=600
+
+[HTTP_PLUG]
+LOG_PATH=./tsglog/fw_http_plug/fw_http_plug
+LOG_LEVEL=10
+
+[DNS_PLUG]
+LOG_PATH=./tsglog/fw_dns_plug/fw_dns_plug
+LOG_LEVEL=10
+
+[MAAT]
+PROFILE=./tsgconf/maat.conf
+SUBSCRIBER_ID_TABLE=TSG_OBJ_SUBSCRIBER_ID
+CB_SUBSCRIBER_IP_TABLE=TSG_DYN_SUBSCRIBER_IP
+IP_ADDR_TABLE=TSG_SECURITY_ADDR
+
+[TSG_LOG]
+MODE=1
+{% if tsg_running_type == 0 or 1 %}
+NIC_NAME={{ server.ethname }}
+{% else %}
+NIC_NAME={{ nic_mgr.name }}
+{% endif %}
+MAX_SERVICE=1
+LOG_LEVEL=10
+LOG_PATH=./tsglog/tsglog
+BROKER_LIST={{ log_kafkabrokers.address }}
+COMMON_FIELD_FILE=tsgconf/tsg_log_field.conf
+
+[STATISTIC]
+CYCLE=1
+TELEGRAF_PORT=8100
+TELEGRAF_IP=127.0.0.1
+OUTPUT_PATH=./tsg_statistic.log
+APP_NAME=statistic
+
+[FIELD_STAT]
+CYCLE=3
+TELEGRAF_PORT=8125
+TELEGRAF_IP=127.0.0.1
+OUTPUT_PATH=./tsg_stat.log
+APP_NAME=tsg_master
+
+[SYSTEM]
+LOG_LEVEL=10
+LOG_PATH=./tsglog/tsg_master
+POLICY_PRIORITY_LABEL=POLICY_PRIORITY