diff --git a/clear_redis_cache.yml b/clear_redis_cache.yml new file mode 100644 index 0000000..a6e147e --- /dev/null +++ b/clear_redis_cache.yml @@ -0,0 +1,8 @@ +- hosts: blade-00 + tasks: + - name: "killall certstore" + command: "killall certstore" + - name: "clear redis cache" + command: "redis-cli flushdb" + - name: "killall sapp" + command: "killall sapp" diff --git a/hosts.astana b/hosts.astana index f7aae56..d201de1 100644 --- a/hosts.astana +++ b/hosts.astana @@ -37,5 +37,22 @@ ansible_user=root 10.4.39.32 10.4.39.36 +[astana-adc-3] +10.4.39.9 +10.4.39.10 +10.4.39.11 +10.4.39.12 + +[astana-adc-9] +10.4.39.33 +10.4.39.34 +10.4.39.35 +10.4.39.36 + +[control-blade-all:children] +blade-01 +blade-02 +blade-03 + [certstore] 10.4.35.1 ansible_user=tsg diff --git a/remove_ansible_block.yml b/remove_ansible_block.yml new file mode 100644 index 0000000..7d8e531 --- /dev/null +++ b/remove_ansible_block.yml @@ -0,0 +1,6 @@ +- hosts: all + tasks: + - name: "remove all ansible manange block" + blockinfile: + dest: "/etc/rc.d/rc.local" + state: absent diff --git a/roles/certstore/files/certstore-base-online-20190701.tar.gz b/roles/certstore/files/certstore-base-online-20190701.tar.gz new file mode 100644 index 0000000..008ac44 Binary files /dev/null and b/roles/certstore/files/certstore-base-online-20190701.tar.gz differ diff --git a/roles/certstore/tasks/main.yml b/roles/certstore/tasks/main.yml new file mode 100644 index 0000000..34a6ad2 --- /dev/null +++ b/roles/certstore/tasks/main.yml @@ -0,0 +1,22 @@ +--- +- name: Ensures /home/tsg exists + file: path=/home/tsg state=directory + tags: mkdir + +- name: install certstore + unarchive: + src: "{{ role_path }}/files/certstore-base-online-20190701.tar.gz" + dest: /home/tsg + +- name: template certstore configure file + template: + src: "{{ role_path }}/templates/cert_store.ini.j2" + dest: /home/tsg/certstore-base/conf/cert_store.ini + +- name: bootup certstore + blockinfile: + marker: "## {mark} bootstrap certstore" + path: /etc/rc.d/rc.local + block: | + cd /home/tsg/certstore-base; ./r2_certstore + diff --git a/roles/certstore/templates/cert_store.ini.j2 b/roles/certstore/templates/cert_store.ini.j2 new file mode 100644 index 0000000..f2895fc --- /dev/null +++ b/roles/certstore/templates/cert_store.ini.j2 @@ -0,0 +1,44 @@ +[SYSTEM] +#1:print on screen, 0:don't +DEBUG_SWITCH = 1 +#10:DEBUG, 20:INFO, 30:FATAL +RUN_LOG_LEVEL = 30 +RUN_LOG_PATH = ./logs +[CONFIG] +#Number of running threads +thread-nu = 4 +#Local default root certificate is valid for 30 days by default +expire_after = 30 +#Local default root certificate path +local_debug = 1 +ca_path = ./cert/tango-ca-v3-trust-ca.pem +untrusted_ca_path = ./cert/mesalab-ca-untrust.pem +[NTC_MAAT] +#Configure the load mode, +#0: using the configuration distribution network +#1: using local json +#2: using Redis reads +maat_json_switch=2 +#When the loading mode is sent to the network, set the scanning configuration modification interval (s). +effective_interval=1 +#Specify the location of the configuration library table file +table_info=./conf/table_info.conf +#Incremental profile path +inc_cfg_dir=./rule/inc/index +#Full profile path +full_cfg_dir=./rule/full/index +#Json file path when json schema is used +pxy_obj_keyring=./conf/pxy_obj_keyring.json +[LIBEVENT] +#Local monitor port number, default is 9991 +port = 9991 +[CERTSTORE_REDIS] +#The Redis server IP address and port number where the certificate is stored locally +ip = 127.0.0.1 +port = 6379 +[MAAT_REDIS] +#Maat monitors the Redsi server IP address and port number +ip = 10.4.35.1 +port = 6379 +dbindex = 0 + diff --git a/roles/framework/files/framework/framework.conf b/roles/framework/files/framework/framework.conf new file mode 100644 index 0000000..446277c --- /dev/null +++ b/roles/framework/files/framework/framework.conf @@ -0,0 +1 @@ +/opt/MESA/lib/ diff --git a/roles/framework/tasks/main.yml b/roles/framework/tasks/main.yml index c5105e4..d29f490 100644 --- a/roles/framework/tasks/main.yml +++ b/roles/framework/tasks/main.yml @@ -13,6 +13,11 @@ - /tmp/ansible_deploy/dkms/dkms-2.7.1-1.el7.noarch.rpm - /tmp/ansible_deploy/framework/framework-2.0.9.f583d06-1.el7.centos.x86_64.rpm +- name: "install framework ld.conf" + synchronize: + src: "{{ role_path }}/files/framework/framework.conf" + dest: /etc/ld.so.conf.d/framework.conf + - name: "install/update rulescan header files" synchronize: src: "{{ role_path }}/files/rulescan/rulescan.h" diff --git a/roles/kni/files/kni_20190630.tar.gz b/roles/kni/files/kni_20190630.tar.gz deleted file mode 100644 index 6925ec8..0000000 Binary files a/roles/kni/files/kni_20190630.tar.gz and /dev/null differ diff --git a/roles/kni/files/kni_20190702.tar.gz b/roles/kni/files/kni_20190702.tar.gz new file mode 100644 index 0000000..1c4c2b3 Binary files /dev/null and b/roles/kni/files/kni_20190702.tar.gz differ diff --git a/roles/kni/files/kni_20190703.tar.gz b/roles/kni/files/kni_20190703.tar.gz new file mode 100644 index 0000000..6306e62 Binary files /dev/null and b/roles/kni/files/kni_20190703.tar.gz differ diff --git a/roles/kni/tasks/main.yml b/roles/kni/tasks/main.yml index 3f96574..c41e311 100644 --- a/roles/kni/tasks/main.yml +++ b/roles/kni/tasks/main.yml @@ -5,7 +5,7 @@ - name: Extract kni.tar.gz unarchive: - src: "{{ role_path }}/files/kni_20190630.tar.gz" + src: "{{ role_path }}/files/kni_20190703.tar.gz" dest: /home/tsg tags: extract @@ -15,9 +15,16 @@ dest: /home/tsg/kni/conf/kni/kni.conf tags: template +- name: Template the main.conf + template: + src: "{{ role_path }}/templates/main.conf.j2" + dest: /home/tsg/kni/conf/main.conf + tags: template + - name: bootup sapp blockinfile: path: /etc/rc.d/rc.local + marker: "## {mark} bootstrap mrzcpd and kni" block: | systemctl start mrzcpd systemctl start mrtunnat diff --git a/roles/kni/templates/kni.conf.j2 b/roles/kni/templates/kni.conf.j2 index 7a56243..f7a07b9 100644 --- a/roles/kni/templates/kni.conf.j2 +++ b/roles/kni/templates/kni.conf.j2 @@ -67,7 +67,7 @@ mho_screen_print_ctrl = 0 mho_thread_safe = 1 mho_mutex_num = 160 mho_hash_slot_size = 160000 -mho_hash_max_element_num = 640000 +mho_hash_max_element_num = 1280000 mho_expire_time = 30 mho_eliminate_type = LRU @@ -76,10 +76,10 @@ mho_screen_print_ctrl = 0 mho_thread_safe = 1 mho_mutex_num = 160 mho_hash_slot_size = 160000 -mho_hash_max_element_num = 640000 +mho_hash_max_element_num = 1280000 #must be 0 mho_expire_time = 0 mho_eliminate_type = LRU [field_stat] -stat_path = ./fs2_kni.status \ No newline at end of file +stat_path = ./fs2_kni.status diff --git a/roles/kni/templates/main.conf.j2 b/roles/kni/templates/main.conf.j2 new file mode 100644 index 0000000..d7f91ca --- /dev/null +++ b/roles/kni/templates/main.conf.j2 @@ -0,0 +1,93 @@ +[Module] +threadnum=16 +cpu_bind_core_mask=0x1fffe +app_instance_name=sapp_master +MaxTcpStreams=200000 +MaxUdpStreams=200000 +LinkTimeout=30 +UdpResetTime=0 +CreatLinkMode=1 +MaxUnorderNum=5 +TcpAllEnable=1 +IPv6_module_enable=1 +IPv6_raw_socket=1 +ipentry_priority_over_ipfrag=0 +dictator_switch=0 +load_plug_switch=1 +discard_ack=0 +kill_tcp_remedy=0 + +signal_take_over_switch=0 + +timestamp_record=0 +#timedelay_threshold unit: CPU CYCLE +timedelay_threshold=99900000 + +analyse_tcp_option=1 + +#in linux kernel 2.6.20 and subsequent version, should enable this +use_MESAsleep=0 + +#skip ethernet, if you don't care MAC address +skip_ethernet_layer=0 + +#for dual-stack send rst +skip_not_ip_layer=0 + +#packet use DDP protocol +encapsulate_with_ddp=0 + +maxrandval=65535 +randkeyval=13 + +#(0:pag,1:pcap,2:dumpfile,3:pfring,4:DPDK,5:ppf,6:NPacket,7:qnf,8:N95,9:pcap-dumpfile-list,10:topsec, +#(11:ipfile, 12:marsio4, 13:agent_smith, 14:dpdk_vxlan, 15:marsio_vxlan, 16:pag_marsio +capdatamodlel=12 +forwardpkt=0 +pcapdevice=vxlan_user +#pcapdevice=lo +pcapdevice2= +pcapfilter= +pcap_dumpfile_list=dumpfile.list +senddevice=lo +gateway_mac=00:90:0b:1b:a1:2b + + +#max pending packet num between capture-thread and handle-thread +queue_max_num=2000 + +#-----network connection mode definition----- +# 0: don't send packet, only capture; +# 1: parallel mode with single card; +# 2: serial mode with two card; +# 3: logic serial mode with G device. +net_connection_mode=3 + + +[ShowStatInfo] +showinterval=3 +iknow_listen_port=65500 +platform_log_level=20 +FS_switch=1 +FS_print_switch=1 +FS_server_ip=127.0.0.1 +FS_server_port=8126 +FS_cycle=10 + +[pkt_dump] +pkt_dump_switch=0 +#1:local file; 2:udp socket +pkt_dump_mode=2 +pkt_dump_cmd_port=12345 +pkt_dump_bpf_filter= +pkt_dump_file_root_dir=/dev/shm/pkt_dump234 + +#config 'pkt_dump_total_size' means summation of all files size in 'root_dir', unit:MB. +pkt_dump_total_size=29900 + +#config 'pkt_dump_file_max_size' means MB per thread. +pkt_dump_file_max_size=10000 + +#config 'pkt_dump_thread_seq' means which thread enable this module, 'all' is total running thread. +pkt_dump_thread_seq=all +#pkt_dump_thread_seq=0,1,2,3 diff --git a/roles/mrzcpd/files/mrzcpd-4.3.12.6ee1589-1.el7.x86_64.rpm b/roles/mrzcpd/files/mrzcpd-4.3.12.6ee1589-1.el7.x86_64.rpm new file mode 100644 index 0000000..be52859 Binary files /dev/null and b/roles/mrzcpd/files/mrzcpd-4.3.12.6ee1589-1.el7.x86_64.rpm differ diff --git a/roles/mrzcpd/tasks/main.yml b/roles/mrzcpd/tasks/main.yml index 1461381..e87cf5d 100644 --- a/roles/mrzcpd/tasks/main.yml +++ b/roles/mrzcpd/tasks/main.yml @@ -10,7 +10,7 @@ state: present vars: packages: - - /tmp/ansible_deploy/mrzcpd-4.3.10.497aaf5-1.el7.x86_64.rpm + - /tmp/ansible_deploy/mrzcpd-4.3.12.6ee1589-1.el7.x86_64.rpm - name: "update sysconfig/mrzcpd" template: @@ -33,4 +33,4 @@ template: src: "{{ role_path }}/templates/mrtunnat.conf.inline.j2" dest: /opt/mrzcpd/etc/mrtunnat.conf - when: nic_traffic_mirror is not defined \ No newline at end of file + when: nic_traffic_mirror is not defined diff --git a/roles/mrzcpd/templates/mrglobal.conf.inline.j2 b/roles/mrzcpd/templates/mrglobal.conf.inline.j2 index 17346fb..2868f2b 100644 --- a/roles/mrzcpd/templates/mrglobal.conf.inline.j2 +++ b/roles/mrzcpd/templates/mrglobal.conf.inline.j2 @@ -17,7 +17,7 @@ jumbo_frame=1 max_rx_pkt_len=15360 clear_tx_flags=1 vlan-filter=1 -vlan-id-allow=3811,3812,3821,3822 +vlan-id-allow=1301,1302,2301,2302,1501,1502,2501,2502,1601,1602,2601,2602,1701,1702,2701,2702,1801,1802,2801,2802,1901,1902,2901,2902 vlan-pvid=0 vlan-pvid-mode=0 @@ -42,7 +42,7 @@ promisc=1 [service] # lcore id for i/o service, use comma to split iocore=53,54 -distmode=3 +distmode=2 hashmode=0 [eal] diff --git a/roles/mrzcpd/templates/mrtunnat.conf.inline.j2 b/roles/mrzcpd/templates/mrtunnat.conf.inline.j2 index 8809344..1ff53fb 100644 --- a/roles/mrzcpd/templates/mrtunnat.conf.inline.j2 +++ b/roles/mrzcpd/templates/mrtunnat.conf.inline.j2 @@ -3,14 +3,36 @@ lcore_id=49,50,51,52 appsym=tunnat phydev={{nic_data_incoming.name}} virtdev=vxlan_fwd -nr_max_sessions=131070 -expire_time=180 +nr_max_sessions=524280 +nr_slots=1048576 +expire_time=60 reverse_tunnel=0 use_recent_tunnel=0 +use_tuple4_as_sskey=1 [vlan_flipping] enable=1 -c_router_vlan_id_0=3811 -i_router_vlan_id_0=3812 -c_router_vlan_id_1=3821 -i_router_vlan_id_1=3822 +c_router_vlan_id_0=1301 +i_router_vlan_id_0=1302 +c_router_vlan_id_1=2301 +i_router_vlan_id_1=2302 +c_router_vlan_id_2=1501 +i_router_vlan_id_2=1502 +c_router_vlan_id_3=2501 +i_router_vlan_id_3=2502 +c_router_vlan_id_4=1601 +i_router_vlan_id_4=1602 +c_router_vlan_id_5=2601 +i_router_vlan_id_5=2602 +c_router_vlan_id_6=1701 +i_router_vlan_id_6=1702 +c_router_vlan_id_7=2701 +i_router_vlan_id_7=2702 +c_router_vlan_id_8=1801 +i_router_vlan_id_8=1802 +c_router_vlan_id_9=2801 +i_router_vlan_id_9=2802 +c_router_vlan_id_10=1901 +i_router_vlan_id_10=1902 +c_router_vlan_id_11=2901 +i_router_vlan_id_11=2902 diff --git a/roles/tfe/files/tfe b/roles/tfe/files/tfe new file mode 100644 index 0000000..83118ca Binary files /dev/null and b/roles/tfe/files/tfe differ diff --git a/roles/tfe/files/tfe-kmod-c498d30-1dkms.noarch.rpm b/roles/tfe/files/tfe-kmod-c498d30-1dkms.noarch.rpm new file mode 100644 index 0000000..76fb5b9 Binary files /dev/null and b/roles/tfe/files/tfe-kmod-c498d30-1dkms.noarch.rpm differ diff --git a/roles/tfe/tasks/main.yml b/roles/tfe/tasks/main.yml index 9ec1be9..0f1ef4e 100644 --- a/roles/tfe/tasks/main.yml +++ b/roles/tfe/tasks/main.yml @@ -8,10 +8,24 @@ yum: name: "{{ packages }}" state: present + allow_downgrade: yes vars: packages: - /tmp/ansible_deploy/tfe-4.0.5.348afbc-1.el7.x86_64.rpm - - /tmp/ansible_deploy/tfe-kmod-6ed8c79-1dkms.noarch.rpm + +- name: "hotfix tfe program" + synchronize: + src: "{{ role_path }}/files/tfe" + dest: /home/tsg/tfe/bin/tfe + +#- name: "remove the old tfe-kmod" +# command: rpm -e tfe-kmod + +#- name: "delete the tfe_kmod.ko" +# command: rm -f /lib/modules/5.1.8-1.el7.elrepo.x86_64/extra/tfe_kmod.ko + +#- name: "reinstall the tfe-kmod" +# command: rpm -i /tmp/ansible_deploy/tfe-kmod-c498d30-1dkms.noarch.rpm --force - name: "template tfe-env config" template: @@ -28,6 +42,11 @@ src: "{{ role_path }}/templates/pangu_pxy.conf.j2" dest: /home/tsg/tfe/conf/pangu/pangu_pxy.conf +- name: "deploy qaznet cert" + copy: + src: "{{ role_path }}/files/qaznet_intermedia.pem" + dest: "/home/tsg/tfe/resource/tfe/qaznet_intermedia.pem" + - name: "start tfe-kmod" modprobe: name: tfe_kmod @@ -48,6 +67,7 @@ - name: "bootup tfe" blockinfile: path: /etc/rc.d/rc.local + marker: "## {mark} bootstrap tfe" block: | systemctl start tfe-env cd /home/tsg/tfe; ./r2_tfe diff --git a/roles/tfe/templates/pangu_pxy.conf.j2 b/roles/tfe/templates/pangu_pxy.conf.j2 index a4d1913..7e3d75a 100644 --- a/roles/tfe/templates/pangu_pxy.conf.j2 +++ b/roles/tfe/templates/pangu_pxy.conf.j2 @@ -1,5 +1,5 @@ [debug] -log_level=10 +log_level=30 [log] nic_name= {{ nic_mgr.name }} diff --git a/roles/tfe/templates/tfe.conf.j2 b/roles/tfe/templates/tfe.conf.j2 index e4bd78a..5816ef7 100644 --- a/roles/tfe/templates/tfe.conf.j2 +++ b/roles/tfe/templates/tfe.conf.j2 @@ -29,7 +29,7 @@ mode= {{ tfe.keykeeper.mode }} no_cache=0 cert_store_host= {{ cert_store_server.address }} cert_store_port= {{ cert_store_server.port }} -ca_path=resource/tfe/tango-ca-v3-trust-ca.pem +ca_path=resource/tfe/qaznet_intermedia.pem untrusted_ca_path=resource/tfe/tango-ca-v3-untrust-ca.pem [debug] @@ -54,11 +54,11 @@ tcp_ttl_upstream=75 tcp_ttl_downstream=70 [log] -level=10 +level=30 [stat] statsd_server=192.168.10.72 statsd_port=8126 [http] -loglevel=20 +loglevel=30 diff --git a/site.yml b/site.yml index fc0e42a..048b369 100644 --- a/site.yml +++ b/site.yml @@ -4,6 +4,7 @@ - framework - mrzcpd - kni + - certstore vars_files: - "vars/common.yml" - "vars/kni.yml" @@ -59,9 +60,3 @@ - nic_traffic_mirror: name: ens8f2 use_mrzcpd: 1 - -- hosts: certstore - roles: - - framework - become: yes - become_user: root diff --git a/tfe_clean_core_and_logs.yml b/tfe_clean_core_and_logs.yml new file mode 100644 index 0000000..25eb1a5 --- /dev/null +++ b/tfe_clean_core_and_logs.yml @@ -0,0 +1,21 @@ +- hosts: control-blade-all + tasks: + - name: "killall tfe" + command: "killall tfe" + + - name: "find corefiles to delete" + find: + paths: /home/tsg/tfe/ + patterns: core.* + register: corefiles_to_delete + + - name: "cleanup coredump" + file: + state: absent + path: '{{ item.path }}' + with_items: "{{ corefiles_to_delete.files }}" + + - name: "cleanup RESTART.log" + file: + state: absent + path: /home/tsg/tfe/RESTART.log diff --git a/vars/common.yml b/vars/common.yml index d683254..b30939a 100644 --- a/vars/common.yml +++ b/vars/common.yml @@ -3,7 +3,7 @@ port: 6379 db: 0 - cert_store_server: - address: 10.4.35.1 + address: 192.168.100.1 port: 9991 - log_kafkabrokers: address: "10.4.37.7:9092,10.4.35.8:9092,10.4.35.9:9092" diff --git a/vars/kni.yml b/vars/kni.yml index 7ab7a78..4a5dcc7 100644 --- a/vars/kni.yml +++ b/vars/kni.yml @@ -2,8 +2,8 @@ global: log_level: 30 tfe_node_count: 3 - tfe_data_recv_thread_num: 3 - keepalive_replay_switch: 1 + tfe_data_recv_thread_num: 6 + keepalive_replay_switch: 1 watch_dog: switch: 1 maat: @@ -17,4 +17,4 @@ - tfe1: enabled: 1 - tfe2: - enabled: 1 \ No newline at end of file + enabled: 1 diff --git a/vars/tfe.yml b/vars/tfe.yml index 318f685..5e718c5 100644 --- a/vars/tfe.yml +++ b/vars/tfe.yml @@ -1,5 +1,5 @@ - tfe: - nr_threads: 16 + nr_threads: 32 keykeeper: - mode: "normal" - no_cache : 0 \ No newline at end of file + mode: "debug" + no_cache : 0