修改配置文件模板和运行环境相关的变量生成新的配置模板

This commit is contained in:
fumingwei
2021-05-08 17:27:23 +08:00
parent b983669beb
commit dc7dfe8de0
13 changed files with 162 additions and 98 deletions

View File

@@ -8,8 +8,9 @@ disable_coredump=0
enable_breakpad=1
breakpad_minidump_dir=/tmp/certstore/crashreport
enable_breakpad_upload=1
breakpad_upload_url= http://127.0.0.1:9000/api/2/minidump/?sentry_key=3556bac347c74585a994eb6823faf5c6
{% raw %}
breakpad_upload_url= {{ breakpad_upload_url }}
{% endraw %}
[CONFIG]
#Number of running threads
thread-nu = 4
@@ -50,9 +51,11 @@ port = 6379
[MAAT_REDIS]
#Maat monitors the Redsi server IP address and port number
ip = 192.168.44.72
port = 7002
dbindex = 0
{% raw %}
ip = {{ maat_redis_server.address }}
port = {{ maat_redis_server.port }}
dbindex = {{ maat_redis_server.db }}
{% endraw %}
[stat]
statsd_server=127.0.0.1
statsd_port=8100

View File

@@ -6,4 +6,4 @@ INFO=20
FATAL=30
[rules]
*.fatal "./logs/error.log.%d(%F)";
*.30 "./logs/certstore.log.%d(%F)"
*.fatal "./logs/certstore.log.%d(%F)"

View File

@@ -6,20 +6,26 @@ PERF_SWITCH=1
TABLE_INFO=conf/capture_packet_tableinfo.conf
STAT_FILE=capture_packet_maat.status
EFFECT_INTERVAL_S=1
REDIS_IP=192.168.44.72
REDIS_PORT_NUM=1
REDIS_PORT=7002
REDIS_INDEX=0
{% raw %}
REDIS_IP={{ maat_redis_server.address }}
REDIS_PORT_NUM={{ maat_redis_server.port_num }}
REDIS_PORT={{ maat_redis_server.port }}
REDIS_INDEX={{ maat_redis_server.db }}
{% endraw %}
JSON_CFG_FILE=conf/capture_packet_maat.json
INC_CFG_DIR=capture_packet_rule/inc/index/
FULL_CFG_DIR=capture_packet_rule/full/index/
EFFECTIVE_RANGE_FILE=/opt/tsg/etc/tsg_device_tag.json
ACCEPT_TAGS={"tags":[{"tag":"data_center","value":"Beijing"}]}
{% raw %}
ACCEPT_TAGS={"tags":[{"tag":"data_center","value":"{{ data_center }}"}]}
{% endraw %}
[LOG]
NIC_NAME=enp131s0f3
BROKER_LIST=192.168.44.12:9092
{% raw %}
BROKER_LIST={{ log_kafkabrokers.address | join(",") }}
{% endraw %}
FIELD_FILE=conf/capture_packet_log_field.conf
[SYSTEM]

View File

@@ -6,10 +6,12 @@ PERF_SWITCH=1
TABLE_INFO=tsgconf/tsg_static_tableinfo.conf
STAT_FILE=tsg_static_maat.status
EFFECT_INTERVAL_S=1
REDIS_IP=192.168.44.72
REDIS_PORT_NUM=1
REDIS_PORT=7002
REDIS_INDEX=0
{% raw %}
REDIS_IP={{ maat_redis_server.address }}
REDIS_PORT_NUM={{ maat_redis_server.port_num }}
REDIS_PORT={{ maat_redis_server.port }}
REDIS_INDEX={{ maat_redis_server.db }}
{% endraw %}
JSON_CFG_FILE=tsgconf/tsg_maat.json
INC_CFG_DIR=tsgrule/inc/index/
FULL_CFG_DIR=tsgrule/full/index/
@@ -23,10 +25,12 @@ PERF_SWITCH=1
TABLE_INFO=tsgconf/tsg_dynamic_tableinfo.conf
STAT_FILE=tsg_dynamic_maat.status
EFFECT_INTERVAL_S=1
REDIS_IP=192.168.44.72
REDIS_PORT_NUM=1
REDIS_PORT=7002
REDIS_INDEX=1
{% raw %}
REDIS_IP={{ dynamic_maat_redis_server.address }}
REDIS_PORT_NUM={{ dynamic_maat_redis_server.port_num }}
REDIS_PORT={{ dynamic_maat_redis_server.port }}
REDIS_INDEX={{ dynamic_maat_redis_server.db }}
{% endraw %}
JSON_CFG_FILE=tsgconf/tsg_maat.json
INC_CFG_DIR=tsgrule/inc/index/
FULL_CFG_DIR=tsgrule/full/index/
@@ -39,14 +43,18 @@ PERF_SWITCH=1
TABLE_INFO=tsgconf/app_sketch_tableinfo.conf
STAT_FILE=app_sketch_maat.status
EFFECT_INTERVAL_S=1
REDIS_IP=192.168.44.72
REDIS_PORT_NUM=1
REDIS_PORT=7002
REDIS_INDEX=0
{% raw %}
REDIS_IP={{ maat_redis_server.address }}
REDIS_PORT_NUM={{ maat_redis_server.port_num }}
REDIS_PORT={{ maat_redis_server.port }}
REDIS_INDEX={{ maat_redis_server.db }}
{% endraw %}
JSON_CFG_FILE=tsgconf/app_sketch_maat.json
INC_CFG_DIR=tsgrule/inc/index/
FULL_CFG_DIR=tsgrule/full/index/
EFFECTIVE_RANGE_FILE=/opt/tsg/etc/tsg_device_tag.json
[MAAT]
ACCEPT_TAGS={"tags":[{"tag":"data_center","value":"Beijing"}]}
{% raw %}
ACCEPT_TAGS={"tags":[{"tag":"data_center","value":"{{ data_center }}"}]}
{% endraw %}

View File

@@ -43,7 +43,9 @@ NIC_NAME="enp131s0f3"
MAX_SERVICE=1
LOG_LEVEL=30
LOG_PATH="./tsglog/tsglog"
BROKER_LIST="192.168.44.12:9092"
{% raw %}
BROKER_LIST="{{ log_kafkabrokers.address | join(",") }}"
{% endraw %}
COMMON_FIELD_FILE="tsgconf/tsg_log_field.conf"
[STATISTIC]
@@ -62,7 +64,9 @@ APP_NAME="tsg_master"
[SYSTEM]
NIC_NAME="enp131s0f3"
ENTRANCE_ID=0
{% raw %}
ENTRANCE_ID={{ tsg_master_entrance_id }}
{% endraw %}
LOG_LEVEL=30
LOG_PATH="./tsglog/tsg_master"
POLICY_PRIORITY_LABEL="POLICY_PRIORITY"
@@ -73,8 +77,10 @@ log_service=2
[HOS_CONF]
hos_serverip="192.168.40.223"
hos_serverport=9098
{% raw %}
hos_serverip="{{ firewall.hos_serverip }}"
hos_serverport={{ firewall.hos_serverport }}
{% endraw %}
hos_accesskeyid="default"
hos_secretkey="default"
hos_poolsize=100
@@ -92,4 +98,6 @@ L7_PROTOCOL_LABEL="BASIC_PROTO_LABEL"
QOS=1
PUBLISH_TOPIC="APP_SIGNATURE_ID"
#CLIENT_ID=
BROKER_LIST="tcp://192.168.40.161:1883"
{% raw %}
BROKER_LIST="{{ firewall.APP_SKETCH_BROKER_LIST }}"
{% endraw %}

View File

@@ -0,0 +1,7 @@
#layer name definition: ipv4, ipv6, ethernet,vlan, arp, gre, mpls, pppoe, tcp, udp, l2tp, ppp, pptp, gtp
#pattern: asymmetric_layer_name[layer index]
vlan[*]
mpls[*]
gre[*]
gtp[*]

View File

@@ -0,0 +1,3 @@
#layer name definition: ipv4, ipv6, ethernet,vlan, arp, gre, mpls, pppoe, tcp, udp, l2tp, ppp, pptp, gtp
#pattern: asymmetric_layer_name under_of_this_asymmetric_layer[layer_index] upper_of_this_asymmetric_layer[layer_index]
#

View File

@@ -1,12 +1,11 @@
[platform]
#./plug/platform/g_device_plug/g_device_plug.inf
#./plug/platform/http_healthcheck/http_healthcheck.inf
#./plug/platform/app_proto_identify/app_proto_identify.inf
./plug/platform/app_proto_identify/app_proto_identify.inf
./plug/platform/tsg_master/tsg_master.inf
#./plug/platform/app_master/app_master.inf
./plug/platform/app_master/app_master.inf
[protocol]
./plug/protocol/mesa_sip/mesa_sip.inf
./plug/protocol/rtp/rtp.inf
./plug/protocol/ssl/ssl.inf
./plug/protocol/http/http.inf
./plug/protocol/dns/dns.inf
@@ -16,8 +15,8 @@
./plug/protocol/l2tp_protocol_plug/l2tp_protocol_plug.inf
[business]
#./plug/business/tsg_conn_sketch/tsg_conn_sketch.inf
#./plug/business/capture_packet_plug/capture_packet_plug.inf
./plug/business/tsg_conn_sketch/tsg_conn_sketch.inf
./plug/business/capture_packet_plug/capture_packet_plug.inf
./plug/business/kni/kni.inf
./plug/business/fw_ssl_plug/fw_ssl_plug.inf
./plug/business/fw_http_plug/fw_http_plug.inf
@@ -25,6 +24,6 @@
./plug/business/fw_mail_plug/fw_mail_plug.inf
./plug/business/fw_ftp_plug/fw_ftp_plug.inf
./plug/business/fw_quic_plug/fw_quic_plug.inf
#./plug/business/conn_telemetry/conn_telemetry.inf
#./plug/business/app_sketch_local/app_sketch_local.inf
./plug/business/fw_voip_plug/fw_voip_plug.inf
./plug/business/conn_telemetry/conn_telemetry.inf
./plug/business/app_sketch_local/app_sketch_local.inf

View File

@@ -15,6 +15,7 @@ LimitCORE=0
TasksMax=infinity
Delegate=yes
KillMode=process
WatchdogSec=10s
[Install]
WantedBy=multi-user.target

View File

@@ -1,7 +1,7 @@
###################################################################################################
# NOTE:
# The format of this file is toml (https://github.com/cktan/tomlc99)
# to make vim editor display colorful and human readable,
# to make vim editor display colorful and human readable,
# you can create a symbolic links named sapp.ini to sapp.toml, ln -sf sapp.toml sapp.ini
###################################################################################################
@@ -22,26 +22,26 @@ dictator_enable=0
[PACKET_IO]
[overlay_tunnel_definition]
### note, since 2020-10-01, L2-L3 tunnel(VLAN,MPLS,PPPOE,etc.) is process and offload by mrtunnat,
### note, since 2020-10-01, L2-L3 tunnel(VLAN,MPLS,PPPOE,etc.) is process and offload by mrtunnat,
### after 2020-10-01, sapp support L2-L3 tunnel(VLAN,MPLS,PPPOE,etc.) without mrtunnat.
l2_l3_tunnel_support=1
### note, optional value is [none, vxlan]
overlay_mode=vxlan
stream_compare_layer_cfg_file="etc/stream_compare_layer.conf"
stream_compare_layer_cfg_file="etc/stream_compare_layer.conf"
vlan_flipping_cfg_file="etc/vlan_flipping_map.conf"
asymmetric_presence_layer_cfg_file="etc/asymmetric_presence_layer.conf"
asymmetric_addr_layer_cfg_file="etc/asymmetric_addr_layer.conf"
prune_inject_layer_cfg_file="etc/prune_inject_layer.conf"
[packet_io.feature]
### note, used to represent inbound or outbound direction value,
### note, used to represent inbound or outbound direction value,
### because it comes from Third party device, so it needs to be specified manually,
### if inbound_route_dir=1, then outbound_route_dir=0, vice versa,
### in other words, outbound_route_dir = 1 ^ inbound_route_dir;
inbound_route_dir=1
### note, BSD_packet_filter, if you do not want to set any filter rule, keep it empty as ""
BSD_packet_filter=""
@@ -52,8 +52,8 @@ dictator_enable=0
### note, depolyment.mode options: [sys_route, vxlan_by_inline_device, raw_ethernet_single_gateway, raw_ethernet_multi_gateway]
### sys_route: send ip(ipv6) packet by system route table, this is default mode in mirror mode;
### vxlan_by_inline_device: encapsulation inject packet with vxlan, and then send to inline device by udp socket.
### raw_ethernet_single_gateway: send layer2 ethernet packet to specific gateway in same broadcast domain.
### raw_ethernet_multi_gateway: send layer2 ethernet packet to multiple gateway in same broadcast domain.
### raw_ethernet_single_gateway: send layer2 ethernet packet to specific gateway in same broadcast domain.
### raw_ethernet_multi_gateway: send layer2 ethernet packet to multiple gateway in same broadcast domain.
inject_pkt_mode=sys_route
### note, this config is valid if inject_pkt_mode==vxlan_by_inline_device, means udp socket src port.
@@ -65,10 +65,10 @@ dictator_enable=0
inject_mode_single_gateway_src_mac="00:11:22:77:88:99"
inject_mode_single_gateway_dst_mac="00:11:22:33:44:55"
dumpfile_sleep_time_before_exit=3
### note, depolyment.mode options: [mirror, inline, transparent]
[packet_io.depolyment]
mode=inline
mode=inline
### note, interface.type options: [pag,pcap,marsio]
[packet_io.internal.interface]
@@ -80,21 +80,23 @@ dictator_enable=0
name=lo
[packet_io.polling]
### note, polling_priority = call sapp_recv_pkt every call polling_entry times,
### note, polling_priority = call sapp_recv_pkt every call polling_entry times,
polling_priority=1
[PROTOCOL_FEATURE]
ipv6_decapsulation_enabled=1
ipv6_send_packet_enabled=1
tcp_drop_pure_ack_pkt=0
tcp_syn_option_parse_enabled=1
tcp_syn_option_parse_enabled=1
skip_not_ip_layer_over_eth=0
treat_vlan_as_mac_in_mac=0
reverse_ethernet_addr=1
[DUPLICATE_PKT]
duplicate_pkt_distinguish=0
[STREAM]
### note, stream_id_base_time format is "%Y-%m-%d %H:%M:%S"
### note, stream_id_base_time format is "%Y-%m-%d %H:%M:%S"
stream_id_base_time="2018-08-08 08:00:00"
[stream.tcp]
max=100000
@@ -106,10 +108,10 @@ dictator_enable=0
meaningful_statistics_minimum_pkt=3
meaningful_statistics_minimum_byte=5
[stream.tcp.inject]
link_mss=1460
[stream.tcp.inject.rst]
auto_remedy=0
number=3
@@ -117,31 +119,31 @@ dictator_enable=0
signature_seed1=65535
signature_seed2=13
remedy_kill_tcp_by_inline_device=0
[stream.udp]
max=100000
timeout=60
meaningful_statistics_minimum_pkt=3
meaningful_statistics_minimum_byte=5
[PROFILING]
[profiling.pkt_latency]
enabled=0
### note, threshold unit is microseconds (us)
threshold=1000000
[profiling.sanity_check]
raw_pkt_broken_enabled=0
symbol_conflict_enabled=0
[profiling.log]
level=40
level=10
interval=5
[profiling.log.local]
enabled=1
### note, if "file_truncate_open_enabled=1", file will be truncated, otherwise open the file for appending.
### note, if "file_truncate_open_enabled=1", file will be truncated, otherwise open the file for appending.
file_truncate_enabled = 1
log_file_name = "fs2_sysinfo.log"
log_conf_name = "etc/sapp_log.conf"
@@ -151,31 +153,31 @@ dictator_enable=0
server_port=8100
[profiling.log.remote.field_stat2]
### note, is valid when "remote_send_out_type=field_stat2"
### note, is valid when "remote_send_out_type=field_stat2"
### note, metric_type option value: [default, json]
metric_type = default
app_name=sapp
[profiling.log.prometheus]
prometheus_enabled=1
prometheus_port=9273
prometheus_url_path="/metrics"
[TOOLS]
[tools.pkt_dump]
enabled=1
### note, mode options value:[storage, udp_socket]
mode=udp_socket
BSD_packet_filter=""
[tools.pkt_dump.threads]
### note, if you want enable pkt dump in all thread, set dump_thread_all_enabled=1, then 'dump_thread_id' is obsoleted.
### if dump_thread_all_enabled=0, then use dump_thread_id to specify separate specified thread index.
all_threads_enabled=1
### note, dump_thread_id start from 0, max is CPU.worker_threads-1
dump_thread_id=[0,1,2,3,4]
[tools.pkt_dump.udp]
command_port=9345
@@ -183,12 +185,12 @@ dictator_enable=0
### note, file path must be double quotation mark extension, for example, path="/dev/shm/pkt_dump"
path="/dev/shm/pkt_dump"
### note, file size unit: MB
file_size_max_per_thread=10000
file_size_max_per_thread=10000
### note:
### These configurations format is complex and difficult to describe with toml grammar,
### so, create a Independent config file to description specific information.
[SPECIAL_CONFIG_LINK]
[SPECIAL_CONFIG_LINK]
project_list_path="./etc/project_list.conf"
plugin_path="./etc/plugin.conf"
entrylist_path="./etc/entrylist.conf"
@@ -200,4 +202,6 @@ dictator_enable=0
enable_breakpad=1
breakpad_minidump_dir="/tmp/crashreport"
enable_breakpad_upload=1
breakpad_upload_url="http://127.0.0.1:9000/api/2/minidump/?sentry_key=3556bac347c74585a994eb6823faf5c6"
{% raw %}
breakpad_upload_url="{{ breakpad_upload_url }}"
{% endraw %}

View File

@@ -22,7 +22,9 @@
rotation_max_archives = 5
[[outputs.kafka]]
brokers = [ "192.168.44.12:9092" ]
{% raw %}
brokers = [ "{{ log_kafkabrokers.address | join("\",\"") }}" ]
{% endraw %}
topic = "TRAFFIC-METRICS-LOG"
data_format = "json"
[[inputs.tcp_listener]]

View File

@@ -2,13 +2,15 @@
enable_plugin=1
[log]
entrance_id=0
# default 1, if enable "en_sendlog", the iterm "tfe.conf [kafka] enable" must set 1
en_sendlog=1
entrance_id=0
#Addresses of hos, Bucket name in hos. Format is defined by WiredLB.
cache_ip_list=192.168.44.67;
cache_listen_port=9098
{% raw %}
cache_ip_list = {{ pangu_pxy.log_cache.address }}
cache_listen_port = {{ pangu_pxy.log_cache.port }}
{% endraw %}
cache_bucket_name=hos/proxy_hos_bucket
cache_token=c21f969b5f03d33d43e04f8f136e7682
@@ -17,21 +19,23 @@ max_used_memroy_size_mb=5120
cache_default_ttl_second=3600
cache_object_key_hash_switch=1
#Refer to the pangu_cahche definition
cache_store_object_way=0
redis_cache_object_size=1024000
redis_cluster_addrs=10.4.20.211:9001,10.4.20.212:9001,10.4.20.213:9001,10.4.20.214:9001,10.4.20.215:9001,10.4.20.216:9001,10.4.20.217:9001,10.4.20.218:9001
#If CACHE_STORE_OBJECT_WAY is not 0, we will use redis to store meta and object.
redis_cluster_ip_list=192.168.10.62-63;
redis_cluster_port_range=6379
#wired load balancer configuration
#Configs of WiredLB for Minios load balancer.
wiredlb_override=1
wiredlb_health_port=42310
wiredlb_topic=MinioFileLog
wiredlb_datacenter=k18consul-tse
wiredlb_health_port=52102
wiredlb_health_port=8560
wiredlb_group=FileLog
log_fsstat_appname=tango_log_file
log_fsstat_filepath=./log/tango_log_file.fs2
log_fsstat_filepath=./tango_log_file.fs
log_fsstat_interval=10
log_fsstat_trig=1
log_fsstat_dst_ip=10.4.20.202
@@ -41,17 +45,18 @@ log_fsstat_dst_port=8125
#hijack flow control
enable=0
token_name=ratelimit
redis_server=192.168.40.137
redis_port=6379
redis_db_index=5
{% raw %}
redis_server={{ maat_redis_server.address }}
redis_port={{ maat_redis_server.port }}
redis_db_index=6
{% endraw %}
[tango_cache]
enable_cache=0
min_cache_obj_size=512
#hos ip, as wiredlb required
cache_ip_list=10.4.35.1-14;
cache_ip_list=192.168.10.61-64;
cache_listen_port=9000
cache_bucket_name=proxybucket
cache_bucket_name=hos/proxy_hos_bucket
cache_token=c21f969b5f03d33d43e04f8f136e7682
max_cnnt_pipeline_num=20
#Maximum size of memory used by tango_cache_client. Upload will fail if the current size of memory used exceeds this value.
@@ -65,12 +70,22 @@ cache_store_object_way=2
#If CACHE_STORE_OBJECT_WAY is 2 and the size of a object is not bigger than this value, object will be stored in redis.
redis_cache_object_size=102400
#If CACHE_STORE_OBJECT_WAY is not 0, we will use redis to store meta and object.
redis_cluster_addrs=10.4.35.15:9001,10.4.35.16:9001,10.4.35.17:9001,10.4.35.18:9001,10.4.35.19:9001,10.4.35.20:9001,10.4.35.21:9001,10.4.35.22:9001,10.4.35.23:9001,10.4.35.24:9001,10.4.35.25:9001,10.4.35.26:9001,10.4.35.27:9001,10.4.35.28:9001,10.4.35.29:9001,10.4.35.30:9001,10.4.35.31:9001,10.4.35.32:9001
redis_cluster_ip_list=192.168.10.62-63;
redis_cluster_port_range=6379
#Configs of WiredLB for Minios load balancer.Refer to the definition at log
cache_undefined_obj=1
query_undefined_obj=0
statsd_server=10.4.20.201
statsd_port=8125
histogram_bins=0.20,0.40,0.6,0.8
statsd_server=192.168.10.72
statsd_port=8126
histogram_bins=0.20,0.40,0.6,0.8
log_fsstat_appname=tango_cache
log_fsstat_filepath=./tango_cache_client.fs
log_fsstat_interval=10
log_fsstat_trig=1
log_fsstat_dst_ip=10.4.20.201
log_fsstat_dst_port=8125

View File

@@ -7,7 +7,9 @@ enable_kni_v2=1
disable_coredump=0
enable_breakpad=1
enable_breakpad_upload=1
breakpad_upload_url=http://127.0.0.1:9000/api/2/minidump/?sentry_key=3556bac347c74585a994eb6823faf5c6
{% raw %}
breakpad_upload_url={{ breakpad_upload_url }}
{% endraw %}
# must be /run/tfe/crashreportdue to tmpfile limit
breakpad_minidump_dir=/run/tfe/crashreport
@@ -80,7 +82,9 @@ key_log_file=log/sslkeylog.log
# mid cert cache
mc_cache_enable=1
mc_cache_eth=enp131s0f2
mc_cache_broker_list=192.168.44.12:9092
{% raw %}
mc_cache_broker_list={{ log_kafkabrokers.address | join(",") }}
{% endraw %}
mc_cache_topic=PXY-EXCH-INTERMEDIA-CERT
ssl_ja3_table=PXY_SSL_FINGERPRINT
@@ -150,7 +154,9 @@ stat_file=log/traffic_mirror.status
[kafka]
enable=1
NIC_NAME=enp131s0f2
kafka_brokerlist=192.168.44.12:9092
{% raw %}
kafka_brokerlist={{ log_kafkabrokers.address | join(",") }}
{% endraw %}
kafka_topic=PROXY-EVENT-LOG
device_id_filepath=/opt/tsg/etc/tsg_sn.json
@@ -169,9 +175,11 @@ deferred_load_on=0
json_cfg_file=resource/pangu/pangu_http.json
# redis mode conf iterm
maat_redis_server=192.168.44.72
maat_redis_port_range=7002
maat_redis_db_index=0
{% raw %}
maat_redis_server={{ maat_redis_server.address }}
maat_redis_port_range={{ maat_redis_server.port }}
maat_redis_db_index={{ maat_redis_server.db }}
{% endraw %}
# iris mode conf iterm
full_cfg_dir=pangu_policy/full/index/