策略验证配置文件、rpm包修改

1. 支持虚拟表配置修改
2. 支持tar、rpm打包
3. 修改table_info配置

CMakeLists.txt

@@ -1,7 +1,8 @@
-#cmake_minimum_required(VERSION 3.5)
+cmake_minimum_required(VERSION 3.5)
 project(verify-policy)
 
 set(CMAKE_MODULE_PATH ${PROJECT_SOURCE_DIR}/cmake)
+set(CMAKE_SCRIPT_PATH ${PROJECT_SOURCE_DIR}/script)
 include(Version)
 include(Package)
 
@@ -14,7 +15,7 @@ if(NOT CMAKE_BUILD_TYPE)
 endif()
 
 if (CMAKE_INSTALL_PREFIX_INITIALIZED_TO_DEFAULT)
-	set (CMAKE_INSTALL_PREFIX "/home/mesasoft/tfe" CACHE PATH "default install path" FORCE )
+	set (CMAKE_INSTALL_PREFIX "/home/tsg/verify-policy" CACHE PATH "default install path" FORCE )
 endif()
 
 # Global compile options
@@ -44,6 +45,7 @@ endif()
 
 add_custom_target("install-program" COMMAND ${CMAKE_COMMAND} ARGS -DCOMPONENT=Program -P cmake_install.cmake)
 add_custom_target("install-profile" COMMAND ${CMAKE_COMMAND} ARGS -DCOMPONENT=Profile -P cmake_install.cmake)
+add_custom_target("tarball" COMMAND sh ${CMAKE_SCRIPT_PATH}/tarball.sh ${TARGET_VERSION} ${TARGET_DESCRIBE})
 
 enable_testing()
 #add_subdirectory(conf)

cmake/Package.cmake

@@ -9,8 +9,9 @@ message(STATUS "Package: ${CPACK_PACKAGE_NAME}")
 set(CPACK_PACKAGE_VENDOR "MESASOFT")
 set(CPACK_PACKAGE_VERSION_MAJOR "${VERIFY_POLIC_VERSION_MAJOR}")
 set(CPACK_PACKAGE_VERSION_MINOR "${VERIFY_POLIC_VERSION_MINOR}")
-set(CPACK_PACKAGE_VERSION_PATCH "${VERIFY_POLIC_VERSION_PATCH}.${VERIFY_POLIC_DESCRIBE}")
+set(CPACK_PACKAGE_VERSION_PATCH "${VERIFY_POLIC_VERSION_PATCH}.${TARGET_DESCRIBE}")
 set(CPACK_PACKAGING_INSTALL_PREFIX ${CMAKE_INSTALL_PREFIX})
+message(STATUS "VERSION: ${VERIFY_POLIC_VERSION_MAJOR}")
 
 # RPM Build
 set(CPACK_GENERATOR "RPM")
@@ -19,10 +20,20 @@ set(CPACK_RPM_FILE_NAME "RPM-DEFAULT")
 set(CPACK_RPM_PACKAGE_AUTOREQPROV "no")
 set(CPACK_RPM_PACKAGE_RELEASE_DIST on)
 set(CPACK_RPM_DEBUGINFO_PACKAGE on)
+set(CPACK_RPM_PRE_INSTALL_SCRIPT_FILE ${CMAKE_SOURCE_DIR}/cmake/PreInstall.in)
 #set(CPACK_RPM_POST_INSTALL_SCRIPT_FILE ${CMAKE_SOURCE_DIR}/cmake/PostInstall.in)
 #set(CPACK_RPM_POST_UNINSTALL_SCRIPT_FILE ${CMAKE_SOURCE_DIR}/cmake/PostUninstall.in)
 #set(CPACK_RPM_PRE_UNINSTALL_SCRIPT_FILE ${CMAKE_SOURCE_DIR}/cmake/PreUninstall.in)
 
+set(CMAKE_INSTALL_PREFIX "/home/tsg/verify-policy")
+set(CPACK_RPM_BUILD_SOURCE_DIRS_PREFIX /home/tsg)
+
+install(PROGRAMS build/platform/verify-policy DESTINATION ./bin)
+install(DIRECTORY ./conf DESTINATION ./)
+install(DIRECTORY ./resource DESTINATION ./)
+
+install(FILES script/verify-policy.service DESTINATION /usr/lib/systemd/system/)
+
 # Must uninstall the debug package before install release package
 if(CMAKE_BUILD_TYPE STREQUAL "Debug")
     set(CPACK_RPM_PACKAGE_CONFLICTS "verify-policy")
@@ -31,7 +42,7 @@ else()
 endif()
 
 # setup %config(noreplace)
-set(CPACK_RPM_USER_FILELIST "%config(noreplace) ${CMAKE_INSTALL_PREFIX}/conf/pangu/pangu_pxy.conf"
+set(CPACK_RPM_USER_FILELIST "%config(noreplace) ${CMAKE_INSTALL_PREFIX}/conf/verify_policy.conf"
-							"%config(noreplace) ${CMAKE_INSTALL_PREFIX}/conf/tfe/decrypt_mirror.conf"
+			                 "%config(noreplace) ${CMAKE_INSTALL_PREFIX}/resource/table_info.conf"
-                            "%config(noreplace) ${CMAKE_INSTALL_PREFIX}/conf/tfe/tfe.conf")
+                           "%config(noreplace) ${CMAKE_INSTALL_PREFIX}/resource/dynamic_maat_table_info.conf")
 include(CPack)

cmake/PostInstall.in

@@ -1,2 +0,0 @@
-%systemd_post mrenv.service mrzcpd.service mrtunnat.service
-/sbin/ldconfig

cmake/PostUninstall.in

@@ -1,2 +0,0 @@
-%systemd_postun_with_restart mrenv.service mrzcpd.service mrtunnat.service
-/sbin/ldconfig

cmake/PreUninstall.in

@@ -1 +0,0 @@
-%systemd_preun mrenv.service mrzcpd.service mrtunnat.service

cmake/Version.cmake

@@ -31,8 +31,14 @@ if(NOT VERIFY_POLIC_VERSION_PATCH)
     set(VERIFY_POLIC_VERSION_PATCH 0)
 endif()
 
-set(VERIFY_POLIC_VERSION "${VERIFY_POLIC_VERSION_MAJOR}.${VERIFY_POLIC_VERSION_MINOR}.${VERIFY_POLIC_VERSION_PATCH}")
+set(TARGET_DESCRIBE "${VCS_SHORT_HASH}")
+set(TARGET_VERSION "${VERIFY_POLICY_VERSION_MAJOR}.${VERIFY_POLIC_VERSION_MINOR}.${VERIFY_POLIC_VERSION_PATCH}-${TARGET_DESCRIBE}")
+set(TARGET_GIT_VERSION "${VERIFY_POLICY_VERSION_MAJOR}.${VERIFY_POLIC_VERSION_MINOR}.${VERIFY_POLIC_VERSION_PATCH}-${TARGET_DESCRIBE}")
+
+# Replace .- with _
+string(REGEX REPLACE "[\\.\\-]" "_" TARGET_VAR_VERSION "${TARGET_GIT_VERSION}")
 
 # print information
-message(STATUS "Welcome to Verify Policy Engine, Version: ${VERIFY_POLIC_VERSION}")
+message(STATUS "Welcome to Verify Policy Engine, Version: ${TARGET_VERSION}")
-add_definitions(-DVERIFY_POLIC_VERSION=${VERIFY_POLIC_VERSION})
+add_definitions(-DTARGET_GIT_VERSION=\"${TARGET_GIT_VERSION}\")
+add_definitions(-DTARGET_VAR_VERSION=${TARGET_VAR_VERSION})

common/CMakeLists.txt

@@ -1,4 +1,4 @@
 add_library(common src/verify_policy_logging.cpp)
 target_include_directories(common PUBLIC ${CMAKE_CURRENT_LIST_DIR}/include)
-target_link_libraries(common PUBLIC MESA_handle_logger)
+target_link_libraries(common PUBLIC MESA_handle_logger libevent-static)
 

common/src/verify_policy_logging.cpp

@@ -47,6 +47,10 @@ void * verify_syslog_init(const char *config)
     }else{
         mesa_runtime_log(RLOG_LV_INFO, MODULE_NAME, "Log module initialization");
     }
+
+	mesa_runtime_log(RLOG_LV_INFO, MODULE_NAME, "%s:%d", "Log level", logging_sc_lid.run_log_level);
+	mesa_runtime_log(RLOG_LV_INFO, MODULE_NAME, "%s:%s", "Log Directory", logging_sc_lid.run_log_path);
+
 	return logging_sc_lid.run_log_handle;
 finish:
 	return NULL;

conf/verify-policy.conf

@@ -1,54 +0,0 @@
-[SYSTEM]
-#1:print on screen, 0:don't
-DEBUG_SWITCH = 1
-#10:DEBUG, 20:INFO, 30:FATAL
-RUN_LOG_LEVEL = 10
-RUN_LOG_PATH = ./logs
-[CONFIG]
-#Number of running threads
-thread-nu = 4
-
-[maat]
-# 0:json 1: redis 2: iris
-maat_input_mode=0
-table_info=resource/pangu/table_info.conf
-json_cfg_file=resource/pangu/pangu_http.json
-stat_file=log/pangu_scan.status
-full_cfg_dir=pangu_policy/
-inc_cfg_dir=pangu_policy/
-
-maat_redis_server=192.168.10.31
-maat_redis_port=6379
-maat_redis_db_index=0
-effect_interval_s=1
-accept_tags={"tags":[{"tag":"location","value":"Astana"}]}
-
-[NTC_MAAT]
-#Configure the load mode,
-#0: using the configuration distribution network
-#1: using local json
-#2: using Redis reads
-maat_json_switch=2
-#When the loading mode is sent to the network, set the scanning configuration modification interval (s).
-effective_interval=1
-#Specify the location of the configuration library table file
-table_info=./conf/table_info.conf
-#Incremental profile path
-inc_cfg_dir=./rule/inc/index
-#Full profile path
-full_cfg_dir=./rule/full/index
-#Json file path when json schema is used
-pxy_obj_keyring=./conf/pxy_obj_keyring.json
-[LIBEVENT]
-#Local monitor port number, default is 9991
-port = 9991
-[CERTSTORE_REDIS]
-#The Redis server IP address and port number where the certificate is stored locally
-ip = 127.0.0.1
-port = 6379
-[MAAT_REDIS]
-#Maat monitors the Redsi server IP address and port number
-ip = 192.168.11.243
-port = 6379
-dbindex = 4
-

conf/verify_policy.conf

@@ -0,0 +1,37 @@
+[SYSTEM]
+#1:print on screen, 0:don't
+DEBUG_SWITCH = 1
+#10:DEBUG, 20:INFO, 30:FATAL
+RUN_LOG_LEVEL = 10
+RUN_LOG_PATH = ./logs
+[CONFIG]
+#Number of running threads
+thread-nu = 4
+
+[maat]
+# 0:json 1: redis 2: iris
+maat_input_mode=1
+table_info=./resource/table_info.conf
+json_cfg_file=./resource/pangu_http.json
+stat_file=logs/verify-policy.status
+full_cfg_dir=verify-policy/
+inc_cfg_dir=verify-policy/
+
+maat_redis_server=192.168.40.120
+maat_redis_port_range=7002
+maat_redis_db_index=0
+effect_interval_s=1
+accept_tags={"tags":[{"tag":"location","value":"Astana"}]}
+
+[dynamic_maat]
+maat_input_mode=1
+table_info=./resource/dynamic_maat_table_info.conf
+maat_redis_server=192.168.40.120
+maat_redis_port_range=7002
+maat_redis_db_index=1
+effect_interval_s=1
+
+[LISTEN]
+#Local monitor port number, default is 9991
+port = 9994
+

platform/CMakeLists.txt

@@ -1,3 +1,7 @@
+if(COMMAND cmake_policy)
+	cmake_policy(SET CMP0003 NEW)
+endif(COMMAND cmake_policy)
+
 add_executable(verify-policy src/verify_policy.cpp)
 
 #target_include_directories(verify-policy PUBLIC ${CMAKE_CURRENT_LIST_DIR}/include)

platform/src/verify_policy.cpp

@@ -44,17 +44,15 @@ struct verify_policy_query
 	struct keyword_obj keywords[16];
 };
 
-#if 0
+/* VERSION STRING */
-#ifdef VERIFY_POLIC_VERSION
+#ifdef TARGET_GIT_VERSION
-char *git_version = VERIFY_POLIC_VERSION;
+static __attribute__((__used__)) const char * git_ver = TARGET_GIT_VERSION;
 #else
-char *default_version = "1.1.1";
+static __attribute__((__used__)) const char * git_ver = "1.1";
 #endif
-#endif
-const char *default_version = "1.1.1";
 const char * version()
 {
-	return default_version;
+	return git_ver;
 }
 
 extern int pangu_policy_init(struct verify_proxy * verify, const char* profile_path);
@@ -71,7 +69,8 @@ static int verify_policy_init(struct verify_proxy * verify, const char *profile)
 	if (xret < 0){
 		mesa_runtime_log(RLOG_LV_FATAL, MODULE_NAME, "Listen Port invalid");
 	}
-	mesa_runtime_log(RLOG_LV_INFO, MODULE_NAME, "Listen Port %d", verify->listen_port);
+	mesa_runtime_log(RLOG_LV_INFO, MODULE_NAME, "%s:%d", "The Threads", verify->nr_work_threads);
+	mesa_runtime_log(RLOG_LV_INFO, MODULE_NAME, "%s:%d", "Libevent Port", verify->listen_port);
     return xret;
 }
 
@@ -466,7 +465,7 @@ int main(int argc, char * argv[])
 		switch (opt)
 		{
 			case 'v':
-				fprintf(stderr, "Tango Frontend Engine, Version: %s\n", version());
+				fprintf(stderr, "Welcome to Verify Policy Engine, Version: %s\n", version());
 				return 0;
 			default:
 				break;

resource/dynamic_maat_table_info.conf

@@ -0,0 +1,19 @@
+#each collumn seperate with '\t'
+#id	(0~65535)
+#name string
+#type one of ip,expr,expr_plus,digest,intval,compile or plugin
+#src_charset one of GBK,BIG5,UNICODE,UTF8
+#dst_charset combined by GBK,BIG5,UNICODE,UTF8,seperate with '/'
+#do_merege [yes/no]
+#cross cache [number]
+#quick mode [quickon/quickoff], default [quickoff]
+#For ip/intval/digest/compile/group
+#id	name	type
+#
+#For plugin table
+#id	name	type	json_descr
+#
+#For expr/expr_plus Table
+#id	name	type	src_charset	dst_charset	do_merge cross_cache quick_mode
+
+1	TSG_DYN_SUBSCRIBER_IP	plugin	{"key":3,"valid":5}

resource/table_info.conf

@@ -0,0 +1,49 @@
+#each collumn seperate with '\t'
+#id	(0~65535)
+#name string
+#type one of ip,expr,expr_plus,digest,intval,compile or plugin
+#src_charset one of GBK,BIG5,UNICODE,UTF8
+#dst_charset combined by GBK,BIG5,UNICODE,UTF8,seperate with '/'
+#do_merege [yes/no]
+#cross cache [number]
+#quick mode [quickon/quickoff], default [quickoff]
+#For ip/intval/digest/compile/group
+#id	name	type
+#
+#For plugin table
+#id	name	type	json_descr
+#
+#For expr/expr_plus Table
+#id	name	type	src_charset	dst_charset	do_merge cross_cache quick_mode
+0	PXY_CTRL_COMPILE	compile	escape	--
+1	GROUP_COMPILE_RELATION	group	--
+2	TSG_OBJ_IP_ADDR	ip_plus	---
+3	TSG_OBJ_URL	expr	UTF8	GBK/UNICODE/UTF8/url_encode_gb2312/url_encode_utf8	yes	0	quickoff
+4	TSG_FIELD_HTTP_URL	virtual	TSG_OBJ_URL	--
+5	TSG_OBJ_FQDN	expr	UTF8	GBK/UNICODE/UTF8/url_encode_gb2312/url_encode_utf8	yes	0	quickoff
+5	TSG_OBJ_FQDN_CAT	expr	UTF8	GBK/UNICODE/UTF8/url_encode_gb2312/url_encode_utf8	yes	0	quickoff
+6	TSG_FIELD_HTTP_HOST	virtual	TSG_OBJ_FQDN	--
+7	TSG_OBJ_HTTP_SIGNATURE	expr_plus	UTF8	UTF8	yes	0	quickoff
+8	TSG_FIELD_HTTP_REQ_HDR	virtual	TSG_OBJ_HTTP_SIGNATURE	--
+9	TSG_FIELD_HTTP_RES_HDR	virtual	TSG_OBJ_HTTP_SIGNATURE	--
+10	TSG_OBJ_KEYWORDS expr	UTF8	GBK/UNICODE/UTF8	yes	128	quickoff
+11	TSG_FIELD_HTTP_REQ_CONTENT	virtual	TSG_OBJ_KEYWORDS --
+12	TSG_FIELD_HTTP_RES_CONTENT	virtual	TSG_OBJ_KEYWORDS --
+13	TSG_OBJ_SUBSCRIBER_ID	expr	UTF8	UTF8	yes	0	quickon
+14  TSG_FIELD_SSL_SNI       virtual TSG_OBJ_FQDN    --
+15  TSG_FIELD_SSL_CN        virtual TSG_OBJ_FQDN    --
+16  TSG_FIELD_SSL_SAN       virtual TSG_OBJ_FQDN    --
+17  TSG_FIELD_DNS_QNAME     virtual TSG_OBJ_FQDN    --
+18  TSG_OBJ_ACCOUNT expr    UTF8    UTF8    yes     0
+19  TSG_FIELD_MAIL_ACCOUNT  virtual TSG_OBJ_ACCOUNT --
+20  TSG_FIELD_MAIL_FROM     virtual TSG_OBJ_ACCOUNT --
+21  TSG_FIELD_MAIL_TO       virtual TSG_OBJ_ACCOUNT --
+22  TSG_FIELD_MAIL_SUBJECT  virtual TSG_OBJ_KEYWORDS        --
+23  TSG_FIELD_MAIL_CONTENT  virtual TSG_OBJ_KEYWORDS        --
+24  TSG_FIELD_MAIL_ATT_NAME virtual TSG_OBJ_KEYWORDS        --
+25  TSG_FIELD_MAIL_ATT_CONTENT      virtual TSG_OBJ_KEYWORDS        --
+26  TSG_FIELD_FTP_URI       virtual TSG_OBJ_URL     --
+27  TSG_FIELD_FTP_CONTENT   virtual TSG_OBJ_KEYWORDS        --
+28  TSG_FIELD_FTP_ACCOUNT   virtual TSG_OBJ_ACCOUNT --
+
+

scan/src/pangu_http.cpp

@@ -342,7 +342,9 @@ static Maat_feather_t create_maat_feather(const char * instance_name, const char
 	MESA_load_profile_string_def(profile, section, "json_cfg_file", json_cfg_file, sizeof(json_cfg_file), "");
 
 	MESA_load_profile_string_def(profile, section, "maat_redis_server", redis_server, sizeof(redis_server), "");
+	mesa_runtime_log(RLOG_LV_INFO, MODULE_NAME, "%s:%s", "Maat Redis Ip", redis_server);
 	MESA_load_profile_string_def(profile, section, "maat_redis_port_range", redis_port_range, sizeof(redis_server), "6379");
+	mesa_runtime_log(RLOG_LV_INFO, MODULE_NAME, "%s:%s", "Maat Redis Port", redis_port_range);
 	ret=sscanf(redis_port_range,"%d-%d", &redis_port_begin, &redis_port_end);
 	if(ret==1)
 	{
@@ -473,14 +475,14 @@ int pangu_policy_init(struct verify_proxy * verify, const char* profile_path)
 	}
 
 	const char * table_name[__SCAN_TABLE_MAX];
-	table_name[PXY_CTRL_IP] = "PXY_CTRL_IP";
+	table_name[PXY_CTRL_IP] = "TSG_OBJ_IP_ADDR";
-	table_name[PXY_CTRL_HTTP_URL] = "PXY_CTRL_HTTP_URL";
+	table_name[PXY_CTRL_HTTP_URL] = "TSG_FIELD_HTTP_URL";
-	table_name[PXY_CTRL_HTTP_FQDN] = "TSG_OBJ_FQDN";
+	table_name[PXY_CTRL_HTTP_FQDN] = "TSG_FIELD_HTTP_HOST";
-	table_name[PXY_CTRL_HTTP_REQ_HDR] = "PXY_CTRL_HTTP_REQ_HDR";
+	table_name[PXY_CTRL_HTTP_REQ_HDR] = "TSG_FIELD_HTTP_REQ_HDR";
-	table_name[PXY_CTRL_HTTP_REQ_BODY] = "TSG_OBJ_CONTENT";
+	table_name[PXY_CTRL_HTTP_REQ_BODY] = "TSG_FIELD_HTTP_REQ_CONTENT";
-	table_name[PXY_CTRL_HTTP_RES_HDR] = "PXY_CTRL_HTTP_RES_HDR";
+	table_name[PXY_CTRL_HTTP_RES_HDR] = "TSG_FIELD_HTTP_RES_HDR";
-	table_name[PXY_CTRL_HTTP_RES_BODY] = "TSG_OBJ_CONTENT";
+	table_name[PXY_CTRL_HTTP_RES_BODY] = "TSG_FIELD_HTTP_RES_CONTENT";
-	table_name[PXY_CTRL_SUBSCRIBE_ID] = "PXY_CTRL_SUBSCRIBE_ID";
+	table_name[PXY_CTRL_SUBSCRIBE_ID] = "TSG_OBJ_SUBSCRIBER_ID";
 	for (int i = 0; i < __SCAN_TABLE_MAX; i++)
 	{
 		g_pangu_rt->scan_table_id[i] = Maat_table_register(g_pangu_rt->maat, table_name[i]);

script/tarball.sh

@@ -0,0 +1,36 @@
+X=verify-policy
+#X=certstore
+
+typeset -l version
+version=`lsb_release -i -s`
+version_id=`lsb_release -r -s`
+machine=`uname -m`
+issue_date=`date "+%Y%m%d"`
+
+tarball="$X-$2-$version$version_id-$machine-$issue_date".tar.gz
+
+do_mkdir(){
+    if [ ! -d "../package" ]; then
+        mkdir ../package
+    fi
+    cd ../package
+    rm $X -rf
+    mkdir $X
+    mkdir $X/bin
+}
+
+do_copy(){
+    cp ../resource/ $X -rf
+    cp ../build/platform/verify-policy $X/bin
+    cp ../conf $X -rf
+}
+
+do_tarball(){
+    tar -zcvf $1 $X
+    rm $X -rf
+    echo "`date`  BUILD=$X  COMMIT=$2 ARCHIVE=$tarball" >> release.log
+}
+
+do_mkdir
+do_copy $1
+do_tarball $tarball

script/verify-policy.service

@@ -0,0 +1,12 @@
+[Unit]
+Description=Verify Policy Engine
+After=redis.service
+
+[Service]
+WorkingDirectory=/home/tsg/verify-policy
+ExecStart=/home/tsg/verify-policy/bin/verify-policy
+Restart=always
+RestartSec=5s
+
+[Install]
+WantedBy=multi-user.target