Merge branch 'feature-intval-plus' into 'develop-version3'

Feature intval plus

See merge request MESA_framework/maat!48

.gitlab-ci.yml

@@ -70,6 +70,22 @@ develop_build_release:
     - /^master.*$/i
 
 
+release_build_debug:
+  stage: build
+  variables:
+    BUILD_TYPE: Debug
+    PACKAGE: 1
+    UPLOAD: 1
+    PULP3_REPO_NAME: framework-stable-x86_64.el7
+    PULP3_DIST_NAME: framework-stable-x86_64.el7
+  extends: .build_by_travis
+  artifacts:
+    name: "maatframe-$CI_COMMIT_REF_NAME-debug"
+    paths:
+      - build/*.rpm
+  only:
+    - tags
+
 release_build_release:
   stage: build
   variables:
@@ -85,20 +101,3 @@ release_build_release:
       - build/*.rpm
   only:
     - tags
-
-release_build_release_devel:
-  stage: build
-  variables:
-    BUILD_TYPE: RelWithDebInfo
-    ENABLE_DEVEL_SWITCH: "ON"
-    PACKAGE: 1
-    UPLOAD: 1
-    PULP3_REPO_NAME: framework-stable-x86_64.el7
-    PULP3_DIST_NAME: framework-stable-x86_64.el7
-  extends: .build_by_travis
-  artifacts:
-    name: "maatframe-$CI_COMMIT_REF_NAME-release"
-    paths:
-      - build/*.rpm
-  only:
-    - tags

cmake/Package.cmake

@@ -14,6 +14,10 @@ set(CPACK_PACKAGING_INSTALL_PREFIX ${CMAKE_INSTALL_PREFIX})
 set(CPACK_PACKAGE_VERSION "${VERSION_MAJOR}.${VERSION_MINOR}.${VERSION_PATCH}.${VERSION_BUILD}")
 execute_process(COMMAND bash -c "echo -ne \"`uname -r | awk -F'.' '{print $5\".\"$6\".\"$7}'`\"" OUTPUT_VARIABLE SYSTEM_VERSION)
 
+execute_process(COMMAND sh changelog.sh WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}/cmake)
+SET(CPACK_RPM_CHANGELOG_FILE ${PROJECT_SOURCE_DIR}/cmake/changelog.txt)
+
+
 # RPM Build
 set(CPACK_GENERATOR "RPM")
 set(CPACK_RPM_PACKAGE_VENDOR "MESA")

cmake/changelog.sh

@@ -0,0 +1,4 @@
+#!/bin/sh
+
+branch=`git status | grep branch | awk '{print $NF}'`
+git log --branches=$branch --no-merges --date=local --show-signature --pretty="* %ad %an %ae %nhash: %H%ncommit:%n%B"  | awk -F"-" '{print "- "$0}' | sed 's/- \*/\*/g' | sed 's/- $//g' | sed 's/-/  -/g' | sed 's/[0-9]\{2\}:[0-9]\{2\}:[0-9]\{2\}//g' > changelog.txt

inc/Maat_command.h

@@ -78,6 +78,7 @@ struct Maat_rgn_addr_t
 };
 struct Maat_rgn_intv_t
 {
+	const char *district;// optional for expr_plus, otherwise set to NULL.
 	unsigned int low_boundary;
 	unsigned int up_boundary;
 };

src/entry/Maat_api.cpp

@@ -1381,6 +1381,13 @@ int Maat_scan_intval(Maat_feather_t feather,int table_id
 	{
 		return 0;
 	}
+
+	if(table_rt->table_type==TABLE_TYPE_INTERVAL_PLUS&&(_mid==NULL||_mid->is_set_district!=1))
+	{
+		_feather->scan_err_cnt++;
+		return -1;
+	}
+	
 	intval_scan_data.rule_type=RULETYPE_INT;
 	intval_scan_data.sub_type=make_sub_type(p_table->table_id, CHARSET_NONE, 0);
 	intval_scan_data.int_data=intval;
@@ -1391,6 +1398,10 @@ int Maat_scan_intval(Maat_feather_t feather,int table_id
 	
 	INC_SCANNER_REF(my_scanner,thread_num);
 	region_ret=rulescan_search(my_scanner->region, thread_num, &intval_scan_data, region_result, MAX_SCANNER_HIT_NUM);
+	if(region_ret>0&&p_table->table_type==TABLE_TYPE_INTERVAL_PLUS)
+	{
+		region_ret=match_district(_mid, region_result, region_ret);
+	}
 	if(region_ret<0)
 	{
 		DEC_SCANNER_REF(my_scanner, thread_num);

src/entry/Maat_command.cpp

@@ -161,7 +161,14 @@ enum MAAT_TABLE_TYPE type_region2table(const struct Maat_region_t* p)
 			}
 			break;
 		case REGION_INTERVAL:
+			if(p->interval_rule.district==NULL)
+			{
 				ret=TABLE_TYPE_INTERVAL;
+			}
+			else
+			{
+				ret=TABLE_TYPE_INTERVAL_PLUS;
+			}
 			break;
 		case REGION_DIGEST:
 			ret=TABLE_TYPE_DIGEST;
@@ -203,6 +210,9 @@ int get_valid_flag_offset(const char* line, enum MAAT_TABLE_TYPE type,int valid_
 		case	TABLE_TYPE_INTERVAL:
 			column_seq=5;
 			break;
+		case	TABLE_TYPE_INTERVAL_PLUS:
+			column_seq=6;
+			break;
 		case	TABLE_TYPE_DIGEST:
 			column_seq=6;
 			break;

src/entry/Maat_rule.cpp

@@ -34,7 +34,7 @@
 #include "stream_fuzzy_hash.h"
 #include "gram_index_engine.h"
 
-int MAAT_FRAME_VERSION_3_0_20200819=1;
+int MAAT_FRAME_VERSION_3_0_20200822=1;
 
 int is_valid_table_name(const char* str)
 {
@@ -1338,9 +1338,16 @@ int add_intval_rule(struct Maat_table_schema* table,struct db_intval_rule* intva
 	scan_rule_t* p_rule=NULL;
 	struct op_expr_t* op_expr=NULL;
 	struct Maat_region_inner* u_para=NULL;
-	int expr_id=0, ret=0;
+	int expr_id=0, ret=0,district_id=-1;
 
-	u_para=Maat_region_inner_new(intval_rule->group_id, intval_rule->region_id, table->table_id, -1);
+	if(table->table_type==TABLE_TYPE_EXPR_PLUS)
+	{
+		assert(strlen(intval_rule->district)>0);
+		str_unescape(intval_rule->district);
+		district_id=get_district_id(scanner, intval_rule->district);
+	}
+
+	u_para=Maat_region_inner_new(intval_rule->group_id, intval_rule->region_id, table->table_id, district_id);
 	ret=Maat_hierarchy_add_region_to_group(scanner->hier, intval_rule->group_id, intval_rule->region_id, table->table_id, u_para);
 	if(ret!=0)
 	{
@@ -1409,6 +1416,7 @@ int del_region_rule(struct Maat_table_schema* table, int region_id, int group_id
 		case	TABLE_TYPE_EXPR:
 		case	TABLE_TYPE_EXPR_PLUS:
 		case	TABLE_TYPE_INTERVAL:
+		case	TABLE_TYPE_INTERVAL_PLUS:
 			for(i=0;i<region->expr_id_cnt;i++)
 			{
 				op_expr=create_op_expr(region->expr_id_lb+i, RULESCAN_OP_DEL, NULL, table->table_id);//del expr
@@ -1904,6 +1912,9 @@ void update_intval_rule(struct Maat_table_schema* table, const char* table_line,
 	struct db_intval_rule* intval_rule=ALLOC(struct db_intval_rule, 1);
 	struct Maat_table_runtime* table_rt=Maat_table_runtime_get(scanner->table_rt_mgr, table->table_id);
 	int ret=0;
+	switch(table->table_type)
+	{
+		case	TABLE_TYPE_INTERVAL:
 			ret=sscanf(table_line,"%d\t%d\t%u\t%u\t%d",&(intval_rule->region_id)
 														,&(intval_rule->group_id)
 														,&(intval_rule->intval.lb)
@@ -1918,6 +1929,30 @@ void update_intval_rule(struct Maat_table_schema* table, const char* table_line,
 				table->udpate_err_cnt++;
 				goto error_out;
 			}
+			break;
+		case	TABLE_TYPE_INTERVAL_PLUS:
+			ret=sscanf(table_line,"%d\t%d\t%s\t%u\t%u\t%d",&(intval_rule->region_id)
+														,&(intval_rule->group_id)
+														,intval_rule->district
+														,&(intval_rule->intval.lb)
+														,&(intval_rule->intval.ub)
+														,&(intval_rule->is_valid));
+
+			if(ret!=6||intval_rule->intval.ub<intval_rule->intval.lb)
+			{
+				MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_module ,
+									"update error,invalid format  of interval table %s:%s"
+									,table->table_name[table->updating_name],table_line);
+				table->udpate_err_cnt++;
+				goto error_out;
+			}
+			break;
+		default:
+			assert(0);
+			break;
+	}
+
+
 	if(intval_rule->is_valid==FALSE)
 	{
 		ret=del_region_rule(table
@@ -2422,6 +2457,7 @@ int maat_update_cb(const char* table_name,const char* line,void *u_para)
 			update_ip_rule(p_table, line, scanner, feather->logger);
 			break;
 		case	TABLE_TYPE_INTERVAL:
+		case	TABLE_TYPE_INTERVAL_PLUS:
 			update_intval_rule(p_table, line, scanner,feather->logger);
 			break;
 		case	TABLE_TYPE_DIGEST:

src/entry/Maat_table.cpp

@@ -492,6 +492,9 @@ struct Maat_table_manager* Maat_table_manager_create(const char* table_info_path
 	map_register(string2int_map,"plugin", TABLE_TYPE_PLUGIN);
 	map_register(string2int_map,"ip_plugin", TABLE_TYPE_IP_PLUGIN);
 	map_register(string2int_map,"intval", TABLE_TYPE_INTERVAL);
+	map_register(string2int_map,"interval", TABLE_TYPE_INTERVAL);
+	map_register(string2int_map,"intval_plus", TABLE_TYPE_INTERVAL_PLUS);
+	map_register(string2int_map,"interval_plus", TABLE_TYPE_INTERVAL_PLUS);
 	map_register(string2int_map,"digest", TABLE_TYPE_DIGEST);
 	map_register(string2int_map,"expr_plus", TABLE_TYPE_EXPR_PLUS);
 	map_register(string2int_map,"group", TABLE_TYPE_GROUP);

src/entry/json2iris.cpp

@@ -164,6 +164,9 @@ int set_iris_descriptor(const char* json_file,cJSON *json, const char* encrypt_k
 	map_register(iris_cfg->str2int_map, "expr",TABLE_TYPE_EXPR);
 	map_register(iris_cfg->str2int_map, "expr_plus",TABLE_TYPE_EXPR_PLUS);
 	map_register(iris_cfg->str2int_map, "intval",TABLE_TYPE_INTERVAL);
+	map_register(iris_cfg->str2int_map, "interval",TABLE_TYPE_INTERVAL);
+	map_register(iris_cfg->str2int_map, "intval_plus",TABLE_TYPE_INTERVAL_PLUS);
+	map_register(iris_cfg->str2int_map, "interval_plus",TABLE_TYPE_INTERVAL_PLUS);
 	map_register(iris_cfg->str2int_map, "digest",TABLE_TYPE_DIGEST);
 	map_register(iris_cfg->str2int_map, "similar",TABLE_TYPE_SIMILARITY);
 
@@ -597,6 +600,13 @@ int write_intval_line(cJSON *region_json, struct iris_description_t *p_iris, str
 	json_cmd[cmd_cnt].json_type=cJSON_Number;
 	cmd_cnt++;
 
+	if(table->table_type==TABLE_TYPE_INTERVAL_PLUS)
+	{
+		json_cmd[cmd_cnt].json_string="district";
+		json_cmd[cmd_cnt].json_type=cJSON_String;
+		cmd_cnt++;
+	}
+		
 	json_cmd[cmd_cnt].json_string="low_boundary";
 	json_cmd[cmd_cnt].json_type=cJSON_Number;
 	cmd_cnt++;
@@ -781,6 +791,7 @@ int write_region_rule(cJSON* region_json, int compile_id, int group_id, iris_des
 			write_ip_plus_line(table_content, p_iris, table_info, logger);
 			break;
 		case TABLE_TYPE_INTERVAL:
+		case TABLE_TYPE_INTERVAL_PLUS:
 			ret=write_intval_line(table_content, p_iris, table_info, logger);
 			break;
 		case TABLE_TYPE_DIGEST:

src/inc_internal/Maat_rule_internal.h

@@ -67,6 +67,7 @@ struct db_intval_rule
 	int group_id;
 	interval_rule_t intval;
 	int is_valid;
+	char district[MAX_DISTRICT_LEN+1];
 };
 struct db_digest_rule
 {

src/inc_internal/Maat_table.h

@@ -35,7 +35,8 @@ enum MAAT_TABLE_TYPE
 	TABLE_TYPE_GROUP,
 	TABLE_TYPE_COMPILE,
 	TABLE_TYPE_PLUGIN,
-	TABLE_TYPE_IP_PLUGIN
+	TABLE_TYPE_IP_PLUGIN,
+	TABLE_TYPE_INTERVAL_PLUS
 };
 
 struct compile_ex_data_idx

test/maat_json.json

@@ -1825,6 +1825,31 @@
 		                "clause_index":1                   
 		            }
             ]
+        },
+        {
+            "compile_id": 179,
+            "service": 1,
+            "action": 1,
+            "do_blacklist": 1,
+            "do_log": 1,
+            "user_region": "anything",
+            "is_valid": "yes",
+            "groups": [
+                {
+                    "group_name": "Untitled",
+                    "regions": [
+                        {
+                            "table_name": "INTERGER_PLUS",
+                            "table_type": "intval_plus",
+                            "table_content": {
+                                "district": "intval.plus",
+                                "low_boundary": 2020,
+                                "up_boundary": 2020
+                            }
+                        }
+                    ]
+                }
+            ]
         }
     ],
     "plugin_table": [

test/table_info.conf

@@ -55,3 +55,4 @@
 32	DESTINATION_IP_ASN	virtual	AS_NUMBER	--
 33	GeoLocation	expr	UTF8	UTF8	yes	0
 34	SOURCE_IP_GEO	virtual	GeoLocation	--
+35	INTERGER_PLUS	intval_plus --

test/test_maatframe.cpp

@@ -444,6 +444,30 @@ TEST(StringScan, PrefixAndSuffix)
 	Maat_clean_status(&mid);
 
 }
+
+#define IntvalPlusTest	1
+TEST(IntvalScan,   IntvalPlusPlus)
+{
+	int table_id=0,ret=0;
+	int scan_val=2020;
+	scan_status_t mid=NULL;
+	struct Maat_rule_t result[4];
+	
+	table_id=Maat_table_register(g_feather, "INTERGER_PLUS");
+	ASSERT_GT(table_id, 0);
+
+	ret=Maat_set_scan_status(g_feather, &mid, MAAT_SET_SCAN_DISTRICT, "intval.plus", strlen("intval.plus"));
+	ASSERT_EQ(ret, 0);
+	
+	ret=Maat_scan_intval(g_feather, table_id, scan_val, result,4, &mid, 0);
+	EXPECT_EQ(ret, 1);
+	EXPECT_EQ(result[0].config_id, 179);
+	
+	Maat_clean_status(&mid);
+	
+	return ;
+}
+
 #define ExprPlusTest	1
 TEST(StringScan,   ExprPlus)
 {