gfwleak/tango-maat
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'feature-intval-plus' into 'develop-version3'

Browse Source 
Feature intval plus

See merge request MESA_framework/maat!48
This commit is contained in:
郑超
2020-08-22 19:19:32 +08:00
parent 3791c9426e a1d23a13e1
commit cd3ec37e5a
14 changed files with 167 additions and 36 deletions
Download Patch File Download Diff File Expand all files Collapse all files

33
.gitlab-ci.yml
View File

@@ -70,6 +70,22 @@ develop_build_release:
- /^master.*$/i
release_build_debug:
stage: build
variables:
BUILD_TYPE: Debug
PACKAGE: 1
UPLOAD: 1
PULP3_REPO_NAME: framework-stable-x86_64.el7
PULP3_DIST_NAME: framework-stable-x86_64.el7
extends: .build_by_travis
artifacts:
name: "maatframe-$CI_COMMIT_REF_NAME-debug"
paths:
- build/*.rpm
only:
- tags
release_build_release:
stage: build
variables:
@@ -85,20 +101,3 @@ release_build_release:
- build/*.rpm
only:
- tags
release_build_release_devel:
stage: build
variables:
BUILD_TYPE: RelWithDebInfo
ENABLE_DEVEL_SWITCH: "ON"
PACKAGE: 1
UPLOAD: 1
PULP3_REPO_NAME: framework-stable-x86_64.el7
PULP3_DIST_NAME: framework-stable-x86_64.el7
extends: .build_by_travis
artifacts:
name: "maatframe-$CI_COMMIT_REF_NAME-release"
paths:
- build/*.rpm
only:
- tags

4
cmake/Package.cmake
View File

@@ -14,6 +14,10 @@ set(CPACK_PACKAGING_INSTALL_PREFIX ${CMAKE_INSTALL_PREFIX})
set(CPACK_PACKAGE_VERSION "${VERSION_MAJOR}.${VERSION_MINOR}.${VERSION_PATCH}.${VERSION_BUILD}")
execute_process(COMMAND bash -c "echo -ne \"`uname -r | awk -F'.' '{print $5\".\"$6\".\"$7}'`\"" OUTPUT_VARIABLE SYSTEM_VERSION)
execute_process(COMMAND sh changelog.sh WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}/cmake)
SET(CPACK_RPM_CHANGELOG_FILE ${PROJECT_SOURCE_DIR}/cmake/changelog.txt)
# RPM Build
set(CPACK_GENERATOR "RPM")
set(CPACK_RPM_PACKAGE_VENDOR "MESA")

4
cmake/changelog.sh Normal file
View File

@@ -0,0 +1,4 @@
#!/bin/sh
branch=`git status | grep branch | awk '{print $NF}'`
git log --branches=$branch --no-merges --date=local --show-signature --pretty="* %ad %an %ae %nhash: %H%ncommit:%n%B" | awk -F"-" '{print "- "$0}' | sed 's/- \*/\*/g' | sed 's/- $//g' | sed 's/-/ -/g' | sed 's/[0-9]\{2\}:[0-9]\{2\}:[0-9]\{2\}//g' > changelog.txt

1
inc/Maat_command.h
View File

@@ -78,6 +78,7 @@ struct Maat_rgn_addr_t
};
struct Maat_rgn_intv_t
{
const char *district;// optional for expr_plus, otherwise set to NULL.
unsigned int low_boundary;
unsigned int up_boundary;
};

11
src/entry/Maat_api.cpp
View File

@@ -1381,6 +1381,13 @@ int Maat_scan_intval(Maat_feather_t feather,int table_id
{
return 0;
}
if(table_rt->table_type==TABLE_TYPE_INTERVAL_PLUS&&(_mid==NULL||_mid->is_set_district!=1))
{
_feather->scan_err_cnt++;
return -1;
}
intval_scan_data.rule_type=RULETYPE_INT;
intval_scan_data.sub_type=make_sub_type(p_table->table_id, CHARSET_NONE, 0);
intval_scan_data.int_data=intval;
@@ -1391,6 +1398,10 @@ int Maat_scan_intval(Maat_feather_t feather,int table_id
INC_SCANNER_REF(my_scanner,thread_num);
region_ret=rulescan_search(my_scanner->region, thread_num, &intval_scan_data, region_result, MAX_SCANNER_HIT_NUM);
if(region_ret>0&&p_table->table_type==TABLE_TYPE_INTERVAL_PLUS)
{
region_ret=match_district(_mid, region_result, region_ret);
}
if(region_ret<0)
{
DEC_SCANNER_REF(my_scanner, thread_num);

12
src/entry/Maat_command.cpp
View File

@@ -161,7 +161,14 @@ enum MAAT_TABLE_TYPE type_region2table(const struct Maat_region_t* p)
}
break;
case REGION_INTERVAL:
ret=TABLE_TYPE_INTERVAL;
if(p->interval_rule.district==NULL)
{
ret=TABLE_TYPE_INTERVAL;
}
else
{
ret=TABLE_TYPE_INTERVAL_PLUS;
}
break;
case REGION_DIGEST:
ret=TABLE_TYPE_DIGEST;
@@ -203,6 +210,9 @@ int get_valid_flag_offset(const char* line, enum MAAT_TABLE_TYPE type,int valid_
case TABLE_TYPE_INTERVAL:
column_seq=5;
break;
case TABLE_TYPE_INTERVAL_PLUS:
column_seq=6;
break;
case TABLE_TYPE_DIGEST:
column_seq=6;
break;

66
src/entry/Maat_rule.cpp
View File

@@ -34,7 +34,7 @@
#include "stream_fuzzy_hash.h"
#include "gram_index_engine.h"
int MAAT_FRAME_VERSION_3_0_20200819=1;
int MAAT_FRAME_VERSION_3_0_20200822=1;
int is_valid_table_name(const char* str)
{
@@ -1338,9 +1338,16 @@ int add_intval_rule(struct Maat_table_schema* table,struct db_intval_rule* intva
scan_rule_t* p_rule=NULL;
struct op_expr_t* op_expr=NULL;
struct Maat_region_inner* u_para=NULL;
int expr_id=0, ret=0;
int expr_id=0, ret=0,district_id=-1;
u_para=Maat_region_inner_new(intval_rule->group_id, intval_rule->region_id, table->table_id, -1);
if(table->table_type==TABLE_TYPE_EXPR_PLUS)
{
assert(strlen(intval_rule->district)>0);
str_unescape(intval_rule->district);
district_id=get_district_id(scanner, intval_rule->district);
}
u_para=Maat_region_inner_new(intval_rule->group_id, intval_rule->region_id, table->table_id, district_id);
ret=Maat_hierarchy_add_region_to_group(scanner->hier, intval_rule->group_id, intval_rule->region_id, table->table_id, u_para);
if(ret!=0)
{
@@ -1409,6 +1416,7 @@ int del_region_rule(struct Maat_table_schema* table, int region_id, int group_id
case TABLE_TYPE_EXPR:
case TABLE_TYPE_EXPR_PLUS:
case TABLE_TYPE_INTERVAL:
case TABLE_TYPE_INTERVAL_PLUS:
for(i=0;i<region->expr_id_cnt;i++)
{
op_expr=create_op_expr(region->expr_id_lb+i, RULESCAN_OP_DEL, NULL, table->table_id);//del expr
@@ -1904,20 +1912,47 @@ void update_intval_rule(struct Maat_table_schema* table, const char* table_line,
struct db_intval_rule* intval_rule=ALLOC(struct db_intval_rule, 1);
struct Maat_table_runtime* table_rt=Maat_table_runtime_get(scanner->table_rt_mgr, table->table_id);
int ret=0;
ret=sscanf(table_line,"%d\t%d\t%u\t%u\t%d",&(intval_rule->region_id)
,&(intval_rule->group_id)
,&(intval_rule->intval.lb)
,&(intval_rule->intval.ub)
,&(intval_rule->is_valid));
if(ret!=5||intval_rule->intval.ub<intval_rule->intval.lb)
switch(table->table_type)
{
MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_module ,
"update error,invalid format of interval table %s:%s"
,table->table_name[table->updating_name],table_line);
table->udpate_err_cnt++;
goto error_out;
case TABLE_TYPE_INTERVAL:
ret=sscanf(table_line,"%d\t%d\t%u\t%u\t%d",&(intval_rule->region_id)
,&(intval_rule->group_id)
,&(intval_rule->intval.lb)
,&(intval_rule->intval.ub)
,&(intval_rule->is_valid));
if(ret!=5||intval_rule->intval.ub<intval_rule->intval.lb)
{
MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_module ,
"update error,invalid format of interval table %s:%s"
,table->table_name[table->updating_name],table_line);
table->udpate_err_cnt++;
goto error_out;
}
break;
case TABLE_TYPE_INTERVAL_PLUS:
ret=sscanf(table_line,"%d\t%d\t%s\t%u\t%u\t%d",&(intval_rule->region_id)
,&(intval_rule->group_id)
,intval_rule->district
,&(intval_rule->intval.lb)
,&(intval_rule->intval.ub)
,&(intval_rule->is_valid));
if(ret!=6||intval_rule->intval.ub<intval_rule->intval.lb)
{
MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_module ,
"update error,invalid format of interval table %s:%s"
,table->table_name[table->updating_name],table_line);
table->udpate_err_cnt++;
goto error_out;
}
break;
default:
assert(0);
break;
}
if(intval_rule->is_valid==FALSE)
{
ret=del_region_rule(table
@@ -2422,6 +2457,7 @@ int maat_update_cb(const char* table_name,const char* line,void *u_para)
update_ip_rule(p_table, line, scanner, feather->logger);
break;
case TABLE_TYPE_INTERVAL:
case TABLE_TYPE_INTERVAL_PLUS:
update_intval_rule(p_table, line, scanner,feather->logger);
break;
case TABLE_TYPE_DIGEST:

3
src/entry/Maat_table.cpp
View File

@@ -492,6 +492,9 @@ struct Maat_table_manager* Maat_table_manager_create(const char* table_info_path
map_register(string2int_map,"plugin", TABLE_TYPE_PLUGIN);
map_register(string2int_map,"ip_plugin", TABLE_TYPE_IP_PLUGIN);
map_register(string2int_map,"intval", TABLE_TYPE_INTERVAL);
map_register(string2int_map,"interval", TABLE_TYPE_INTERVAL);
map_register(string2int_map,"intval_plus", TABLE_TYPE_INTERVAL_PLUS);
map_register(string2int_map,"interval_plus", TABLE_TYPE_INTERVAL_PLUS);
map_register(string2int_map,"digest", TABLE_TYPE_DIGEST);
map_register(string2int_map,"expr_plus", TABLE_TYPE_EXPR_PLUS);
map_register(string2int_map,"group", TABLE_TYPE_GROUP);

11
src/entry/json2iris.cpp
View File

@@ -164,6 +164,9 @@ int set_iris_descriptor(const char* json_file,cJSON *json, const char* encrypt_k
map_register(iris_cfg->str2int_map, "expr",TABLE_TYPE_EXPR);
map_register(iris_cfg->str2int_map, "expr_plus",TABLE_TYPE_EXPR_PLUS);
map_register(iris_cfg->str2int_map, "intval",TABLE_TYPE_INTERVAL);
map_register(iris_cfg->str2int_map, "interval",TABLE_TYPE_INTERVAL);
map_register(iris_cfg->str2int_map, "intval_plus",TABLE_TYPE_INTERVAL_PLUS);
map_register(iris_cfg->str2int_map, "interval_plus",TABLE_TYPE_INTERVAL_PLUS);
map_register(iris_cfg->str2int_map, "digest",TABLE_TYPE_DIGEST);
map_register(iris_cfg->str2int_map, "similar",TABLE_TYPE_SIMILARITY);
@@ -597,6 +600,13 @@ int write_intval_line(cJSON *region_json, struct iris_description_t *p_iris, str
json_cmd[cmd_cnt].json_type=cJSON_Number;
cmd_cnt++;
if(table->table_type==TABLE_TYPE_INTERVAL_PLUS)
{
json_cmd[cmd_cnt].json_string="district";
json_cmd[cmd_cnt].json_type=cJSON_String;
cmd_cnt++;
}
json_cmd[cmd_cnt].json_string="low_boundary";
json_cmd[cmd_cnt].json_type=cJSON_Number;
cmd_cnt++;
@@ -781,6 +791,7 @@ int write_region_rule(cJSON* region_json, int compile_id, int group_id, iris_des
write_ip_plus_line(table_content, p_iris, table_info, logger);
break;
case TABLE_TYPE_INTERVAL:
case TABLE_TYPE_INTERVAL_PLUS:
ret=write_intval_line(table_content, p_iris, table_info, logger);
break;
case TABLE_TYPE_DIGEST:

1
src/inc_internal/Maat_rule_internal.h
View File

@@ -67,6 +67,7 @@ struct db_intval_rule
int group_id;
interval_rule_t intval;
int is_valid;
char district[MAX_DISTRICT_LEN+1];
};
struct db_digest_rule
{

3
src/inc_internal/Maat_table.h
View File

@@ -35,7 +35,8 @@ enum MAAT_TABLE_TYPE
TABLE_TYPE_GROUP,
TABLE_TYPE_COMPILE,
TABLE_TYPE_PLUGIN,
TABLE_TYPE_IP_PLUGIN
TABLE_TYPE_IP_PLUGIN,
TABLE_TYPE_INTERVAL_PLUS
};
struct compile_ex_data_idx

25
test/maat_json.json
View File

@@ -1825,6 +1825,31 @@
"clause_index":1
}
]
},
{
"compile_id": 179,
"service": 1,
"action": 1,
"do_blacklist": 1,
"do_log": 1,
"user_region": "anything",
"is_valid": "yes",
"groups": [
{
"group_name": "Untitled",
"regions": [
{
"table_name": "INTERGER_PLUS",
"table_type": "intval_plus",
"table_content": {
"district": "intval.plus",
"low_boundary": 2020,
"up_boundary": 2020
}
}
]
}
]
}
],
"plugin_table": [

1
test/table_info.conf
View File

@@ -55,3 +55,4 @@
32 DESTINATION_IP_ASN virtual AS_NUMBER --
33 GeoLocation expr UTF8 UTF8 yes 0
34 SOURCE_IP_GEO virtual GeoLocation --
35 INTERGER_PLUS intval_plus --

24
test/test_maatframe.cpp
View File

@@ -444,6 +444,30 @@ TEST(StringScan, PrefixAndSuffix)
Maat_clean_status(&mid);
}
#define IntvalPlusTest 1
TEST(IntvalScan, IntvalPlusPlus)
{
int table_id=0,ret=0;
int scan_val=2020;
scan_status_t mid=NULL;
struct Maat_rule_t result[4];
table_id=Maat_table_register(g_feather, "INTERGER_PLUS");
ASSERT_GT(table_id, 0);
ret=Maat_set_scan_status(g_feather, &mid, MAAT_SET_SCAN_DISTRICT, "intval.plus", strlen("intval.plus"));
ASSERT_EQ(ret, 0);
ret=Maat_scan_intval(g_feather, table_id, scan_val, result,4, &mid, 0);
EXPECT_EQ(ret, 1);
EXPECT_EQ(result[0].config_id, 179);
Maat_clean_status(&mid);
return ;
}
#define ExprPlusTest 1
TEST(StringScan, ExprPlus)
{