diff --git a/src/entry/Maat_api.cpp b/src/entry/Maat_api.cpp index ad092fa..79b86e9 100644 --- a/src/entry/Maat_api.cpp +++ b/src/entry/Maat_api.cpp @@ -152,8 +152,8 @@ static int compare_compile_id(const void *a, const void *b) int region_compile(_Maat_feather_t*feather,struct _INNER_scan_status_t *_mid,int is_last_region,void* region_hit,int region_type_size,int group_offset,int region_hit_num,struct Maat_rule_t* result,_compile_result_t *rs_result, int size,int thread_num) { - int scan_ret=0,result_cnt=0; - int ret=0,i=0; + int scan_ret=0, result_cnt=0; + int ret=0, i=0, j=0; size_t r_in_c_cnt=0; int shortcut_avilable_cnt=0; unsigned char has_not_flag=0; @@ -177,11 +177,14 @@ int region_compile(_Maat_feather_t*feather,struct _INNER_scan_status_t *_mid,int } _mid->cur_hit_id[_mid->cur_hit_cnt]=group_rule->group_id; _mid->cur_hit_cnt++; - ret=insert_set_id(&(_mid->hitted_group_id), - &(_mid->hit_group_size), - _mid->hit_group_cnt, - group_rule->group_id); - _mid->hit_group_cnt+=ret; + for(j=0; jendpoint_cnt; j++) + { + ret=insert_set_id(&(_mid->hitted_group_id), + &(_mid->hit_group_size), + _mid->hit_group_cnt, + group_rule->endpoints[j]); + _mid->hit_group_cnt+=ret; + } } if((region_hit_num>0 &&shortcut_avilable_cnt==region_hit_num) || shortcut_avilable_cnt==MAX_SCANNER_HIT_NUM) @@ -195,7 +198,7 @@ int region_compile(_Maat_feather_t*feather,struct _INNER_scan_status_t *_mid,int //This shortcut is NO longger valid after bool macher support NOT-logic. //short cut for last scan and combination rules //region_hit_num==1 : for current scan hitted rules, one and each other group may statisfy a compile rule. - //_mid->hit_group_cnt==1: With pre scan hitted group rules, one group may staisfy a compile rule + //_mid->hit_group_cnt==1: With previously scan hitted group rules, one group may staisfy a compile rule scan_ret=0; } else diff --git a/src/entry/Maat_rule.cpp b/src/entry/Maat_rule.cpp index cd038c9..37bfed1 100644 --- a/src/entry/Maat_rule.cpp +++ b/src/entry/Maat_rule.cpp @@ -838,7 +838,7 @@ error_jump: } struct Maat_group_inner* create_group_rule(int group_id) { - struct Maat_group_inner* group=(struct Maat_group_inner*)malloc(sizeof(struct Maat_group_inner)); + struct Maat_group_inner* group=ALLOC(struct Maat_group_inner, 1); group->group_id=group_id; group->region_cnt=0; group->region_boundary=0; @@ -861,6 +861,8 @@ void _destroy_group_rule(struct Maat_group_inner* group) group->table_id=-1; free(group->group_name); group->group_name=NULL; + free(group->endpoints); + group->endpoints=NULL; pthread_mutex_destroy(&(group->mutex)); free(group); @@ -996,7 +998,7 @@ void EMPTY_FREE(void*p) } struct Maat_compile_inner * create_compile_rule(int compile_id) { - struct Maat_compile_inner* p=ALLOC(struct Maat_compile_inner,1); + struct Maat_compile_inner* p=ALLOC(struct Maat_compile_inner, 1); p->compile_id=compile_id; p->group_cnt=0; p->group_boundary=1; @@ -3318,6 +3320,43 @@ void update_plugin_table(struct Maat_table_desc* table,const char* table_line,Ma table_rt->plugin.cache_line_num++; } } +void walk_group_hash(const uchar * key, uint size, void * data, void * user) +{ + struct Maat_group_inner* group_rule=(struct Maat_group_inner*)data; + struct Maat_group_inner* parent_group=NULL; + struct Maat_scanner_t* scanner=(struct Maat_scanner_t*)user; + igraph_vector_t vids; + igraph_dfs(&(scanner->group_graph), group_rule->group_id, IGRAPH_OUT, + /*NO search unreachable*/ 0, &vids, NULL, NULL, NULL, NULL, NULL, NULL); + long int i; + int* temp_group_ids=ALLOC(int, igraph_vector_size(&vids)); + size_t path_endpoint_cnt=0; + for(i=0; igroup_hash, (int) VECTOR(vids)[i]); + if(parent_group->has_compile_neighbors)//including itself? + { + temp_group_ids[path_endpoint_cnt]=parent_group->group_id; + path_endpoint_cnt++; + } + } + pthread_mutex_lock(&(group_rule->mutex)); + free(group_rule->endpoints); + group_rule->endpoint_cnt=path_endpoint_cnt; + group_rule->endpoints=ALLOC(int, group_rule->endpoint_cnt); + memcpy(group_rule->endpoints, temp_group_ids, sizeof(int)*group_rule->endpoint_cnt); + pthread_mutex_unlock(&(group_rule->mutex)); + igraph_vector_destroy(&vids); + free(temp_group_ids); + temp_group_ids=NULL; + return; +} + +void find_group_paths(struct Maat_scanner_t* scanner) +{ + MESA_htable_iterate(scanner->group_hash, walk_group_hash, scanner); + return; +} void do_scanner_update(struct Maat_scanner_t* scanner,MESA_lqueue_head garbage_q,int scan_thread_num,void* logger) { struct bool_matcher *tmp1=NULL,*tmp2=NULL; diff --git a/src/entry/json2iris.cpp b/src/entry/json2iris.cpp index 0a5bac8..4f3186a 100644 --- a/src/entry/json2iris.cpp +++ b/src/entry/json2iris.cpp @@ -321,7 +321,7 @@ error_out: } return -1; } -int write_ip_rule(cJSON *region_json,struct iris_description_t *p_iris,const char* path,void * logger) +int write_ip_line(cJSON *region_json, struct iris_description_t *p_iris, const char* path, void * logger) { struct traslate_command_t json_cmd[MAX_COLUMN_NUM]; int cmd_cnt=0; @@ -407,7 +407,7 @@ int write_ip_rule(cJSON *region_json,struct iris_description_t *p_iris,const cha return direct_write_rule(region_json, p_iris->str2int_map,json_cmd, cmd_cnt,path,logger); } -int write_expr_rule(cJSON *region_json,struct iris_description_t *p_iris,const char* path,enum MAAT_TABLE_TYPE table_type,void * logger) +int write_expr_line(cJSON *region_json,struct iris_description_t *p_iris,const char* path,enum MAAT_TABLE_TYPE table_type,void * logger) { struct traslate_command_t json_cmd[MAX_COLUMN_NUM]; int cmd_cnt=0; @@ -454,7 +454,7 @@ int write_expr_rule(cJSON *region_json,struct iris_description_t *p_iris,const c return direct_write_rule(region_json, p_iris->str2int_map,json_cmd, cmd_cnt,path,logger); } -int write_intval_rule(cJSON *region_json,struct iris_description_t *p_iris,const char* path,void * logger) +int write_intval_line(cJSON *region_json,struct iris_description_t *p_iris,const char* path,void * logger) { struct traslate_command_t json_cmd[MAX_COLUMN_NUM]; int cmd_cnt=0; @@ -483,7 +483,7 @@ int write_intval_rule(cJSON *region_json,struct iris_description_t *p_iris,const return direct_write_rule(region_json, p_iris->str2int_map,json_cmd, cmd_cnt,path,logger); } -int write_digest_rule(cJSON *region_json,struct iris_description_t *p_iris,const char* path,void * logger) +int write_digest_line(cJSON *region_json, struct iris_description_t *p_iris, const char* path, void * logger) { struct traslate_command_t json_cmd[MAX_COLUMN_NUM]; int cmd_cnt=0; @@ -516,7 +516,7 @@ int write_digest_rule(cJSON *region_json,struct iris_description_t *p_iris,const return direct_write_rule(region_json, p_iris->str2int_map,json_cmd, cmd_cnt,path,logger); } -int write_similar_rule(cJSON *region_json,struct iris_description_t *p_iris,const char* path,void * logger) +int write_similar_line(cJSON *region_json, struct iris_description_t *p_iris, const char* path, void * logger) { struct traslate_command_t json_cmd[MAX_COLUMN_NUM]; int cmd_cnt=0; @@ -546,7 +546,7 @@ int write_similar_rule(cJSON *region_json,struct iris_description_t *p_iris,cons } -int write_plugin_table(cJSON* plug_table_json,int sequence,iris_description_t* p_iris,void* logger) +int write_plugin_line(cJSON* plug_table_json,int sequence,iris_description_t* p_iris,void* logger) { cJSON* item=NULL,*table_content=NULL,*each_line=NULL; struct iris_table_t* table_info=NULL; @@ -665,19 +665,19 @@ int write_region_rule(cJSON* region_json,int compile_id,int group_id,iris_descri { case TABLE_TYPE_EXPR: case TABLE_TYPE_EXPR_PLUS: - ret=write_expr_rule(table_content, p_iris, table_info->table_path,table_type, logger); + ret=write_expr_line(table_content, p_iris, table_info->table_path,table_type, logger); break; case TABLE_TYPE_IP: - ret=write_ip_rule(table_content, p_iris, table_info->table_path, logger); + ret=write_ip_line(table_content, p_iris, table_info->table_path, logger); break; case TABLE_TYPE_INTERVAL: - ret=write_intval_rule(table_content, p_iris, table_info->table_path, logger); + ret=write_intval_line(table_content, p_iris, table_info->table_path, logger); break; case TABLE_TYPE_DIGEST: - ret=write_digest_rule(table_content, p_iris, table_info->table_path, logger); + ret=write_digest_line(table_content, p_iris, table_info->table_path, logger); break; case TABLE_TYPE_SIMILARITY: - write_similar_rule(table_content, p_iris,table_info->table_path, logger); + write_similar_line(table_content, p_iris,table_info->table_path, logger); break; default: assert(0); @@ -691,7 +691,7 @@ int write_region_rule(cJSON* region_json,int compile_id,int group_id,iris_descri return ret; } -int write_compile_rule(cJSON *compile,struct iris_description_t *p_iris,void * logger) +int write_compile_line(cJSON *compile, struct iris_description_t *p_iris, void * logger) { int compile_id=-1,cmd_cnt=0,ret=-1; cJSON* item=NULL; @@ -772,7 +772,7 @@ int write_compile_rule(cJSON *compile,struct iris_description_t *p_iris,void * l set_file_rulenum(table_info->table_path,table_info->line_count,logger); return compile_id; } -int write_group_rule(int compile_id ,int group_id, int group_not_flag, struct iris_description_t *p_iris,void * logger) +int write_group_line(int group_id, int parent_id, int group_not_flag, int parent_type, struct iris_description_t *p_iris, void * logger) { FILE*fp=NULL; int ret=0; @@ -792,7 +792,7 @@ int write_group_rule(int compile_id ,int group_id, int group_not_flag, struct ir "fopen %s error %s.",p_iris->group_table->table_path,strerror(errno)); return -1; } - fprintf(fp,"%d\t%d\t1\t%d\n",group_id, compile_id, group_not_flag); + fprintf(fp,"%d\t%d\t1\t%d\t%d\n",group_id, parent_id, group_not_flag, parent_type); fclose(fp); p_iris->group_table->line_count++; ret=set_file_rulenum(p_iris->group_table->table_path,p_iris->group_table->line_count,logger); @@ -818,17 +818,104 @@ int write_index_file(struct iris_description_t *p_iris,void* logger) fclose(fp); return 0; } -int write_iris(cJSON *json,struct iris_description_t *p_iris,void* logger) +int write_group_rule(cJSON *group_json, int parent_id, int parent_type, int tracking_compile_id, struct iris_description_t *p_iris, void* logger) { - int i=0,j=0,k=0; - int compile_id=-1,compile_cnt=0,group_cnt=0,region_cnt=0,plug_table_cnt=0; + const char* _str_parent_type[2]={"compile", "group"}; + int i=0; + int sub_group_cnt=0, region_cnt=0; int ret=0; int group_not_flag=0; - cJSON *c_rules=NULL,*g_rules=NULL,*r_rules=NULL,*item=NULL,*plug_tables=NULL; - cJSON *compile_rule=NULL,*group_rule=NULL,*region_rule=NULL,*each_plug_table=NULL; + cJSON *region_json=NULL, *item=NULL; + cJSON *sub_groups=NULL, *region_rule=NULL; const char* group_name=NULL; struct group_info_t *group_info=NULL; struct group_info_t untitled_group; + + item=cJSON_GetObjectItem(group_json, "group_name"); + if(item==NULL||item->type!=cJSON_String) + { + group_name=untitled_group_name; + } + else + { + group_name=item->valuestring; + } + item=cJSON_GetObjectItem(group_json,"not_flag"); + if(item==NULL||item->type!=cJSON_Number) + { + group_not_flag=0; + } + else + { + group_not_flag=item->valueint; + } + if(parent_type!=PARENT_TYPE_GROUP) + { + group_not_flag=0; + } + + group_info=(struct group_info_t*)MESA_htable_search(p_iris->group_name_map, (const unsigned char*)group_name, strlen(group_name)); + if(group_info==NULL)//exist group name, region already read + { + if(0==strncasecmp(group_name, untitled_group_name, strlen(untitled_group_name))) + { + group_info=&untitled_group; + group_info->group_id=get_group_seq(p_iris); + } + else + { + group_info=ALLOC(struct group_info_t, 1); + group_info->group_id=get_group_seq(p_iris); + MESA_htable_add(p_iris->group_name_map,(const unsigned char*)group_name, strlen(group_name),group_info); + } + } + ret=write_group_line(group_info->group_id, parent_id, group_not_flag, parent_type, p_iris, logger); + if(ret<0) + { + MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_json, + "%s rule %d write group error.", _str_parent_type[parent_type], parent_id); + return -1; + } + region_json=cJSON_GetObjectItem(group_json,"regions"); + if(region_json!=NULL) + { + region_cnt=cJSON_GetArraySize(region_json); + for(i=0; igroup_id, p_iris, logger); + if(ret<0) + { + MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_json, + "compile rule %d write region error.", tracking_compile_id); + return -1; + } + } + } + sub_groups=cJSON_GetObjectItem(group_json,"sub_groups"); + if(sub_groups!=NULL) + { + //recursively + sub_group_cnt=cJSON_GetArraySize(sub_groups); + for(i=0; igroup_id, PARENT_TYPE_GROUP, tracking_compile_id, p_iris, logger); + if(ret<0) + { + return -1; + } + } + } + return 0; +} +int write_iris(cJSON *json, struct iris_description_t *p_iris, void* logger) +{ + int i=0,j=0; + int compile_id=-1, compile_cnt=0, group_cnt=0, plug_table_cnt=0; + int ret=0; + cJSON *c_rules=NULL, *g_rules=NULL, *plug_tables=NULL; + cJSON *compile_rule=NULL,*group_rule=NULL, *each_plug_table=NULL; plug_tables=cJSON_GetObjectItem(json,"plugin_table"); if(NULL!=plug_tables) { @@ -836,7 +923,7 @@ int write_iris(cJSON *json,struct iris_description_t *p_iris,void* logger) for(i=0;itype!=cJSON_Number) - { - group_not_flag=0; - } - else - { - group_not_flag=item->valueint; - } - item=cJSON_GetObjectItem(group_rule,"group_name"); - if(item==NULL||item->type!=cJSON_String) - { - group_name=untitled_group_name; - } - else - { - group_name=item->valuestring; - } - group_info=(struct group_info_t*)MESA_htable_search(p_iris->group_name_map, (const unsigned char*)group_name, strlen(group_name)); - if(group_info!=NULL)//exist group name ,region already read - { - ret=write_group_rule(compile_id, group_info->group_id, group_not_flag, p_iris, logger); - if(ret<0) - { - MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_json, - "compile rule %d write group error.",compile_id); - return -1; - } - r_rules=cJSON_GetObjectItem(group_rule,"regions"); - if(r_rules!=NULL) - { - MESA_handle_runtime_log(logger,RLOG_LV_INFO,maat_json, - "compile rule %d's %s declared in previous compile rule, regions NOT take effect." - ,compile_id,group_name); - } - continue; - } - if(0==strncasecmp(group_name,untitled_group_name,strlen(untitled_group_name))) - { - group_info=&untitled_group; - group_info->group_id=get_group_seq(p_iris); - } - else - { - group_info=(struct group_info_t*)malloc(sizeof(struct group_info_t)); - group_info->group_id=get_group_seq(p_iris); - MESA_htable_add(p_iris->group_name_map,(const unsigned char*)group_name, strlen(group_name),group_info); - } - r_rules=cJSON_GetObjectItem(group_rule,"regions"); - if(r_rules==NULL) - { - MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_json, - "compile rule %d's %s has no region.",compile_id,group_name); - return -1; - } - region_cnt=cJSON_GetArraySize(r_rules); - if(region_cnt<=0) - { - MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_json, - "compile rule %d's %s has no region.",compile_id,group_name); - return -1; - } - for(k=0;kgroup_id, p_iris, logger); - if(ret<0) - { - MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_json, - "compile rule %d write region error.",compile_id); - return -1; - } - } - ret=write_group_rule(compile_id, group_info->group_id, group_not_flag, p_iris, logger); + ret=write_group_rule(group_rule, compile_id, PARENT_TYPE_COMPILE, compile_id, p_iris, logger); if(ret<0) { - MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_json, - "compile rule %d write group error.",compile_id); return -1; } - } } ret=write_index_file(p_iris,logger); diff --git a/src/inc_internal/Maat_rule_internal.h b/src/inc_internal/Maat_rule_internal.h index b8cf6b3..ee50471 100644 --- a/src/inc_internal/Maat_rule_internal.h +++ b/src/inc_internal/Maat_rule_internal.h @@ -142,6 +142,8 @@ struct Maat_group_inner int ref_cnt; char* group_name; int has_compile_neighbors; + int endpoint_cnt; + int* endpoints; dynamic_array_t *regions; void* compile_shortcut; pthread_mutex_t mutex; diff --git a/test/maat_json.json b/test/maat_json.json index d9dbbe6..d837d85 100644 --- a/test/maat_json.json +++ b/test/maat_json.json @@ -881,7 +881,44 @@ } } ], - "group_name": "Untitled" + "group_name": "152_mail_addr" + } + ] + }, + { + "compile_id": 153, + "service": 0, + "action": 0, + "do_blacklist": 0, + "do_log": 0, + "effective_rage": 0, + "user_region": "group_referer_group", + "is_valid": "yes", + "groups": [ + { + "regions": [ + { + "table_type": "expr", + "table_name": "MAIL_ADDR", + "table_content": { + "keywords": "ceshi4@mailhost.cn", + "expr_type": "none", + "format": "uncase plain", + "match_method": "prefix" + } + } + ], + "group_name": "Untitled", + "sub_groups": [ + {"group_name": "152_mail_addr"} + ], + "not_flag" : 0 + }, + { + "group_name": "IP_group_refered", + "sub_groups": [ + {"group_name": "IP_group"} + ] } ] } diff --git a/test/test_maatframe.cpp b/test/test_maatframe.cpp index cdb545e..cda125f 100644 --- a/test/test_maatframe.cpp +++ b/test/test_maatframe.cpp @@ -959,7 +959,7 @@ void accept_tags_entry_cb(int table_id,const char* table_line,void* u_para) return; } -TEST(RuleTags, Plugin1) +TEST(Policy, PluginRuleTags1) { #define RuleTags_Plugin int ret=0; @@ -985,7 +985,7 @@ void accept_tags_entry2_cb(int table_id,const char* table_line,void* u_para) return; } -TEST(RuleTags, Plugin2) +TEST(Policy, PluginRuleTags2) { #define RuleTags_Plugin2 int ret=0; @@ -1005,7 +1005,7 @@ TEST(RuleTags, Plugin2) return; } -TEST(RuleTags, Compile) +TEST(Policy, CompileRuleTags) { #define RuleTags_Compile int ret=0; @@ -1086,7 +1086,7 @@ void compile_ex_param_dup(int idx, MAAT_RULE_EX_DATA *to, MAAT_RULE_EX_DATA *fro -TEST(EX_DATA_INDEX, MaatRuleEXData) +TEST(Policy, CompileEXData) { #define rule_EX_data_index @@ -1120,6 +1120,42 @@ TEST(EX_DATA_INDEX, MaatRuleEXData) Maat_clean_status(&mid); return; } +TEST(Policy, SubGroup) +{ +#define TestSubGroup + int ret=0, table_id=0; + const char* scan_string="ceshi3@mailhost.cn"; + struct Maat_rule_t result[4]; + memset(result, 0, sizeof(result)); + + scan_status_t mid=NULL; + struct ipaddr ipv4_addr; + struct stream_tuple4_v4 v4_addr; + ipv4_addr.addrtype=ADDR_TYPE_IPV4; + inet_pton(AF_INET,"10.0.6.205",&(v4_addr.saddr)); + v4_addr.source=htons(50001); + inet_pton(AF_INET,"10.0.6.201",&(v4_addr.daddr)); + v4_addr.dest=htons(80); + ipv4_addr.v4=&v4_addr; + + table_id=Maat_table_register(g_feather,"MAIL_ADDR"); + ASSERT_GT(table_id, 0); + + ret=Maat_full_scan_string(g_feather, table_id,CHARSET_GBK, scan_string, strlen(scan_string), + result,NULL, 4, &mid, 0); + EXPECT_EQ(ret, -2); + + table_id=Maat_table_register(g_feather, "IP_CONFIG"); + ASSERT_GT(table_id, 0); + + ret=Maat_scan_proto_addr(g_feather,table_id,&ipv4_addr,6,result,4, &mid,0); + + EXPECT_EQ(ret, 1); + EXPECT_EQ(result[0].config_id, 153); + + Maat_clean_status(&mid); + return; +} TEST(StreamFuzzyHash, Pure) @@ -2105,7 +2141,7 @@ int main(int argc, char ** argv) const char* decrypt_key="mesa2017wy"; const char* accept_tags="{\"tags\":[{\"tag\":\"location\",\"value\":\"北京/朝阳/华严北里/甲22号\"},{\"tag\":\"isp\",\"value\":\"移动\"},{\"tag\":\"location\",\"value\":\"Astana\"}]}"; // const char* subsitute_acc_tags="{\"tags\":[{\"tag\":\"location\",\"value\":\"Astana\"}]}"; - int scan_detail=0; + int scan_detail=0, ret=0; ::testing::InitGoogleTest(&argc, argv); g_logger=MESA_create_runtime_log_handle(log_file, 0); @@ -2113,8 +2149,8 @@ int main(int argc, char ** argv) g_feather=Maat_feather(g_iThreadNum, table_info_path, g_logger); Maat_set_feather_opt(g_feather, MAAT_OPT_INSTANCE_NAME, "demo", strlen("demo")+1); Maat_set_feather_opt(g_feather, MAAT_OPT_DECRYPT_KEY, decrypt_key, strlen(decrypt_key)+1); - Maat_set_feather_opt(g_feather, MAAT_OPT_JSON_FILE_PATH, json_path, strlen(json_path)+1); - + ret=Maat_set_feather_opt(g_feather, MAAT_OPT_JSON_FILE_PATH, json_path, strlen(json_path)+1); + assert(ret==0); Maat_set_feather_opt(g_feather, MAAT_OPT_SCANDIR_INTERVAL_MS, &scan_interval_ms, sizeof(scan_interval_ms)); //Set a short intevral for testing. Maat_set_feather_opt(g_feather, MAAT_OPT_EFFECT_INVERVAL_MS, &effective_interval_ms, sizeof(effective_interval_ms)); @@ -2127,7 +2163,7 @@ int main(int argc, char ** argv) Maat_initiate_feather(g_feather); printf("Maat initiating, see %s\n",log_file); - int ret=RUN_ALL_TESTS(); + ret=RUN_ALL_TESTS(); Maat_burn_feather(g_feather); MESA_destroy_runtime_log_handle(g_logger); return ret;