From d954c5a01192411829e062a52a0123bc17d21741 Mon Sep 17 00:00:00 2001 From: zhengchao Date: Wed, 20 Oct 2021 00:02:43 +0300 Subject: [PATCH] =?UTF-8?q?expr=5Fplus=E5=92=8Cinterval=5Fplus=E8=A1=A8?= =?UTF-8?q?=E7=9A=84district=E5=AF=B9=E8=8B=B1=E6=96=87=E5=A4=A7=E5=B0=8F?= =?UTF-8?q?=E5=86=99=E4=B8=8D=E6=95=8F=E6=84=9F=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/entry/Maat_api.cpp | 2 +- src/entry/Maat_rule.cpp | 20 +-- src/entry/Maat_table.cpp | 173 ++++++++++++------------ src/entry/json2iris.cpp | 74 +++++----- src/entry/map_str2int.cpp | 188 ++++++++++++++------------ src/inc_internal/Maat_rule_internal.h | 4 +- src/inc_internal/map_str2int.h | 20 +-- test/maat_json.json | 2 +- test/test_maatframe.cpp | 2 +- 9 files changed, 248 insertions(+), 237 deletions(-) diff --git a/src/entry/Maat_api.cpp b/src/entry/Maat_api.cpp index c71ce00..8e0da0c 100644 --- a/src/entry/Maat_api.cpp +++ b/src/entry/Maat_api.cpp @@ -2494,7 +2494,7 @@ int Maat_set_scan_status(Maat_feather_t feather,scan_status_t* mid,enum MAAT_SCA _feather->scan_err_cnt++; return -1; } - map_ret=map_unNullstr2int(_feather->scanner->district_map,(const char*)value,size,&(_mid->district_id)); + map_ret=maat_kv_read_unNull(_feather->scanner->district_map, (const char*)value, size, &(_mid->district_id)); if(map_ret<0) { //May be the district is not effected yet. diff --git a/src/entry/Maat_rule.cpp b/src/entry/Maat_rule.cpp index 34a269d..3dc42c0 100644 --- a/src/entry/Maat_rule.cpp +++ b/src/entry/Maat_rule.cpp @@ -57,7 +57,7 @@ extern "C" } #endif -int MAAT_FRAME_VERSION_3_4_11_20211012=1; +int MAAT_FRAME_VERSION_3_5_0_20211020=1; int is_valid_table_name(const char* str) { @@ -763,7 +763,7 @@ struct Maat_scanner* create_maat_scanner(unsigned int version, _Maat_feather_t * Maat_hierarchy_set_compile_user_data_free_func(scanner->hier, (void (*)(void*))destroy_compile_rule); Maat_hierarchy_set_region_user_data_free_func(scanner->hier, (void (*)(void*))Maat_region_inner_free); - scanner->district_map=map_create(); + scanner->district_map=maat_kv_store_new(); scanner->version=version; scanner->cfg_num=0; @@ -809,7 +809,7 @@ void destroy_maat_scanner(struct Maat_scanner*scanner) } rulescan_destroy(scanner->region); - map_destroy(scanner->district_map); + maat_kv_store_free(scanner->district_map); scanner->district_map=NULL; assert(scanner->tmp_district_map==NULL); @@ -1036,19 +1036,19 @@ void rulescan_batch_update(rule_scanner_t rs_handle,MESA_lqueue_head expr_queue, int get_district_id(Maat_scanner *scanner,const char* district_str) { - int map_ret=0,district_id=-1; - map_ret=map_str2int(scanner->district_map, district_str,&district_id); + int map_ret=0, district_id=-1; + map_ret=maat_kv_read(scanner->district_map, district_str, &district_id); if(map_ret<0) { if(scanner->tmp_district_map==NULL) { - scanner->tmp_district_map=map_duplicate(scanner->district_map); + scanner->tmp_district_map=maat_kv_store_duplicate(scanner->district_map); } - map_ret=map_str2int(scanner->tmp_district_map, district_str,&district_id); + map_ret=maat_kv_read(scanner->tmp_district_map, district_str, &district_id); if(map_ret<0) { - district_id= scanner->district_num; - map_register(scanner->tmp_district_map,district_str, district_id); + district_id=scanner->district_num; + maat_kv_register(scanner->tmp_district_map, district_str, district_id); scanner->district_num++; } } @@ -2391,7 +2391,7 @@ void do_scanner_update(struct Maat_scanner* scanner, int scan_thread_num, void* tmp_map=scanner->district_map; scanner->district_map=scanner->tmp_district_map; scanner->tmp_district_map=NULL; - Maat_garbage_bagging(scanner->ref_garbage_bin, tmp_map, (void (*)(void*))map_destroy); + Maat_garbage_bagging(scanner->ref_garbage_bin, tmp_map, (void (*)(void*))maat_kv_store_free); } scanner->last_update_time=time(NULL); scanner->xx_plugin_rule_to_update_cnt=0; diff --git a/src/entry/Maat_table.cpp b/src/entry/Maat_table.cpp index 552eb94..3d8615b 100644 --- a/src/entry/Maat_table.cpp +++ b/src/entry/Maat_table.cpp @@ -15,7 +15,7 @@ struct Maat_table_manager { struct Maat_table_schema* p_table_info[MAX_TABLE_NUM]; size_t table_cnt; - MESA_htable_handle map_tablename2id; + struct maat_kv_store* map_tablename2id; int active_plugin_table_num; int is_last_plugin_table_updating; void* logger; @@ -58,31 +58,32 @@ enum MAAT_SCAN_TYPE Maat_table_get_scan_type(enum MAAT_TABLE_TYPE table_type) return ret; } -int read_expr_table_info(const char* line, struct Maat_table_schema* table, MESA_htable_handle string2int_map) +int read_expr_table_info(const char* line, struct Maat_table_schema* table, struct maat_kv_store* string2int_map) { - int j=0,ret[4]={0}; - char table_type[16],src_charset[256],dst_charset[256],merge[4],quick_str_scan[32]={0}; - char *token=NULL,*sub_token=NULL,*saveptr; + int j=0, ret[4]={0}; + char table_type[16], src_charset[256], dst_charset[256], merge[4], quick_str_scan[32]={0}; + char *token=NULL, *sub_token=NULL, *saveptr; struct expr_table_schema* p=&(table->expr); - sscanf(line,"%d\t%s\t%s\t%s\t%s\t%s\t%d\t%s",&(table->table_id) - ,table->table_name[0] - ,table_type - ,src_charset - ,dst_charset - ,merge - ,&(p->cross_cache_size) - ,quick_str_scan); - memset(ret,0,sizeof(ret)); - ret[0]=map_str2int(string2int_map,str_tolower(table_type),(int*)&(table->table_type)); - ret[1]=map_str2int(string2int_map,str_tolower(src_charset),(int*)&(p->src_charset)); - ret[2]=map_str2int(string2int_map,str_tolower(merge),&(p->do_charset_merge)); + sscanf(line, "%d\t%s\t%s\t%s\t%s\t%s\t%d\t%s",&(table->table_id), + table->table_name[0], + table_type, + src_charset, + dst_charset, + merge, + &(p->cross_cache_size), + quick_str_scan); + memset(ret, 0, sizeof(ret)); + + ret[0]=maat_kv_read(string2int_map, table_type, (int*)&(table->table_type)); + ret[1]=maat_kv_read(string2int_map, src_charset, (int*)&(p->src_charset)); + ret[2]=maat_kv_read(string2int_map, merge, &(p->do_charset_merge)); if(strlen(quick_str_scan)>0) { - ret[3]=map_str2int(string2int_map,str_tolower(quick_str_scan),&(p->quick_expr_switch)); + ret[3]=maat_kv_read(string2int_map, quick_str_scan, &(p->quick_expr_switch)); } - memset(quick_str_scan,0,sizeof(quick_str_scan)); + memset(quick_str_scan, 0, sizeof(quick_str_scan)); - for(j=0;j<4;j++) + for(j=0; j<4; j++) { if(ret[j]<0) { @@ -92,10 +93,10 @@ int read_expr_table_info(const char* line, struct Maat_table_schema* table, MESA j=0; for (token = dst_charset; ; token= NULL) { - sub_token= strtok_r(token,"/", &saveptr); + sub_token= strtok_r(token, "/", &saveptr); if (sub_token == NULL) break; - ret[3]=map_str2int(string2int_map,str_tolower(sub_token),(int*)&(p->dst_charset[j])); + ret[3]=maat_kv_read(string2int_map, sub_token, (int*)&(p->dst_charset[j])); if(ret[3]>0) { if(p->dst_charset[j]==p->src_charset) @@ -181,7 +182,7 @@ int read_virtual_table_schema(struct Maat_table_manager* table_mgr, const char* { goto error_out; } - ret=map_str2int(table_mgr->map_tablename2id, tmp->valuestring, &tmp_table_id); + ret=maat_kv_read(table_mgr->map_tablename2id, tmp->valuestring, &tmp_table_id); if(ret<0) { goto error_out; @@ -197,7 +198,7 @@ int read_virtual_table_schema(struct Maat_table_manager* table_mgr, const char* } else //For compatible non-json physical description { - ret=map_str2int(table_mgr->map_tablename2id, json_str, &tmp_table_id); + ret=maat_kv_read(table_mgr->map_tablename2id, json_str, &tmp_table_id); if(ret<0) { goto error_out; @@ -483,7 +484,7 @@ int read_composition_table_schema(struct Maat_table_manager* table_mgr, const ch tmp=cJSON_GetObjectItem(json, "source"); if(tmp!=NULL && tmp->type==cJSON_String) { - ret=map_str2int(table_mgr->map_tablename2id, tmp->valuestring, &(composition_schema->component_table_id[COMPONENT_TABLE_TYPE_SOURCE_IP])); + ret=maat_kv_read(table_mgr->map_tablename2id, tmp->valuestring, &(composition_schema->component_table_id[COMPONENT_TABLE_TYPE_SOURCE_IP])); if(ret<0) { MESA_handle_runtime_log(table_mgr->logger, RLOG_LV_FATAL, maat_module, @@ -497,7 +498,7 @@ int read_composition_table_schema(struct Maat_table_manager* table_mgr, const ch tmp=cJSON_GetObjectItem(json, "destination"); if(tmp!=NULL && tmp->type==cJSON_String) { - ret=map_str2int(table_mgr->map_tablename2id, tmp->valuestring, &(composition_schema->component_table_id[COMPONENT_TABLE_TYPE_DESTINATION_IP])); + ret=maat_kv_read(table_mgr->map_tablename2id, tmp->valuestring, &(composition_schema->component_table_id[COMPONENT_TABLE_TYPE_DESTINATION_IP])); if(ret<0) { MESA_handle_runtime_log(table_mgr->logger, RLOG_LV_FATAL, maat_module, @@ -512,7 +513,7 @@ int read_composition_table_schema(struct Maat_table_manager* table_mgr, const ch tmp=cJSON_GetObjectItem(json, "session"); if(tmp!=NULL && tmp->type==cJSON_String) { - ret=map_str2int(table_mgr->map_tablename2id, tmp->valuestring, &(composition_schema->component_table_id[COMPONENT_TABLE_TYPE_SESSION])); + ret=maat_kv_read(table_mgr->map_tablename2id, tmp->valuestring, &(composition_schema->component_table_id[COMPONENT_TABLE_TYPE_SESSION])); if(ret<0) { MESA_handle_runtime_log(table_mgr->logger, RLOG_LV_FATAL, maat_module, @@ -544,7 +545,7 @@ void Maat_table_manager_destroy(struct Maat_table_manager* table_mgr) table_info_free(table_mgr->p_table_info[i]); table_mgr->p_table_info[i]=NULL; } - MESA_htable_destroy(table_mgr->map_tablename2id, free); + maat_kv_store_free(table_mgr->map_tablename2id); free(table_mgr); return; } @@ -556,7 +557,7 @@ struct Maat_table_manager* Maat_table_manager_create(const char* table_info_path char line[MAX_TABLE_LINE_SIZE]; int i=0, ret=0; char table_type_str[16]={0},not_care[1024]={0}, tmp_str[32]={0}; - MESA_htable_handle reserved_word_map=NULL;; + struct maat_kv_store* reserved_word_map=NULL;; struct Maat_table_schema*p=NULL; struct Maat_table_schema*conj_table=NULL; fp=fopen(table_info_path,"r"); @@ -571,39 +572,39 @@ struct Maat_table_manager* Maat_table_manager_create(const char* table_info_path struct Maat_table_schema** p_table_info=table_mgr->p_table_info; size_t n_table=MAX_TABLE_NUM; table_mgr->logger=logger; - table_mgr->map_tablename2id=map_create(); + table_mgr->map_tablename2id=maat_kv_store_new(); - reserved_word_map=map_create(); - map_register(reserved_word_map, "expr", TABLE_TYPE_EXPR); - map_register(reserved_word_map, "ip", TABLE_TYPE_IP); - map_register(reserved_word_map, "ip_plus", TABLE_TYPE_IP_PLUS); - map_register(reserved_word_map, "compile", TABLE_TYPE_COMPILE); - map_register(reserved_word_map, "plugin", TABLE_TYPE_PLUGIN); - map_register(reserved_word_map, "ip_plugin", TABLE_TYPE_IP_PLUGIN); - map_register(reserved_word_map, "fqdn_plugin", TABLE_TYPE_FQDN_PLUGIN); - map_register(reserved_word_map, "intval", TABLE_TYPE_INTERVAL); - map_register(reserved_word_map, "interval", TABLE_TYPE_INTERVAL); - map_register(reserved_word_map, "intval_plus", TABLE_TYPE_INTERVAL_PLUS); - map_register(reserved_word_map, "interval_plus", TABLE_TYPE_INTERVAL_PLUS); - map_register(reserved_word_map, "digest", TABLE_TYPE_DIGEST); - map_register(reserved_word_map, "expr_plus", TABLE_TYPE_EXPR_PLUS); - map_register(reserved_word_map, "group", TABLE_TYPE_GROUP); - map_register(reserved_word_map, "group2group", TABLE_TYPE_GROUP2GROUP); - map_register(reserved_word_map, "group2compile", TABLE_TYPE_GROUP2COMPILE); - map_register(reserved_word_map, "similar", TABLE_TYPE_SIMILARITY); - map_register(reserved_word_map, "virtual", TABLE_TYPE_VIRTUAL); - map_register(reserved_word_map, "composition", TABLE_TYPE_COMPOSITION); - map_register(reserved_word_map, "quickoff", 0); - map_register(reserved_word_map, "quickon", 1); - map_register(reserved_word_map, "escape", USER_REGION_ENCODE_ESCAPE); -// map_register(reserved_word_map,"base64",USER_REGION_ENCODE_BASE64); //NOT supported yet + reserved_word_map=maat_kv_store_new(); + maat_kv_register(reserved_word_map, "expr", TABLE_TYPE_EXPR); + maat_kv_register(reserved_word_map, "ip", TABLE_TYPE_IP); + maat_kv_register(reserved_word_map, "ip_plus", TABLE_TYPE_IP_PLUS); + maat_kv_register(reserved_word_map, "compile", TABLE_TYPE_COMPILE); + maat_kv_register(reserved_word_map, "plugin", TABLE_TYPE_PLUGIN); + maat_kv_register(reserved_word_map, "ip_plugin", TABLE_TYPE_IP_PLUGIN); + maat_kv_register(reserved_word_map, "fqdn_plugin", TABLE_TYPE_FQDN_PLUGIN); + maat_kv_register(reserved_word_map, "intval", TABLE_TYPE_INTERVAL); + maat_kv_register(reserved_word_map, "interval", TABLE_TYPE_INTERVAL); + maat_kv_register(reserved_word_map, "intval_plus", TABLE_TYPE_INTERVAL_PLUS); + maat_kv_register(reserved_word_map, "interval_plus", TABLE_TYPE_INTERVAL_PLUS); + maat_kv_register(reserved_word_map, "digest", TABLE_TYPE_DIGEST); + maat_kv_register(reserved_word_map, "expr_plus", TABLE_TYPE_EXPR_PLUS); + maat_kv_register(reserved_word_map, "group", TABLE_TYPE_GROUP); + maat_kv_register(reserved_word_map, "group2group", TABLE_TYPE_GROUP2GROUP); + maat_kv_register(reserved_word_map, "group2compile", TABLE_TYPE_GROUP2COMPILE); + maat_kv_register(reserved_word_map, "similar", TABLE_TYPE_SIMILARITY); + maat_kv_register(reserved_word_map, "virtual", TABLE_TYPE_VIRTUAL); + maat_kv_register(reserved_word_map, "composition", TABLE_TYPE_COMPOSITION); + maat_kv_register(reserved_word_map, "quickoff", 0); + maat_kv_register(reserved_word_map, "quickon", 1); + maat_kv_register(reserved_word_map, "escape", USER_REGION_ENCODE_ESCAPE); +// maat_kv_register(reserved_word_map,"base64",USER_REGION_ENCODE_BASE64); //NOT supported yet const char** charset_name_list=charset_get_all_name(); for(i=0;i0) { - map_register(reserved_word_map, charset_name_list[i], i); + maat_kv_register(reserved_word_map, charset_name_list[i], i); } else { @@ -611,8 +612,8 @@ struct Maat_table_manager* Maat_table_manager_create(const char* table_info_path } } - map_register(reserved_word_map,"yes", 1); - map_register(reserved_word_map,"no", 0); + maat_kv_register(reserved_word_map,"yes", 1); + maat_kv_register(reserved_word_map,"no", 0); i=0; @@ -626,20 +627,20 @@ struct Maat_table_manager* Maat_table_manager_create(const char* table_info_path } p=table_info_new(); - ret=sscanf(line,"%d\t%s\t%s\t%[a-z0-9\t ]",&(p->table_id) - ,p->table_name[0] - ,table_type_str - ,not_care); + ret=sscanf(line, "%d\t%s\t%s\t%[a-z0-9\t ]", &(p->table_id), + p->table_name[0], + table_type_str, + not_care); if(ret<3) { - MESA_handle_runtime_log(logger, RLOG_LV_FATAL,maat_module, + MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, "Maat read table info %s line %d error: not enough column.",table_info_path,i); continue; } - ret=map_str2int(reserved_word_map,str_tolower(table_type_str),(int*)&(p->table_type)); + ret=maat_kv_read(reserved_word_map, table_type_str, (int*)&(p->table_type)); if(ret<0) { - MESA_handle_runtime_log(logger, RLOG_LV_FATAL,maat_module, + MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, "Maat read table info %s line %d error:invalid table type.",table_info_path,i); goto invalid_table; } @@ -650,9 +651,9 @@ struct Maat_table_manager* Maat_table_manager_create(const char* table_info_path ret=read_expr_table_info(line, p, reserved_word_map); if(ret<0) { - fprintf(stderr,"Maat read table info %s line %d error:illegal column.\n",table_info_path,i); + fprintf(stderr, "Maat read table info %s line %d error: illegal column.\n", table_info_path, i); MESA_handle_runtime_log(logger, RLOG_LV_FATAL,maat_module, - "Maat read table info %s line %d error:illegal column.",table_info_path,i); + "Maat read table info %s line %d error:illegal column.", table_info_path, i); goto invalid_table; } break; @@ -660,9 +661,9 @@ struct Maat_table_manager* Maat_table_manager_create(const char* table_info_path ret=read_plugin_table_schema(line, p); if(ret<0) { - fprintf(stderr,"Maat read table info %s line %d error:illegal plugin table schema.\n", table_info_path,i); + fprintf(stderr, "Maat read table info %s line %d error: illegal plugin table schema.\n", table_info_path, i); MESA_handle_runtime_log(logger, RLOG_LV_FATAL,maat_module, - "Maat read table info %s line %d error:illegal plugin table schema.", table_info_path,i); + "Maat read table info %s line %d error: illegal plugin table schema.", table_info_path, i); goto invalid_table; } break; @@ -670,9 +671,9 @@ struct Maat_table_manager* Maat_table_manager_create(const char* table_info_path ret=read_ip_plugin_table_schema(line, p); if(ret<0) { - fprintf(stderr,"Maat read table info %s line %d error:illegal ip_plugin table schema.\n", table_info_path,i); - MESA_handle_runtime_log(logger, RLOG_LV_FATAL,maat_module, - "Maat read table info %s line %d error:illegal ip_plugin table schema.", table_info_path,i); + fprintf(stderr, "Maat read table info %s line %d error: illegal ip_plugin table schema.\n", table_info_path, i); + MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, + "Maat read table info %s line %d error: illegal ip_plugin table schema.", table_info_path, i); goto invalid_table; } break; @@ -680,9 +681,9 @@ struct Maat_table_manager* Maat_table_manager_create(const char* table_info_path ret=read_fqdn_plugin_table_schema(line, p); if(ret<0) { - fprintf(stderr,"Maat read table info %s line %d error:illegal fqdn_plugin table schema.\n", table_info_path,i); - MESA_handle_runtime_log(logger, RLOG_LV_FATAL,maat_module, - "Maat read table info %s line %d error:illegal fqdn_plugin table schema.", table_info_path,i); + fprintf(stderr, "Maat read table info %s line %d error: illegal fqdn_plugin table schema.\n", table_info_path, i); + MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, + "Maat read table info %s line %d error: illegal fqdn_plugin table schema.", table_info_path, i); goto invalid_table; } break; @@ -690,9 +691,9 @@ struct Maat_table_manager* Maat_table_manager_create(const char* table_info_path ret=read_composition_table_schema(table_mgr, line, p, reserved_word_map); if(ret<0) { - fprintf(stderr,"Maat read table info %s line %d error:illegal composition table schema.\n", table_info_path,i); - MESA_handle_runtime_log(logger, RLOG_LV_FATAL,maat_module, - "Maat read table info %s line %d error:illegal composition table schema.", table_info_path,i); + fprintf(stderr, "Maat read table info %s line %d error: illegal composition table schema.\n", table_info_path, i); + MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, + "Maat read table info %s line %d error: illegal composition table schema.", table_info_path, i); goto invalid_table; } break; @@ -700,17 +701,17 @@ struct Maat_table_manager* Maat_table_manager_create(const char* table_info_path ret=read_virtual_table_schema(table_mgr, line, p, reserved_word_map); if(ret<0) { - fprintf(stderr,"Maat read table info %s line %d error:illegal virtual table schema.\n", table_info_path,i); - MESA_handle_runtime_log(logger, RLOG_LV_FATAL,maat_module, - "Maat read table info %s line %d error:illegal virtual table schema.", table_info_path,i); + fprintf(stderr, "Maat read table info %s line %d error: illegal virtual table schema.\n", table_info_path, i); + MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, + "Maat read table info %s line %d error: illegal virtual table schema.", table_info_path, i); goto invalid_table; } break; case TABLE_TYPE_COMPILE: - ret=sscanf(not_care,"%[a-z0-9]",tmp_str); + ret=sscanf(not_care,"%[a-z0-9]", tmp_str); if(ret>0) { - ret=map_str2int(reserved_word_map,str_tolower(tmp_str),(int*)&(p->compile.user_region_encoding)); + ret=maat_kv_read(reserved_word_map, tmp_str, (int*)&(p->compile.user_region_encoding)); } if(ret!=1) { @@ -729,7 +730,7 @@ struct Maat_table_manager* Maat_table_manager_create(const char* table_info_path goto invalid_table; } - ret=map_register(table_mgr->map_tablename2id, p->table_name[0], p->table_id); + ret=maat_kv_register(table_mgr->map_tablename2id, p->table_name[0], p->table_id); if(ret<0) { MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, @@ -744,7 +745,7 @@ struct Maat_table_manager* Maat_table_manager_create(const char* table_info_path if(conj_table->conj_cnt==MAX_CONJUNCTION_TABLE_NUM) { MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, - "Maat read table info %s line %d error:reach tableid %d conjunction upper limit." + "Maat read table info %s line %d error: reach tableid %d conjunction upper limit." ,table_info_path,i,p->table_id); goto invalid_table; } @@ -766,7 +767,7 @@ invalid_table: p=NULL; } fclose(fp); - map_destroy(reserved_word_map); + maat_kv_store_free(reserved_word_map); return table_mgr; } size_t Maat_table_manager_get_size(struct Maat_table_manager* table_mgr) @@ -892,7 +893,7 @@ struct Maat_table_schema * Maat_table_get_scan_by_id(struct Maat_table_manager* int Maat_table_get_id_by_name(struct Maat_table_manager* table_mgr, const char* table_name) { int table_id=-1,ret=0; - ret=map_str2int(table_mgr->map_tablename2id, table_name, &table_id); + ret=maat_kv_read(table_mgr->map_tablename2id, table_name, &table_id); if(ret>0) { return table_id; diff --git a/src/entry/json2iris.cpp b/src/entry/json2iris.cpp index 037839e..bebf4be 100644 --- a/src/entry/json2iris.cpp +++ b/src/entry/json2iris.cpp @@ -52,7 +52,7 @@ struct iris_description_t struct iris_table_t* compile_table; MESA_htable_handle group_name_map; MESA_htable_handle iris_table_map; - MESA_htable_handle str2int_map; + struct maat_kv_store* str2int_map; redisContext *redis_write_ctx; char* encrypt_key; char* encrypt_algo; @@ -155,46 +155,46 @@ int set_iris_descriptor(const char* json_file,cJSON *json, const char* encrypt_k iris_cfg->iris_table_map=MESA_htable_create(&hargs, sizeof(hargs)); MESA_htable_print_crtl(iris_cfg->iris_table_map, 0); - iris_cfg->str2int_map=map_create(); + iris_cfg->str2int_map=maat_kv_store_new(); - map_register(iris_cfg->str2int_map, "yes",1); - map_register(iris_cfg->str2int_map, "no",0); + maat_kv_register(iris_cfg->str2int_map, "yes",1); + maat_kv_register(iris_cfg->str2int_map, "no",0); - map_register(iris_cfg->str2int_map, "ip",TABLE_TYPE_IP); - map_register(iris_cfg->str2int_map, "ip_plus",TABLE_TYPE_IP_PLUS); - map_register(iris_cfg->str2int_map, "string",TABLE_TYPE_EXPR); - map_register(iris_cfg->str2int_map, "expr",TABLE_TYPE_EXPR); - map_register(iris_cfg->str2int_map, "expr_plus",TABLE_TYPE_EXPR_PLUS); - map_register(iris_cfg->str2int_map, "intval",TABLE_TYPE_INTERVAL); - map_register(iris_cfg->str2int_map, "interval",TABLE_TYPE_INTERVAL); - map_register(iris_cfg->str2int_map, "intval_plus",TABLE_TYPE_INTERVAL_PLUS); - map_register(iris_cfg->str2int_map, "interval_plus",TABLE_TYPE_INTERVAL_PLUS); - map_register(iris_cfg->str2int_map, "digest",TABLE_TYPE_DIGEST); - map_register(iris_cfg->str2int_map, "similar",TABLE_TYPE_SIMILARITY); + maat_kv_register(iris_cfg->str2int_map, "ip",TABLE_TYPE_IP); + maat_kv_register(iris_cfg->str2int_map, "ip_plus",TABLE_TYPE_IP_PLUS); + maat_kv_register(iris_cfg->str2int_map, "string",TABLE_TYPE_EXPR); + maat_kv_register(iris_cfg->str2int_map, "expr",TABLE_TYPE_EXPR); + maat_kv_register(iris_cfg->str2int_map, "expr_plus",TABLE_TYPE_EXPR_PLUS); + maat_kv_register(iris_cfg->str2int_map, "intval",TABLE_TYPE_INTERVAL); + maat_kv_register(iris_cfg->str2int_map, "interval",TABLE_TYPE_INTERVAL); + maat_kv_register(iris_cfg->str2int_map, "intval_plus",TABLE_TYPE_INTERVAL_PLUS); + maat_kv_register(iris_cfg->str2int_map, "interval_plus",TABLE_TYPE_INTERVAL_PLUS); + maat_kv_register(iris_cfg->str2int_map, "digest",TABLE_TYPE_DIGEST); + maat_kv_register(iris_cfg->str2int_map, "similar",TABLE_TYPE_SIMILARITY); - map_register(iris_cfg->str2int_map, "ipv4",4); - map_register(iris_cfg->str2int_map, "ipv6",6); + maat_kv_register(iris_cfg->str2int_map, "ipv4",4); + maat_kv_register(iris_cfg->str2int_map, "ipv6",6); - map_register(iris_cfg->str2int_map, "double",0); - map_register(iris_cfg->str2int_map, "single",1); + maat_kv_register(iris_cfg->str2int_map, "double",0); + maat_kv_register(iris_cfg->str2int_map, "single",1); - map_register(iris_cfg->str2int_map, "none",0); - map_register(iris_cfg->str2int_map, "and",1); - map_register(iris_cfg->str2int_map, "regex",2); - map_register(iris_cfg->str2int_map, "offset",3); + maat_kv_register(iris_cfg->str2int_map, "none",0); + maat_kv_register(iris_cfg->str2int_map, "and",1); + maat_kv_register(iris_cfg->str2int_map, "regex",2); + maat_kv_register(iris_cfg->str2int_map, "offset",3); - map_register(iris_cfg->str2int_map, "sub",0); - map_register(iris_cfg->str2int_map, "right",1); - map_register(iris_cfg->str2int_map, "suffix",1); - map_register(iris_cfg->str2int_map, "left",2); - map_register(iris_cfg->str2int_map, "prefix",2); - map_register(iris_cfg->str2int_map, "complete",3); - map_register(iris_cfg->str2int_map, "exact",3); + maat_kv_register(iris_cfg->str2int_map, "sub",0); + maat_kv_register(iris_cfg->str2int_map, "right",1); + maat_kv_register(iris_cfg->str2int_map, "suffix",1); + maat_kv_register(iris_cfg->str2int_map, "left",2); + maat_kv_register(iris_cfg->str2int_map, "prefix",2); + maat_kv_register(iris_cfg->str2int_map, "complete",3); + maat_kv_register(iris_cfg->str2int_map, "exact",3); - map_register(iris_cfg->str2int_map, "uncase plain",0); - map_register(iris_cfg->str2int_map, "hexbin",1); - map_register(iris_cfg->str2int_map, "case plain",2); + maat_kv_register(iris_cfg->str2int_map, "uncase plain",0); + maat_kv_register(iris_cfg->str2int_map, "hexbin",1); + maat_kv_register(iris_cfg->str2int_map, "case plain",2); iris_cfg->compile_table=query_table_info(iris_cfg, compile_tn, TABLE_TYPE_COMPILE); iris_cfg->group2compile_table=query_table_info(iris_cfg, group2compile_tn, TABLE_TYPE_GROUP2COMPILE); @@ -218,7 +218,7 @@ void clear_iris_descriptor(struct iris_description_t *iris_cfg) { MESA_htable_destroy(iris_cfg->iris_table_map, NULL); } - map_destroy(iris_cfg->str2int_map); + maat_kv_store_free(iris_cfg->str2int_map); free(iris_cfg->encrypt_algo); free(iris_cfg->encrypt_key); return; @@ -264,7 +264,7 @@ int set_file_rulenum(const char* path,int rulenum,void* logger) fclose(fp); return 0; } -int direct_write_rule(cJSON* json, MESA_htable_handle str2int, struct traslate_command_t*cmd, int cmd_cnt, struct iris_table_t* table, void* logger) +int direct_write_rule(cJSON* json, struct maat_kv_store* str2int, struct traslate_command_t*cmd, int cmd_cnt, struct iris_table_t* table, void* logger) { int i=0,ret=-1; cJSON* item=NULL; @@ -293,7 +293,7 @@ int direct_write_rule(cJSON* json, MESA_htable_handle str2int, struct traslate_c if(cmd[i].str2int_flag==1) { p=item->valuestring; - ret=map_str2int(str2int, p, &int_value); + ret=maat_kv_read(str2int, p, &int_value); if(ret<0) { MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_json, @@ -748,7 +748,7 @@ int write_region_rule(cJSON* region_json, int compile_id, int group_id, iris_des return -1; } table_type_str=item->valuestring; - ret=map_str2int(p_iris->str2int_map, table_type_str, (int*)&(table_type)); + ret=maat_kv_read(p_iris->str2int_map, table_type_str, (int*)&(table_type)); if(ret!=1) { MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_json, diff --git a/src/entry/map_str2int.cpp b/src/entry/map_str2int.cpp index af252ea..a4265f7 100644 --- a/src/entry/map_str2int.cpp +++ b/src/entry/map_str2int.cpp @@ -1,114 +1,124 @@ -#include -#include +#include +#include "Maat_utils.h" +#include "uthash/uthash.h" -void map_tmp_free(void* ptr) -{ - free(ptr); -} +const size_t MAAT_KV_MAX_KEY_LEN=512; -long read_map_val(void *data, const uchar *key, uint size, void *user_arg) +struct maat_kv_pair { - if(data!=NULL) + char* key; //must be lower case. + size_t keylen; + int val; + UT_hash_handle hh; +}; +void strlowercase(const char* src, size_t src_len, char* dst, size_t dst_sz) +{ + size_t i=0; + for(i=0; ikeylen=keylen; + kv->val=value; + kv->key=ALLOC(char, keylen); + strlowercase(key, keylen, kv->key, kv->keylen); + return kv; +} +void maat_kv_pair_free(struct maat_kv_pair* kv) { - unsigned int size=strlen(string); - unsigned char *key=(unsigned char *)string; - int ret=0; - int * data= (int*)malloc(sizeof(int)); - *data=value; - ret=MESA_htable_add(handle,key,size,data); - if(ret<0) + free(kv->key); + kv->key=NULL; + free(kv); +} +struct maat_kv_store +{ + struct maat_kv_pair* hash; +}; + +struct maat_kv_store* maat_kv_store_new(void) +{ + + struct maat_kv_store* store=ALLOC(struct maat_kv_store, 1); + return store; +} +void maat_kv_store_free(struct maat_kv_store* store) +{ + struct maat_kv_pair* kv=NULL, *tmp_kv=NULL; + if(store==NULL) return; + HASH_ITER(hh, store->hash, kv, tmp_kv) { - free(data); + HASH_DEL(store->hash, kv); + maat_kv_pair_free(kv); } + free(store); + return; +} +int maat_kv_register_unNull(struct maat_kv_store* store, const char* key, size_t keylen, int value) +{ + struct maat_kv_pair *kv=NULL; + struct maat_kv_pair *tmp_kv=NULL; + if(keylen>MAAT_KV_MAX_KEY_LEN) + { + return -1; + } + kv=maat_kv_pair_new(key, keylen, value); + HASH_FIND(hh, store->hash, kv->key, keylen, tmp_kv); + if(tmp_kv) + { + maat_kv_pair_free(kv); + return -1; + } + HASH_ADD_KEYPTR(hh, store->hash, kv->key, keylen, kv); + return 1; +} + +int maat_kv_register(struct maat_kv_store* store, const char* key, int value) +{ + int ret=0; + ret=maat_kv_register_unNull(store, key, strlen(key), value); return ret; } -int map_str2int(MESA_htable_handle handle,const char* string,int* value) + +int maat_kv_read_unNull(struct maat_kv_store* store, const char* key, size_t keylen, int* value) { - long cb_ret=0; - unsigned int size=strlen(string); - - MESA_htable_search_cb(handle,(unsigned char*)string,size, - read_map_val,value,&cb_ret); - -// data=(int*)MESA_htable_search(handle,(unsigned char*)string,size); - if(cb_ret>0) + struct maat_kv_pair *kv=NULL; + char key_lowercase[MAAT_KV_MAX_KEY_LEN]={0}; + if(keylen>MAAT_KV_MAX_KEY_LEN) { + return -1; + } + strlowercase(key, keylen, key_lowercase, sizeof(key_lowercase)); + HASH_FIND(hh, store->hash, key_lowercase, keylen, kv); + if(kv) + { + *value=kv->val; return 1; } else { return -1; - } -} -int map_unNullstr2int(MESA_htable_handle handle,const char* string,int len,int* value) -{ - long cb_ret=0; - - MESA_htable_search_cb(handle,(unsigned char*)string,len, - read_map_val,value,&cb_ret); - -// data=(int*)MESA_htable_search(handle,(unsigned char*)string,len); - if(cb_ret>0) - { - return 1; } - else +} + +int maat_kv_read(struct maat_kv_store * store, const char * key, int * value) +{ + return maat_kv_read_unNull(store, key, strlen(key), value); +} +struct maat_kv_store* maat_kv_store_duplicate(struct maat_kv_store* origin_map) +{ + struct maat_kv_store* target=maat_kv_store_new(); + struct maat_kv_pair* kv=NULL, *tmp_kv=NULL, *copy_kv=NULL; + HASH_ITER(hh, origin_map->hash, kv, tmp_kv) { - return -1; + copy_kv=maat_kv_pair_new(kv->key, kv->keylen, kv->val); + HASH_ADD_KEYPTR(hh, target->hash, copy_kv->key, copy_kv->keylen, copy_kv); } -} -void duplicate_func(const uchar * key, uint size, void * data, void * user) -{ - MESA_htable_handle target=(MESA_htable_handle)user; - int value=*(int*)data; - __attribute__((unused)) int ret=0; - char* string=(char*)calloc(sizeof(char),(size+1)); - memcpy(string,key,size); - ret=map_register(target,string,value); - assert(ret>0); - free(string); -} -MESA_htable_handle map_duplicate(MESA_htable_handle origin_map) -{ - __attribute__((unused)) int ret=0; - MESA_htable_handle target=NULL; - target=map_create(); - ret=MESA_htable_iterate(origin_map, duplicate_func, target); - assert(ret==0); return target; } diff --git a/src/inc_internal/Maat_rule_internal.h b/src/inc_internal/Maat_rule_internal.h index 83f4928..a8689e6 100644 --- a/src/inc_internal/Maat_rule_internal.h +++ b/src/inc_internal/Maat_rule_internal.h @@ -223,8 +223,8 @@ struct Maat_scanner struct Maat_garbage_bin* ref_garbage_bin; - MESA_htable_handle district_map; - MESA_htable_handle tmp_district_map; + struct maat_kv_store* district_map; + struct maat_kv_store* tmp_district_map; int most_popular_sub_group; diff --git a/src/inc_internal/map_str2int.h b/src/inc_internal/map_str2int.h index dce9d97..b9c53c6 100644 --- a/src/inc_internal/map_str2int.h +++ b/src/inc_internal/map_str2int.h @@ -1,11 +1,11 @@ -#ifndef __MAP_STR2INT_H_INCLUDE_ -#define __MAP_STR2INT_H_INCLUDE_ -#include -MESA_htable_handle map_create(void); -void map_destroy(MESA_htable_handle p); -int map_register(MESA_htable_handle handle,const char* string,int value); -int map_str2int(MESA_htable_handle handle,const char* string,int* value); -int map_unNullstr2int(MESA_htable_handle handle,const char* string,int size,int* value); -MESA_htable_handle map_duplicate(MESA_htable_handle origin_map); -#endif +#pragma once + +struct maat_kv_store; +struct maat_kv_store* maat_kv_store_new(void); +void maat_kv_store_free(struct maat_kv_store* store); +int maat_kv_register(struct maat_kv_store* store, const char* key, int value); +int maat_kv_read(struct maat_kv_store* store, const char* key, int* value); +int maat_kv_read_unNull(struct maat_kv_store* store, const char* key, size_t key_sz, int* value); +struct maat_kv_store* maat_kv_store_duplicate(struct maat_kv_store* store); + diff --git a/test/maat_json.json b/test/maat_json.json index e57df32..19d086b 100644 --- a/test/maat_json.json +++ b/test/maat_json.json @@ -376,7 +376,7 @@ "table_name": "HTTP_SIGNATURE", "table_type": "expr_plus", "table_content": { - "district": "HTTP\\bURL", + "district": "HtTP\\bUrL", "keywords": "abckkk&123", "expr_type": "and", "match_method": "sub", diff --git a/test/test_maatframe.cpp b/test/test_maatframe.cpp index 3bb6bf6..336500c 100644 --- a/test/test_maatframe.cpp +++ b/test/test_maatframe.cpp @@ -581,7 +581,7 @@ TEST(StringScan, ExprPlus) ret=Maat_set_scan_status(g_feather, &mid, MAAT_SET_SCAN_DISTRICT, region_name, strlen(region_name)); ASSERT_EQ(ret, 0); - ret=Maat_full_scan_string(g_feather, table_id,CHARSET_GBK, scan_data, strlen(scan_data), + ret=Maat_full_scan_string(g_feather, table_id, CHARSET_GBK, scan_data, strlen(scan_data), result, found_pos, 4, &mid, 0); EXPECT_EQ(ret, 1);