diff --git a/src/entry/Maat_rule.cpp b/src/entry/Maat_rule.cpp index 50814a9..785a214 100644 --- a/src/entry/Maat_rule.cpp +++ b/src/entry/Maat_rule.cpp @@ -26,7 +26,7 @@ #include "mesa_fuzzy.h" #include "great_index_engine.h" -int MAAT_FRAME_VERSION_1_8_20160607_PERF_TEST=1; +int MAAT_FRAME_VERSION_1_8_20160617_PERF_TEST=1; const char *maat_module="MAAT Frame"; const char* CHARSET_STRING[]={"NONE","gbk","big5","unicode","utf8","bin", @@ -387,6 +387,7 @@ _Maat_table_info_t* create_table_info(int max_thread_num) p->stream_num=aligment_int64_array_alloc(max_thread_num); p->hit_cnt=aligment_int64_array_alloc(max_thread_num); p->cross_cache_size=0; + p->quick_expr_switch=0; return p; } void destroy_table_info(struct _Maat_table_info_t*p) @@ -458,13 +459,14 @@ int read_table_info(struct _Maat_table_info_t** p_table_info,int num,const char* } p=create_table_info(max_thread_num); - sscanf(line,"%hu\t%s\t%s\t%s\t%s\t%s\t%d",&(p->table_id) + sscanf(line,"%hu\t%s\t%s\t%s\t%s\t%s\t%d\t%d",&(p->table_id) ,p->table_name ,table_type ,src_charset ,dst_charset ,merge - ,&(p->cross_cache_size)); + ,&(p->cross_cache_size) + ,&(p->quick_expr_switch)); ret[0]=map_str2int(string2int_map,strlwr(table_type),(int*)&(p->table_type)); ret[1]=map_str2int(string2int_map,strlwr(src_charset),(int*)&(p->src_charset)); ret[2]=map_str2int(string2int_map,strlwr(merge),&(p->do_charset_merge)); @@ -837,9 +839,17 @@ void destroy_digest_rule(GIE_digest_t*rule) rule=NULL; return; } -struct _Maat_scanner_t* create_maat_scanner(unsigned int version,int scan_thread_num,MESA_lqueue_head tomb,int rs_scan_type) +struct _Maat_scanner_t* create_maat_scanner(unsigned int version,_Maat_feather_t *feather) { - int i=0; + int scan_thread_num=feather->scan_thread_num; + MESA_lqueue_head tomb=feather->garbage_q; +// int rs_scan_type=feather->rule_scan_type; + struct _Maat_table_info_t ** pp_table=feather->p_table_info; + int table_cnt=feather->table_cnt; + + int i=0,j=0; + unsigned int sub_type=0; + int ret=0; MESA_htable_create_args_t hargs; memset(&hargs,0,sizeof(hargs)); @@ -883,16 +893,38 @@ struct _Maat_scanner_t* create_maat_scanner(unsigned int version,int scan_thread //For best performance test: //1.Do NOT set this option,rulescan return no hit detail as default; - //2.Set all STR rule to QUICK; + //2.Set necessary STR rule to QUICK; //rulescan_set_param(scanner->region,RULESCAN_DETAIL_RESULT,); - rulescan_set_param(scanner->region,RULESCAN_QUICK_SCAN,NULL,0); scanner->tomb_ref=tomb; scanner->region_rslt_buff=(scan_result_t*)malloc(sizeof(scan_result_t)*MAX_SCANNER_HIT_NUM*scan_thread_num); - for(i=0;idigest_update_q[i]=MESA_lqueue_create(0,0); - pthread_rwlock_init(&(scanner->digest_rwlock[i]),NULL); + switch(pp_table[i]->table_type) + { + case TABLE_TYPE_DIGEST: + scanner->digest_update_q[i]=MESA_lqueue_create(0,0); + pthread_rwlock_init(&(scanner->digest_rwlock[i]),NULL); + break; + case TABLE_TYPE_EXPR: + case TABLE_TYPE_EXPR_PLUS: + if(pp_table[i]->quick_expr_switch==1) + { + for(j=0;jdst_charset[j]!=CHARSET_NONE;j++) + { + sub_type=make_sub_type(pp_table[i]->table_id, pp_table[i]->dst_charset[j], pp_table[i]->do_charset_merge); + ret=rulescan_set_param(scanner->region,RULESCAN_QUICK_SCAN,&sub_type,sizeof(sub_type)); + assert(ret==1); + if(pp_table[i]->do_charset_merge==1) + { + break; + } + } + } + break; + default: + break; + } } return scanner; } @@ -1229,7 +1261,7 @@ int sync_region(MESA_htable_handle region_hash,int region_id,const char* table_n if(ret<0) { MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_module , - "region id %d of table %s is not unique.",region_id,table_name); + "region id %d of table_char2matrix %s is not unique.",region_id,table_name); return -1; } @@ -1240,7 +1272,7 @@ int sync_region(MESA_htable_handle region_hash,int region_id,const char* table_n if(ret==-1) { MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_module , - "region delete error,id %d in table %s does not exisit." + "region delete error,id %d in table_char2matrix %s does not exisit." ,region_id ,table_name); return -1; @@ -2429,7 +2461,7 @@ void maat_start_cb(unsigned int new_version,int update_type,void*u_para) int i=0,j=0; if(update_type==CM_UPDATE_TYPE_FULL) { - feather->update_tmp_scanner=create_maat_scanner(new_version,feather->scan_thread_num,feather->garbage_q,feather->rule_scan_type); + feather->update_tmp_scanner=create_maat_scanner(new_version,feather); MESA_handle_runtime_log(feather->logger,RLOG_LV_INFO,maat_module, "Full config version %u -> %u update start", feather->maat_version,new_version); diff --git a/src/entry/Maat_rule_internal.h b/src/entry/Maat_rule_internal.h index 81d2c78..2efa83e 100644 --- a/src/entry/Maat_rule_internal.h +++ b/src/entry/Maat_rule_internal.h @@ -228,6 +228,7 @@ struct _Maat_table_info_t int do_charset_merge; int cfg_num; int cross_cache_size; + int quick_expr_switch; union { int expr_rule_cnt; //expr_type=0,1,3 diff --git a/src/entry/zt_hash.h b/src/entry/zt_hash.h index 8acf0be..a8b45a7 100644 --- a/src/entry/zt_hash.h +++ b/src/entry/zt_hash.h @@ -10,7 +10,7 @@ struct zt_state_t }; }; -const unsigned char table[256][4] = +const unsigned char table_char2matrix[256][4] = { {76,28,128,81},{76,204,128,209},{204,128,209,81},{204,76,209,128},{238,209,196,115},{238,63,196,183},{63,209,183,115},{63,238,183,196},{230,196,193,123},{230,34,193,186},{34,196,186,123},{34,230,186,193},{0,183,175,76},{0,183,175,227},{183,183,227,76},{183,0,227,175},{228,193,192,121},{228,37,192,185},{37,193,185,121},{37,228,185,192},{18,186,164,75},{18,168,164,239},{168,186,239,75},{168,18,239,164},{15,175,169,67},{15,160,169,234},{160,175,234,67},{160,15,234,169},{151,227,247,108},{151,116,247,155},{116,227,155,108},{116,151,155,247},{228,192,193,121},{228,36,193,184},{36,192,184,121},{36,228,184,193},{22,185,167,74},{22,175,167,237},{175,185,237,74},{175,22,237,167},{30,164,162,71},{30,186,162,229},{186,164,229,71},{186,30,229,162},{136,239,250,107},{136,103,250,145},{103,239,145,107},{103,136,145,250},{12,169,169,64},{12,165,169,233},{165,169,233,64},{165,12,233,169},{138,234,251,105},{138,96,251,146},{96,234,146,105},{96,138,146,251},{159,247,243,100},{159,104,243,151},{104,247,151,100},{104,159,151,243},{71,155,133,95},{71,220,133,218},{220,155,218,95},{220,71,218,133},{230,193,196,123},{230,39,196,191},{39,193,191,123},{39,230,191,196},{20,184,160,73},{20,172,160,233},{172,184,233,73},{172,20,233,160},{25,167,167,69},{25,190,167,226},{190,167,226,69},{190,25,226,167},{141,237,253,104},{141,96,253,149},{96,237,149,104},{96,141,149,253},{30,162,164,71},{30,188,164,227},{188,162,227,71},{188,30,227,164},{144,229,240,109},{144,117,240,157},{117,229,157,109},{117,144,157,240},{130,250,251,97},{130,120,251,154},{120,250,154,97},{120,130,154,251},{84,145,137,88},{84,197,137,209},{197,145,209,88},{197,84,209,137},{15,169,175,67},{15,166,175,236},{166,169,236,67},{166,15,236,175},{143,233,253,106},{143,102,253,151},{102,233,151,106},{102,143,151,253},{130,251,250,97},{130,121,250,155},{121,251,155,97},{121,130,155,250},{80,146,138,89},{80,194,138,211},{194,146,211,89},{194,80,211,138},{159,243,247,100},{159,108,247,147},{108,243,147,100},{108,159,147,247},{87,151,137,91},{87,192,137,210},{192,151,210,91},{192,87,210,137},{72,133,133,80},{72,205,133,213},{205,133,213,80},{205,72,213,133},{246,218,207,117},{246,44,207,186},{44,218,186,117},{44,246,186,207},{238,196,209,115},{238,42,209,162},{42,196,162,115},{42,238,162,209},{24,191,191,68},{24,167,191,251},{167,191,251,68},{167,24,251,191},{20,160,184,73},{20,180,184,241},{180,160,241,73},{180,20,241,184},{134,233,236,99},{134,111,236,143},{111,233,143,99},{111,134,143,236},{22,167,185,74},{22,177,185,243},{177,167,243,74},{177,22,243,185},{156,226,227,103},{156,126,227,132},{126,226,132,103},{126,156,132,227},{143,253,233,106},{143,114,233,131},{114,253,131,106},{114,143,131,233},{95,149,147,87},{95,202,147,196},{202,149,196,87},{202,95,196,147},{18,164,186,75},{18,182,186,241},{182,164,241,75},{182,18,241,186},{156,227,226,103},{156,127,226,133},{127,227,133,103},{127,156,133,226},{144,240,229,109},{144,96,229,136},{96,240,136,109},{96,144,136,229},{74,157,155,82},{74,215,155,201},{215,157,201,82},{215,74,201,155},{138,251,234,105},{138,113,234,131},{113,251,131,105},{113,138,131,234},{72,154,154,81},{72,210,154,203},{210,154,203,81},{210,72,203,154},{87,137,151,91},{87,222,151,204},{222,137,204,91},{222,87,204,151},{231,209,213,122},{231,54,213,175},{54,209,175,122},{54,231,175,213},{0,175,183,76},{0,175,183,251},{175,175,251,76},{175,0,251,183},{134,236,233,99},{134,106,233,138},{106,236,138,99},{106,134,138,233},{141,253,237,104},{141,112,237,133},{112,253,133,104},{112,141,133,237},{89,151,151,85},{89,206,151,194},{206,151,194,85},{206,89,194,151},{136,250,239,107},{136,114,239,132},{114,250,132,107},{114,136,132,239},{74,155,157,82},{74,209,157,207},{209,155,207,82},{209,74,207,157},{80,138,146,89},{80,218,146,203},{218,138,203,89},{218,80,203,146},{226,211,210,121},{226,49,210,171},{49,211,171,121},{49,226,171,210},{151,247,227,108},{151,96,227,143},{96,247,143,108},{96,151,143,227},{95,147,149,87},{95,204,149,194},{204,147,194,87},{204,95,194,149},{84,137,145,88},{84,221,145,201},{221,137,201,88},{221,84,201,145},{226,210,211,121},{226,48,211,170},{48,210,170,121},{48,226,170,211},{71,133,155,95},{71,194,155,196},{194,133,196,95},{194,71,196,155},{231,213,209,122},{231,50,209,171},{50,213,171,122},{50,231,171,209},{246,207,218,117},{246,57,218,175},{57,207,175,117},{57,246,175,218},{28,186,186,69},{28,166,186,255},{166,186,255,69},{166,28,255,186} }; @@ -124,7 +124,7 @@ static inline void zt_hash_arymul(struct zt_state_t * a, struct zt_state_t* b) } /* - ** this function is used to create the table[4][256] + ** this function is used to create the table_char2matrix[4][256] */ /*void convert(int number, unsigned char * ret) { @@ -172,26 +172,26 @@ static inline void zt_hash_arymul(struct zt_state_t * a, struct zt_state_t* b) /* - ** this function is used to create table[4][256] + ** this function is used to create table_char2matrix[4][256] */ /*void zt_hash_create_table() { unsigned char ret[4]={0}; int i = 0; FILE * fp; - fp = fopen("/home/lixiang/zt_hash/table.txt","a"); + fp = fopen("/home/lixiang/zt_hash/table_char2matrix.txt","a"); //galois_create_mult_tables(8); //it should not be a comment for(i = 0; i < 256; i++) { convert(i, ret); - table[i].matrix[0] = ret[0]; - table[i].matrix[1] = ret[1]; - table[i].matrix[2] = ret[2]; - table[i].matrix[3] = ret[3]; - fprintf(fp, "{%d,%d,%d,%d},", table[i].matrix[0], - table[i].matrix[1], - table[i].matrix[2], - table[i].matrix[3]); + table_char2matrix[i].matrix[0] = ret[0]; + table_char2matrix[i].matrix[1] = ret[1]; + table_char2matrix[i].matrix[2] = ret[2]; + table_char2matrix[i].matrix[3] = ret[3]; + fprintf(fp, "{%d,%d,%d,%d},", table_char2matrix[i].matrix[0], + table_char2matrix[i].matrix[1], + table_char2matrix[i].matrix[2], + table_char2matrix[i].matrix[3]); } } @@ -207,7 +207,7 @@ void zt_hash_destroy_table() inline void zt_hash(struct zt_state_t* array, unsigned char c) { - zt_hash_arymul(array, (struct zt_state_t *)(table[c])); + zt_hash_arymul(array, (struct zt_state_t *)(table_char2matrix[c])); } unsigned char ZT_INIT_VAL[4]={1,0,0,1};