支持单独设置expr和expr_plus类的表的是否进行快速扫描（IOScan）
This commit is contained in:
zhengchao
@@ -26,7 +26,7 @@
|
||||
#include "mesa_fuzzy.h"
|
||||
#include "great_index_engine.h"
|
||||
|
||||
int MAAT_FRAME_VERSION_1_8_20160607_PERF_TEST=1;
|
||||
int MAAT_FRAME_VERSION_1_8_20160617_PERF_TEST=1;
|
||||
const char *maat_module="MAAT Frame";
|
||||
|
||||
const char* CHARSET_STRING[]={"NONE","gbk","big5","unicode","utf8","bin",
|
||||
@@ -387,6 +387,7 @@ _Maat_table_info_t* create_table_info(int max_thread_num)
|
||||
p->stream_num=aligment_int64_array_alloc(max_thread_num);
|
||||
p->hit_cnt=aligment_int64_array_alloc(max_thread_num);
|
||||
p->cross_cache_size=0;
|
||||
p->quick_expr_switch=0;
|
||||
return p;
|
||||
}
|
||||
void destroy_table_info(struct _Maat_table_info_t*p)
|
||||
@@ -458,13 +459,14 @@ int read_table_info(struct _Maat_table_info_t** p_table_info,int num,const char*
|
||||
}
|
||||
p=create_table_info(max_thread_num);
|
||||
|
||||
sscanf(line,"%hu\t%s\t%s\t%s\t%s\t%s\t%d",&(p->table_id)
|
||||
sscanf(line,"%hu\t%s\t%s\t%s\t%s\t%s\t%d\t%d",&(p->table_id)
|
||||
,p->table_name
|
||||
,table_type
|
||||
,src_charset
|
||||
,dst_charset
|
||||
,merge
|
||||
,&(p->cross_cache_size));
|
||||
,&(p->cross_cache_size)
|
||||
,&(p->quick_expr_switch));
|
||||
ret[0]=map_str2int(string2int_map,strlwr(table_type),(int*)&(p->table_type));
|
||||
ret[1]=map_str2int(string2int_map,strlwr(src_charset),(int*)&(p->src_charset));
|
||||
ret[2]=map_str2int(string2int_map,strlwr(merge),&(p->do_charset_merge));
|
||||
@@ -837,9 +839,17 @@ void destroy_digest_rule(GIE_digest_t*rule)
|
||||
rule=NULL;
|
||||
return;
|
||||
}
|
||||
struct _Maat_scanner_t* create_maat_scanner(unsigned int version,int scan_thread_num,MESA_lqueue_head tomb,int rs_scan_type)
|
||||
struct _Maat_scanner_t* create_maat_scanner(unsigned int version,_Maat_feather_t *feather)
|
||||
{
|
||||
int i=0;
|
||||
int scan_thread_num=feather->scan_thread_num;
|
||||
MESA_lqueue_head tomb=feather->garbage_q;
|
||||
// int rs_scan_type=feather->rule_scan_type;
|
||||
struct _Maat_table_info_t ** pp_table=feather->p_table_info;
|
||||
int table_cnt=feather->table_cnt;
|
||||
|
||||
int i=0,j=0;
|
||||
unsigned int sub_type=0;
|
||||
int ret=0;
|
||||
|
||||
MESA_htable_create_args_t hargs;
|
||||
memset(&hargs,0,sizeof(hargs));
|
||||
@@ -883,16 +893,38 @@ struct _Maat_scanner_t* create_maat_scanner(unsigned int version,int scan_thread
|
||||
|
||||
//For best performance test:
|
||||
//1.Do NOT set this option,rulescan return no hit detail as default;
|
||||
//2.Set all STR rule to QUICK;
|
||||
//2.Set necessary STR rule to QUICK;
|
||||
//rulescan_set_param(scanner->region,RULESCAN_DETAIL_RESULT,);
|
||||
rulescan_set_param(scanner->region,RULESCAN_QUICK_SCAN,NULL,0);
|
||||
scanner->tomb_ref=tomb;
|
||||
scanner->region_rslt_buff=(scan_result_t*)malloc(sizeof(scan_result_t)*MAX_SCANNER_HIT_NUM*scan_thread_num);
|
||||
|
||||
for(i=0;i<MAX_TABLE_NUM;i++)
|
||||
for(i=0;i<table_cnt;i++)
|
||||
{
|
||||
switch(pp_table[i]->table_type)
|
||||
{
|
||||
case TABLE_TYPE_DIGEST:
|
||||
scanner->digest_update_q[i]=MESA_lqueue_create(0,0);
|
||||
pthread_rwlock_init(&(scanner->digest_rwlock[i]),NULL);
|
||||
break;
|
||||
case TABLE_TYPE_EXPR:
|
||||
case TABLE_TYPE_EXPR_PLUS:
|
||||
if(pp_table[i]->quick_expr_switch==1)
|
||||
{
|
||||
for(j=0;j<MAX_CHARSET_NUM&& pp_table[i]->dst_charset[j]!=CHARSET_NONE;j++)
|
||||
{
|
||||
sub_type=make_sub_type(pp_table[i]->table_id, pp_table[i]->dst_charset[j], pp_table[i]->do_charset_merge);
|
||||
ret=rulescan_set_param(scanner->region,RULESCAN_QUICK_SCAN,&sub_type,sizeof(sub_type));
|
||||
assert(ret==1);
|
||||
if(pp_table[i]->do_charset_merge==1)
|
||||
{
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
break;
|
||||
default:
|
||||
break;
|
||||
}
|
||||
}
|
||||
return scanner;
|
||||
}
|
||||
@@ -1229,7 +1261,7 @@ int sync_region(MESA_htable_handle region_hash,int region_id,const char* table_n
|
||||
if(ret<0)
|
||||
{
|
||||
MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_module ,
|
||||
"region id %d of table %s is not unique.",region_id,table_name);
|
||||
"region id %d of table_char2matrix %s is not unique.",region_id,table_name);
|
||||
return -1;
|
||||
}
|
||||
|
||||
@@ -1240,7 +1272,7 @@ int sync_region(MESA_htable_handle region_hash,int region_id,const char* table_n
|
||||
if(ret==-1)
|
||||
{
|
||||
MESA_handle_runtime_log(logger,RLOG_LV_FATAL,maat_module ,
|
||||
"region delete error,id %d in table %s does not exisit."
|
||||
"region delete error,id %d in table_char2matrix %s does not exisit."
|
||||
,region_id
|
||||
,table_name);
|
||||
return -1;
|
||||
@@ -2429,7 +2461,7 @@ void maat_start_cb(unsigned int new_version,int update_type,void*u_para)
|
||||
int i=0,j=0;
|
||||
if(update_type==CM_UPDATE_TYPE_FULL)
|
||||
{
|
||||
feather->update_tmp_scanner=create_maat_scanner(new_version,feather->scan_thread_num,feather->garbage_q,feather->rule_scan_type);
|
||||
feather->update_tmp_scanner=create_maat_scanner(new_version,feather);
|
||||
MESA_handle_runtime_log(feather->logger,RLOG_LV_INFO,maat_module,
|
||||
"Full config version %u -> %u update start",
|
||||
feather->maat_version,new_version);
|
||||
|
||||
@@ -228,6 +228,7 @@ struct _Maat_table_info_t
|
||||
int do_charset_merge;
|
||||
int cfg_num;
|
||||
int cross_cache_size;
|
||||
int quick_expr_switch;
|
||||
union
|
||||
{
|
||||
int expr_rule_cnt; //expr_type=0,1,3
|
||||
|
||||
@@ -10,7 +10,7 @@ struct zt_state_t
|
||||
};
|
||||
};
|
||||
|
||||
const unsigned char table[256][4] =
|
||||
const unsigned char table_char2matrix[256][4] =
|
||||
{
|
||||
{76,28,128,81},{76,204,128,209},{204,128,209,81},{204,76,209,128},{238,209,196,115},{238,63,196,183},{63,209,183,115},{63,238,183,196},{230,196,193,123},{230,34,193,186},{34,196,186,123},{34,230,186,193},{0,183,175,76},{0,183,175,227},{183,183,227,76},{183,0,227,175},{228,193,192,121},{228,37,192,185},{37,193,185,121},{37,228,185,192},{18,186,164,75},{18,168,164,239},{168,186,239,75},{168,18,239,164},{15,175,169,67},{15,160,169,234},{160,175,234,67},{160,15,234,169},{151,227,247,108},{151,116,247,155},{116,227,155,108},{116,151,155,247},{228,192,193,121},{228,36,193,184},{36,192,184,121},{36,228,184,193},{22,185,167,74},{22,175,167,237},{175,185,237,74},{175,22,237,167},{30,164,162,71},{30,186,162,229},{186,164,229,71},{186,30,229,162},{136,239,250,107},{136,103,250,145},{103,239,145,107},{103,136,145,250},{12,169,169,64},{12,165,169,233},{165,169,233,64},{165,12,233,169},{138,234,251,105},{138,96,251,146},{96,234,146,105},{96,138,146,251},{159,247,243,100},{159,104,243,151},{104,247,151,100},{104,159,151,243},{71,155,133,95},{71,220,133,218},{220,155,218,95},{220,71,218,133},{230,193,196,123},{230,39,196,191},{39,193,191,123},{39,230,191,196},{20,184,160,73},{20,172,160,233},{172,184,233,73},{172,20,233,160},{25,167,167,69},{25,190,167,226},{190,167,226,69},{190,25,226,167},{141,237,253,104},{141,96,253,149},{96,237,149,104},{96,141,149,253},{30,162,164,71},{30,188,164,227},{188,162,227,71},{188,30,227,164},{144,229,240,109},{144,117,240,157},{117,229,157,109},{117,144,157,240},{130,250,251,97},{130,120,251,154},{120,250,154,97},{120,130,154,251},{84,145,137,88},{84,197,137,209},{197,145,209,88},{197,84,209,137},{15,169,175,67},{15,166,175,236},{166,169,236,67},{166,15,236,175},{143,233,253,106},{143,102,253,151},{102,233,151,106},{102,143,151,253},{130,251,250,97},{130,121,250,155},{121,251,155,97},{121,130,155,250},{80,146,138,89},{80,194,138,211},{194,146,211,89},{194,80,211,138},{159,243,247,100},{159,108,247,147},{108,243,147,100},{108,159,147,247},{87,151,137,91},{87,192,137,210},{192,151,210,91},{192,87,210,137},{72,133,133,80},{72,205,133,213},{205,133,213,80},{205,72,213,133},{246,218,207,117},{246,44,207,186},{44,218,186,117},{44,246,186,207},{238,196,209,115},{238,42,209,162},{42,196,162,115},{42,238,162,209},{24,191,191,68},{24,167,191,251},{167,191,251,68},{167,24,251,191},{20,160,184,73},{20,180,184,241},{180,160,241,73},{180,20,241,184},{134,233,236,99},{134,111,236,143},{111,233,143,99},{111,134,143,236},{22,167,185,74},{22,177,185,243},{177,167,243,74},{177,22,243,185},{156,226,227,103},{156,126,227,132},{126,226,132,103},{126,156,132,227},{143,253,233,106},{143,114,233,131},{114,253,131,106},{114,143,131,233},{95,149,147,87},{95,202,147,196},{202,149,196,87},{202,95,196,147},{18,164,186,75},{18,182,186,241},{182,164,241,75},{182,18,241,186},{156,227,226,103},{156,127,226,133},{127,227,133,103},{127,156,133,226},{144,240,229,109},{144,96,229,136},{96,240,136,109},{96,144,136,229},{74,157,155,82},{74,215,155,201},{215,157,201,82},{215,74,201,155},{138,251,234,105},{138,113,234,131},{113,251,131,105},{113,138,131,234},{72,154,154,81},{72,210,154,203},{210,154,203,81},{210,72,203,154},{87,137,151,91},{87,222,151,204},{222,137,204,91},{222,87,204,151},{231,209,213,122},{231,54,213,175},{54,209,175,122},{54,231,175,213},{0,175,183,76},{0,175,183,251},{175,175,251,76},{175,0,251,183},{134,236,233,99},{134,106,233,138},{106,236,138,99},{106,134,138,233},{141,253,237,104},{141,112,237,133},{112,253,133,104},{112,141,133,237},{89,151,151,85},{89,206,151,194},{206,151,194,85},{206,89,194,151},{136,250,239,107},{136,114,239,132},{114,250,132,107},{114,136,132,239},{74,155,157,82},{74,209,157,207},{209,155,207,82},{209,74,207,157},{80,138,146,89},{80,218,146,203},{218,138,203,89},{218,80,203,146},{226,211,210,121},{226,49,210,171},{49,211,171,121},{49,226,171,210},{151,247,227,108},{151,96,227,143},{96,247,143,108},{96,151,143,227},{95,147,149,87},{95,204,149,194},{204,147,194,87},{204,95,194,149},{84,137,145,88},{84,221,145,201},{221,137,201,88},{221,84,201,145},{226,210,211,121},{226,48,211,170},{48,210,170,121},{48,226,170,211},{71,133,155,95},{71,194,155,196},{194,133,196,95},{194,71,196,155},{231,213,209,122},{231,50,209,171},{50,213,171,122},{50,231,171,209},{246,207,218,117},{246,57,218,175},{57,207,175,117},{57,246,175,218},{28,186,186,69},{28,166,186,255},{166,186,255,69},{166,28,255,186}
|
||||
};
|
||||
@@ -124,7 +124,7 @@ static inline void zt_hash_arymul(struct zt_state_t * a, struct zt_state_t* b)
|
||||
}
|
||||
|
||||
/*
|
||||
** this function is used to create the table[4][256]
|
||||
** this function is used to create the table_char2matrix[4][256]
|
||||
*/
|
||||
/*void convert(int number, unsigned char * ret)
|
||||
{
|
||||
@@ -172,26 +172,26 @@ static inline void zt_hash_arymul(struct zt_state_t * a, struct zt_state_t* b)
|
||||
|
||||
|
||||
/*
|
||||
** this function is used to create table[4][256]
|
||||
** this function is used to create table_char2matrix[4][256]
|
||||
*/
|
||||
/*void zt_hash_create_table()
|
||||
{
|
||||
unsigned char ret[4]={0};
|
||||
int i = 0;
|
||||
FILE * fp;
|
||||
fp = fopen("/home/lixiang/zt_hash/table.txt","a");
|
||||
fp = fopen("/home/lixiang/zt_hash/table_char2matrix.txt","a");
|
||||
//galois_create_mult_tables(8); //it should not be a comment
|
||||
for(i = 0; i < 256; i++)
|
||||
{
|
||||
convert(i, ret);
|
||||
table[i].matrix[0] = ret[0];
|
||||
table[i].matrix[1] = ret[1];
|
||||
table[i].matrix[2] = ret[2];
|
||||
table[i].matrix[3] = ret[3];
|
||||
fprintf(fp, "{%d,%d,%d,%d},", table[i].matrix[0],
|
||||
table[i].matrix[1],
|
||||
table[i].matrix[2],
|
||||
table[i].matrix[3]);
|
||||
table_char2matrix[i].matrix[0] = ret[0];
|
||||
table_char2matrix[i].matrix[1] = ret[1];
|
||||
table_char2matrix[i].matrix[2] = ret[2];
|
||||
table_char2matrix[i].matrix[3] = ret[3];
|
||||
fprintf(fp, "{%d,%d,%d,%d},", table_char2matrix[i].matrix[0],
|
||||
table_char2matrix[i].matrix[1],
|
||||
table_char2matrix[i].matrix[2],
|
||||
table_char2matrix[i].matrix[3]);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -207,7 +207,7 @@ void zt_hash_destroy_table()
|
||||
|
||||
inline void zt_hash(struct zt_state_t* array, unsigned char c)
|
||||
{
|
||||
zt_hash_arymul(array, (struct zt_state_t *)(table[c]));
|
||||
zt_hash_arymul(array, (struct zt_state_t *)(table_char2matrix[c]));
|
||||
}
|
||||
|
||||
unsigned char ZT_INIT_VAL[4]={1,0,0,1};
|
||||
|
||||
Reference in New Issue
Block a user