gfwleak/tango-maat
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

重构table runtime工作量较大，准备放弃。

Browse Source
This commit is contained in:
zhengchao
2021-08-05 00:22:00 +08:00
parent 7e3b746eaa
commit e356d74bc0
6 changed files with 61 additions and 37 deletions
Download Patch File Download Diff File Expand all files Collapse all files

34
src/entry/Maat_api.cpp
View File

@@ -1282,7 +1282,8 @@ int Maat_ip_plugin_get_EX_data(Maat_feather_t feather, int table_id, const struc
table_schema=Maat_table_get_scan_by_id(_feather->table_mgr, table_id, SCAN_TYPE_IP_PLUGIN, NULL);
table_rt=Maat_table_runtime_get(_feather->scanner->table_rt_mgr, table_id);
if(table_rt->table_type!=TABLE_TYPE_IP_PLUGIN)
enum MAAT_TABLE_TYPE table_type=Maat_table_runtime_get_type(table_rt);
if(table_type!=TABLE_TYPE_IP_PLUGIN)
{
return -1;
}
@@ -1328,7 +1329,8 @@ int Maat_fqdn_plugin_get_EX_data(Maat_feather_t feather, int table_id, const cha
}
table_schema=Maat_table_get_scan_by_id(_feather->table_mgr, table_id, SCAN_TYPE_FQDN_PLUGIN, NULL);
table_rt=Maat_table_runtime_get(_feather->scanner->table_rt_mgr, table_id);
if(table_rt->table_type!=TABLE_TYPE_FQDN_PLUGIN)
enum MAAT_TABLE_TYPE table_type=Maat_table_runtime_get_type(table_rt);
if(table_type!=TABLE_TYPE_FQDN_PLUGIN)
{
return -1;
}
@@ -1338,11 +1340,11 @@ int Maat_fqdn_plugin_get_EX_data(Maat_feather_t feather, int table_id, const cha
if(_feather->perf_on==1)
{
clock_gettime(CLOCK_MONOTONIC,&end);
maat_stat_table(table_rt, 0, &start, &end, 0);
Maat_table_runtime_perf_stat(table_rt, 0, &start, &end, 0);
}
else
{
maat_stat_table(table_rt, 0, NULL, NULL, 0);
Maat_table_runtime_perf_stat(table_rt, 0, NULL, NULL, 0);
}
return n_get;
@@ -1389,7 +1391,9 @@ int Maat_full_scan_string_detail(Maat_feather_t feather,int table_id
return -1;
}
expr_desc=&(p_table->expr);
if(p_table->table_type==TABLE_TYPE_EXPR_PLUS&&(_mid==NULL||_mid->is_set_district!=1))
enum MAAT_TABLE_TYPE table_type=Maat_table_runtime_get_type(p_table);
if(table_type==TABLE_TYPE_EXPR_PLUS&&(_mid==NULL||_mid->is_set_district!=1))
{
_feather->scan_err_cnt++;
return -1;
@@ -1473,11 +1477,11 @@ int Maat_full_scan_string_detail(Maat_feather_t feather,int table_id
if(_feather->perf_on==1)
{
clock_gettime(CLOCK_MONOTONIC,&end);
maat_stat_table(table_rt, data_len, &start, &end, thread_num);
Maat_table_runtime_perf_stat(table_rt, data_len, &start, &end, thread_num);
}
else
{
maat_stat_table(table_rt, data_len, NULL, NULL, thread_num);
Maat_table_runtime_perf_stat(table_rt, data_len, NULL, NULL, thread_num);
}
DEC_SCANNER_REF(my_scanner, thread_num);
if(compile_ret==0&&hit_region_cnt>0)
@@ -1538,7 +1542,8 @@ int Maat_scan_intval(Maat_feather_t feather,int table_id
return 0;
}
if(table_rt->table_type==TABLE_TYPE_INTERVAL_PLUS&&(_mid==NULL||_mid->is_set_district!=1))
enum MAAT_TABLE_TYPE table_type=Maat_table_runtime_get_type(p_table);
if(table_type==TABLE_TYPE_INTERVAL_PLUS&&(_mid==NULL||_mid->is_set_district!=1))
{
_feather->scan_err_cnt++;
return -1;
@@ -1588,11 +1593,11 @@ int Maat_scan_intval(Maat_feather_t feather,int table_id
if(_feather->perf_on==1)
{
clock_gettime(CLOCK_MONOTONIC,&end);
maat_stat_table(table_rt, 0, &start, &end, thread_num);
Maat_table_runtime_perf_stat(table_rt, 0, &start, &end, thread_num);
}
else
{
maat_stat_table(table_rt, 0, NULL, NULL, thread_num);
Maat_table_runtime_perf_stat(table_rt, 0, NULL, NULL, thread_num);
}
DEC_SCANNER_REF(my_scanner,thread_num);
@@ -1629,6 +1634,12 @@ int Maat_similar_scan_string(Maat_feather_t feather,int table_id
_feather->scan_err_cnt++;
return -1;
}
enum MAAT_TABLE_TYPE table_type=Maat_table_runtime_get_type(p_table);
if(table_type!=TABLE_TYPE_SIMILARITY)
{
return -1;
}
my_scanner=_feather->scanner;
if(my_scanner==NULL)
{
@@ -1640,7 +1651,8 @@ int Maat_similar_scan_string(Maat_feather_t feather,int table_id
if(table_rt->origin_rule_num==0)
{
return 0;
}
}
GIE_handle_t* gie_handle=table_rt->similar.gie_handle;
INC_SCANNER_REF(my_scanner,thread_num);
alignment_int64_array_add(_feather->thread_call_cnt, thread_num, 1);

21
src/entry/Maat_rule.cpp
View File

@@ -2328,6 +2328,7 @@ void do_scanner_update(struct Maat_scanner* scanner, int scan_thread_num, void*
int i=0, ret=0;
struct ip_matcher* old_ip_matcher=NULL;
struct FQDN_engine* old_fqdn_engine=NULL;
enum MAAT_TABLE_TYPE table_type=0;
ret=Maat_hierarchy_rebuild(scanner->hier);
if(ret!=0)
@@ -2357,7 +2358,8 @@ void do_scanner_update(struct Maat_scanner* scanner, int scan_thread_num, void*
{
continue;
}
switch(table_rt->table_type)
table_type=Maat_table_runtime_get_type(table_rt);
switch(table_type)
{
case TABLE_TYPE_DIGEST:
case TABLE_TYPE_SIMILARITY:
@@ -2371,14 +2373,21 @@ void do_scanner_update(struct Maat_scanner* scanner, int scan_thread_num, void*
case TABLE_TYPE_PLUGIN:
break;
case TABLE_TYPE_IP_PLUGIN:
ret=Maat_table_runtime_ip_plugin_build_new_ip_matcher(table_rt);
break;
case TABLE_TYPE_FQDN_PLUGIN:
ret=Maat_table_runtime_fqdn_plugin_build_new_fqdn_engine(table_rt);
ret=Maat_table_runtime_ip_plugin_rebuild_ip_matcher(table_rt);
if(ret<0)
{
MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module,
"FQDN plugin table %s build failed.", Maat_table_runtime_get_name);
"IP plugin table_id %d build failed.", i);
}
break;
case TABLE_TYPE_FQDN_PLUGIN:
ret=Maat_table_runtime_fqdn_plugin_rebuild_fqdn_engine(table_rt);
if(ret<0)
{
MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module,
"FQDN plugin table_id %d build failed.", i);
}
break;

11
src/entry/Maat_stat.cpp
View File

@@ -152,16 +152,7 @@ void maat_stat_init(struct _Maat_feather_t* feather)
FS_start(feather->stat_handle);
return;
}
void maat_stat_table(struct Maat_table_runtime* p,int scan_len,struct timespec* start, struct timespec* end,int thread_num)
{
alignment_int64_array_add(p->scan_cnt,thread_num,1);
alignment_int64_array_add(p->input_bytes,thread_num,scan_len);
if(start!=NULL&&end!=NULL)
{
alignment_int64_array_add(p->scan_cpu_time,thread_num,(end->tv_sec-start->tv_sec)*1000000000+end->tv_nsec-start->tv_nsec);
}
return;
}
void maat_stat_output(struct _Maat_feather_t* feather)
{
long value=0;

21
src/entry/Maat_table_runtime.cpp
View File

@@ -343,6 +343,21 @@ struct Maat_table_runtime* Maat_table_runtime_get(struct Maat_table_runtime_mana
assert(table_id<(int)table_rt_mgr->n_table_rt);
return table_rt_mgr->table_rt[table_id];
}
enum MAAT_TABLE_TYPE Maat_table_runtime_get_type(struct Maat_table_runtime* table_rt)
{
return table_rt->table_type;
}
void Maat_table_runtime_perf_stat(struct Maat_table_runtime* p, int scan_len, struct timespec* start, struct timespec* end,int thread_num)
{
alignment_int64_array_add(p->scan_cnt,thread_num,1);
alignment_int64_array_add(p->input_bytes,thread_num,scan_len);
if(start!=NULL&&end!=NULL)
{
alignment_int64_array_add(p->scan_cpu_time,thread_num,(end->tv_sec-start->tv_sec)*1000000000+end->tv_nsec-start->tv_nsec);
}
return;
}
size_t Maat_table_runtime_plugin_cached_row_count(struct Maat_table_runtime* table_rt)
{
struct plugin_runtime* plugin_rt=&(table_rt->plugin);
@@ -578,7 +593,7 @@ int Maat_table_runtime_fqdn_plugin_commit_ex_schema(struct Maat_table_runtime* t
Maat_table_runtime_fqdn_plugin_new_row(table_rt, table_schema, row, logger);
}
EX_data_rt_clear_row_cache(fqdn_plugin_rt->ex_data_rt);
Maat_table_runtime_fqdn_plugin_build_new_fqdn_engine(table_rt);
Maat_table_runtime_fqdn_plugin_rebuild_fqdn_engine(table_rt);
Maat_table_runtime_apply_new_fqdn_engine(table_rt);//returned NULL.
return 0;
}
@@ -668,7 +683,7 @@ int Maat_table_runtime_digest_batch_udpate(struct Maat_table_runtime* table_rt)
return q_cnt;
}
int Maat_table_runtime_ip_plugin_build_new_ip_matcher(struct Maat_table_runtime* table_rt)
int Maat_table_runtime_ip_plugin_rebuild_ip_matcher(struct Maat_table_runtime* table_rt)
{
struct ip_matcher* new_ip_matcher=NULL, *old_ip_matcher=NULL;
size_t rule_cnt=0;
@@ -771,7 +786,7 @@ int Maat_table_runtime_ip_plugin_commit_ex_schema(struct Maat_table_runtime* tab
Maat_table_runtime_ip_plugin_new_row(table_rt, table_schema, row, logger);
}
EX_data_rt_clear_row_cache(ip_plugin_rt->ex_data_rt);
Maat_table_runtime_ip_plugin_build_new_ip_matcher(table_rt);
Maat_table_runtime_ip_plugin_rebuild_ip_matcher(table_rt);
return 0;
}

1
src/inc_internal/Maat_rule_internal.h
View File

@@ -282,7 +282,6 @@ struct _Maat_feather_t
struct Maat_table_manager* table_mgr;
int DEFERRED_LOAD_ON;
int REDIS_MODE_ON;
enum data_source input_mode;
union
{

10
src/inc_internal/Maat_table_runtime.h
View File

@@ -8,6 +8,8 @@ struct Maat_table_runtime;
struct Maat_table_runtime_manager* Maat_table_runtime_manager_create(struct Maat_table_manager* table_manager, int max_thread_num, struct Maat_garbage_bin* bin);
void Maat_table_rt_manager_destroy(struct Maat_table_runtime_manager* table_rt_mgr);
struct Maat_table_runtime* Maat_table_runtime_get(struct Maat_table_runtime_manager* table_rt_mgr, int table_id);
void Maat_table_runtime_perf_stat(struct Maat_table_runtime* p, int scan_len, struct timespec* start, struct timespec* end,int thread_num);
enum MAAT_TABLE_TYPE Maat_table_runtime_get_type(struct Maat_table_runtime* table_rt);
size_t Maat_table_runtime_plugin_cached_row_count(struct Maat_table_runtime* table_rt);
const char* Maat_table_runtime_plugin_get_cached_row(struct Maat_table_runtime* table_rt, size_t Nth_row);
@@ -24,14 +26,10 @@ void Maat_table_runtime_ip_plugin_new_row(struct Maat_table_runtime* table_rt, s
int Maat_table_runtime_ip_plugin_commit_ex_schema(struct Maat_table_runtime* table_rt, struct Maat_table_schema* table_schema, void* logger);
int Maat_table_runtime_ip_plugin_get_N_ex_data(struct Maat_table_runtime* table_rt, struct Maat_table_schema* table_schema, const struct ip_data* ip, MAAT_PLUGIN_EX_DATA* ex_data_array, size_t size);
int Maat_table_runtime_ip_plugin_build_new_ip_matcher(struct Maat_table_runtime* table_rt);
struct ip_matcher* Maat_table_runtime_apply_new_ip_matcher(struct Maat_table_runtime* table_rt);
int Maat_table_runtime_fqdn_plugin_build_new_fqdn_engine(struct Maat_table_runtime* table_rt);
int Maat_table_runtime_ip_plugin_rebuild_ip_matcher(struct Maat_table_runtime* table_rt);
void Maat_table_runtime_fqdn_plugin_new_row(struct Maat_table_runtime* table_rt, struct Maat_table_schema* table_schema, const char* row, void *logger);
int Maat_table_runtime_fqdn_plugin_build_new_fqdn_engine(struct Maat_table_runtime* table_rt);
struct FQDN_engine* Maat_table_runtime_apply_new_fqdn_engine(struct Maat_table_runtime* table_rt);
int Maat_table_runtime_fqdn_plugin_rebuild_fqdn_engine(struct Maat_table_runtime* table_rt);
int Maat_table_runtime_fqdn_plugin_commit_ex_schema(struct Maat_table_runtime* table_rt, struct Maat_table_schema* table_schema, void* logger);
int Maat_table_runtime_fqdn_plugin_get_N_ex_data(struct Maat_table_runtime* table_rt, struct Maat_table_schema* table_schema, const char* query_fqdn, MAAT_PLUGIN_EX_DATA* ex_data_array, size_t size);