From c7e9b293098470f45ac52365d5b2a764876d0a1b Mon Sep 17 00:00:00 2001 From: zhengchao Date: Thu, 5 Aug 2021 00:39:32 +0800 Subject: [PATCH] =?UTF-8?q?IP=20Plugin=E5=92=8CFQDN=20Plugin=E5=86=85?= =?UTF-8?q?=E9=83=A8=E5=AE=9E=E7=8E=B0=E5=9E=83=E5=9C=BE=E5=9B=9E=E6=94=B6?= =?UTF-8?q?=EF=BC=8C=E4=B8=8D=E5=86=8D=E7=94=B1=E5=A4=96=E9=83=A8=E8=B0=83?= =?UTF-8?q?=E7=94=A8=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/entry/Maat_api.cpp | 36 ++++++------ src/entry/Maat_rule.cpp | 4 +- src/entry/Maat_table_runtime.cpp | 82 +-------------------------- src/inc_internal/Maat_rule_internal.h | 1 - src/inc_internal/Maat_table_runtime.h | 67 +++++++++++++++++++++- 5 files changed, 86 insertions(+), 104 deletions(-) diff --git a/src/entry/Maat_api.cpp b/src/entry/Maat_api.cpp index b214762..ecdf880 100644 --- a/src/entry/Maat_api.cpp +++ b/src/entry/Maat_api.cpp @@ -1195,11 +1195,11 @@ MAAT_PLUGIN_EX_DATA Maat_plugin_get_EX_data(Maat_feather_t feather, int table_id if(_feather->perf_on==1) { clock_gettime(CLOCK_MONOTONIC,&end); - maat_stat_table(table_rt, 0, &start, &end, 0); + Maat_table_runtime_perf_stat(table_rt, 0, &start, &end, 0); } else { - maat_stat_table(table_rt, 0, NULL, NULL, 0); + Maat_table_runtime_perf_stat(table_rt, 0, NULL, NULL, 0); } return exdata; @@ -1301,11 +1301,11 @@ int Maat_ip_plugin_get_EX_data(Maat_feather_t feather, int table_id, const struc if(_feather->perf_on==1) { clock_gettime(CLOCK_MONOTONIC,&end); - maat_stat_table(table_rt, 0, &start, &end, 0); + Maat_table_runtime_perf_stat(table_rt, 0, &start, &end, 0); } else { - maat_stat_table(table_rt, 0, NULL, NULL, 0); + Maat_table_runtime_perf_stat(table_rt, 0, NULL, NULL, 0); } return n_get; @@ -1391,9 +1391,8 @@ int Maat_full_scan_string_detail(Maat_feather_t feather,int table_id return -1; } expr_desc=&(p_table->expr); - - enum MAAT_TABLE_TYPE table_type=Maat_table_runtime_get_type(p_table); - if(table_type==TABLE_TYPE_EXPR_PLUS&&(_mid==NULL||_mid->is_set_district!=1)) + + if(p_table->table_type==TABLE_TYPE_EXPR_PLUS&&(_mid==NULL||_mid->is_set_district!=1)) { _feather->scan_err_cnt++; return -1; @@ -1541,9 +1540,7 @@ int Maat_scan_intval(Maat_feather_t feather,int table_id { return 0; } - - enum MAAT_TABLE_TYPE table_type=Maat_table_runtime_get_type(p_table); - if(table_type==TABLE_TYPE_INTERVAL_PLUS&&(_mid==NULL||_mid->is_set_district!=1)) + if(p_table->table_type==TABLE_TYPE_INTERVAL_PLUS&&(_mid==NULL||_mid->is_set_district!=1)) { _feather->scan_err_cnt++; return -1; @@ -1634,8 +1631,7 @@ int Maat_similar_scan_string(Maat_feather_t feather,int table_id _feather->scan_err_cnt++; return -1; } - enum MAAT_TABLE_TYPE table_type=Maat_table_runtime_get_type(p_table); - if(table_type!=TABLE_TYPE_SIMILARITY) + if(p_table->table_type!=TABLE_TYPE_SIMILARITY) { return -1; } @@ -1686,11 +1682,11 @@ int Maat_similar_scan_string(Maat_feather_t feather,int table_id if(_feather->perf_on==1) { clock_gettime(CLOCK_MONOTONIC,&end); - maat_stat_table(table_rt,0,&start, &end,thread_num); + Maat_table_runtime_perf_stat(table_rt,0,&start, &end,thread_num); } else { - maat_stat_table(table_rt,0,NULL, NULL,thread_num); + Maat_table_runtime_perf_stat(table_rt,0,NULL, NULL,thread_num); } if(compile_ret==0&&hit_region_cnt>0) { @@ -1919,11 +1915,11 @@ int Maat_scan_proto_addr(Maat_feather_t feather,int table_id if(_feather->perf_on==1) { clock_gettime(CLOCK_MONOTONIC,&end); - maat_stat_table(table_rt, 0, &start, &end, thread_num); + Maat_table_runtime_perf_stat(table_rt, 0, &start, &end, thread_num); } else { - maat_stat_table(table_rt, 0, NULL, NULL, thread_num); + Maat_table_runtime_perf_stat(table_rt, 0, NULL, NULL, thread_num); } if(compile_ret==0&®ion_hit_cnt>0) { @@ -2168,11 +2164,11 @@ int Maat_stream_scan_string_detail(stream_para_t* stream_para if(sp->feather->perf_on==1) { clock_gettime(CLOCK_MONOTONIC,&end); - maat_stat_table(table_rt,data_len,&start, &end,sp->thread_num); + Maat_table_runtime_perf_stat(table_rt,data_len,&start, &end,sp->thread_num); } else { - maat_stat_table(table_rt,data_len,NULL, NULL,sp->thread_num); + Maat_table_runtime_perf_stat(table_rt,data_len,NULL, NULL,sp->thread_num); } if(compile_ret==0&&hit_region_cnt>0) { @@ -2393,11 +2389,11 @@ fast_out: if(sp->feather->perf_on==1) { clock_gettime(CLOCK_MONOTONIC,&end); - maat_stat_table(table_rt, data_len, &start, &end, sp->thread_num); + Maat_table_runtime_perf_stat(table_rt, data_len, &start, &end, sp->thread_num); } else { - maat_stat_table(table_rt, data_len, NULL, NULL, sp->thread_num); + Maat_table_runtime_perf_stat(table_rt, data_len, NULL, NULL, sp->thread_num); } if(compile_ret==0&&hit_region_cnt>0) { diff --git a/src/entry/Maat_rule.cpp b/src/entry/Maat_rule.cpp index 7903c3d..783be7b 100644 --- a/src/entry/Maat_rule.cpp +++ b/src/entry/Maat_rule.cpp @@ -2326,9 +2326,7 @@ void do_scanner_update(struct Maat_scanner* scanner, int scan_thread_num, void* MESA_htable_handle tmp_map=NULL; struct Maat_table_runtime* table_rt=NULL; int i=0, ret=0; - struct ip_matcher* old_ip_matcher=NULL; - struct FQDN_engine* old_fqdn_engine=NULL; - enum MAAT_TABLE_TYPE table_type=0; + enum MAAT_TABLE_TYPE table_type=TABLE_TYPE_INVALID; ret=Maat_hierarchy_rebuild(scanner->hier); if(ret!=0) diff --git a/src/entry/Maat_table_runtime.cpp b/src/entry/Maat_table_runtime.cpp index f7ce1c7..91b7533 100644 --- a/src/entry/Maat_table_runtime.cpp +++ b/src/entry/Maat_table_runtime.cpp @@ -1,82 +1,16 @@ #include "Maat_rule.h" #include "Maat_utils.h" - +#include "Maat_table_runtime.h" #include -#include #include #include -#include "IPMatcher.h" -#include "gram_index_engine.h" -#include "FQDN_engine.h" -#include "alignment_int64.h" + #include "dynamic_array.h" #include #include -struct similar_runtime -{ - GIE_handle_t* gie_handle; - MESA_lqueue_head update_q; -}; -struct fqdn_plugin_runtime -{ - struct FQDN_engine* fqdn_engine; - struct FQDN_engine* new_fqdn_engine; - struct EX_data_rt* ex_data_rt; - int changed_flag; -}; -struct plugin_runtime -{ - struct EX_data_rt* ex_data_rt; - long long acc_line_num; -}; - -struct ip_plugin_runtime -{ - struct EX_data_rt* ex_data_rt; - struct ip_matcher* ip_matcher; - int changed_flag; -}; -struct expr_runtime -{ - long long expr_rule_cnt; //expr_type=0,1,3 - long long regex_rule_cnt; //expr_type=2 -}; -struct ip_runtime -{ - long long ipv4_rule_cnt; - long long ipv6_rule_cnt; - -}; -struct group2compile_runtime -{ - long long not_flag_group; -}; -struct Maat_table_runtime -{ - enum MAAT_TABLE_TYPE table_type; - long origin_rule_num; - union - { - struct similar_runtime similar; //for digest and similarity - struct fqdn_plugin_runtime fqdn_plugin;//for fqdn_plugin and fqdn_plugin - struct plugin_runtime plugin; - struct ip_plugin_runtime ip_plugin; - struct expr_runtime expr; - struct ip_runtime ip; - struct group2compile_runtime group2compile; - void * other; - }; - mcore_long_t scan_cnt; - mcore_long_t scan_cpu_time; //nano - mcore_long_t input_bytes; - mcore_long_t stream_num; - mcore_long_t hit_cnt; - struct Maat_garbage_bin* ref_garbage_bin; -}; - struct ip_rule* ip_plugin_row2ip_rule(const struct ip_plugin_table_schema* schema, const char* row) { struct ip_rule* range_rule=ALLOC(struct ip_rule, 1); @@ -530,7 +464,7 @@ void Maat_table_runtime_fqdn_plugin_new_row(struct Maat_table_runtime* table_rt, return; } -int Maat_table_runtime_fqdn_plugin_build_new_fqdn_engine(struct Maat_table_runtime* table_rt) +int Maat_table_runtime_fqdn_plugin_rebuild_fqdn_engine(struct Maat_table_runtime* table_rt) { struct FQDN_engine* new_fqdn_engine=NULL, *old_fqdn_engine=NULL; struct fqdn_plugin_runtime* fqdn_rt=&table_rt->fqdn_plugin; @@ -571,15 +505,6 @@ int Maat_table_runtime_fqdn_plugin_build_new_fqdn_engine(struct Maat_table_runti table_rt->fqdn_plugin.changed_flag=0; return ret; } -struct FQDN_engine* Maat_table_runtime_apply_new_fqdn_engine(struct Maat_table_runtime* table_rt) -{ - struct FQDN_engine* old_one=table_rt->fqdn_plugin.fqdn_engine; - table_rt->fqdn_plugin.fqdn_engine=table_rt->fqdn_plugin.new_fqdn_engine; - assert(table_rt->table_type==TABLE_TYPE_FQDN_PLUGIN); - table_rt->fqdn_plugin.new_fqdn_engine=NULL; - table_rt->origin_rule_num=EX_data_rt_get_ex_container_count(table_rt->fqdn_plugin.ex_data_rt); - return old_one; -} int Maat_table_runtime_fqdn_plugin_commit_ex_schema(struct Maat_table_runtime* table_rt, struct Maat_table_schema* table_schema, void* logger) { @@ -594,7 +519,6 @@ int Maat_table_runtime_fqdn_plugin_commit_ex_schema(struct Maat_table_runtime* t } EX_data_rt_clear_row_cache(fqdn_plugin_rt->ex_data_rt); Maat_table_runtime_fqdn_plugin_rebuild_fqdn_engine(table_rt); - Maat_table_runtime_apply_new_fqdn_engine(table_rt);//returned NULL. return 0; } int Maat_table_runtime_fqdn_plugin_get_N_ex_data(struct Maat_table_runtime* table_rt, struct Maat_table_schema* table_schema, const char* query_fqdn, MAAT_PLUGIN_EX_DATA* ex_data_array, size_t size) diff --git a/src/inc_internal/Maat_rule_internal.h b/src/inc_internal/Maat_rule_internal.h index a2ebacb..1c3c511 100644 --- a/src/inc_internal/Maat_rule_internal.h +++ b/src/inc_internal/Maat_rule_internal.h @@ -388,7 +388,6 @@ int HASH_add_by_id(MESA_htable_handle hash,int id,void*data); int HASH_delete_by_id(MESA_htable_handle hash,int id); void maat_read_full_config(_Maat_feather_t* _feather); void maat_stat_init(struct _Maat_feather_t* feather); -void maat_stat_table(struct Maat_table_runtime* p,int scan_len,struct timespec* start, struct timespec* end,int thread_num); void maat_stat_output(struct _Maat_feather_t* feather); redisReply *_wrap_redisCommand(redisContext *c, const char *format, ...); diff --git a/src/inc_internal/Maat_table_runtime.h b/src/inc_internal/Maat_table_runtime.h index 023585a..3e2e8f7 100644 --- a/src/inc_internal/Maat_table_runtime.h +++ b/src/inc_internal/Maat_table_runtime.h @@ -1,10 +1,75 @@ #include "Maat_table.h" #include "Maat_ex_data.h" +#include "IPMatcher.h" +#include "gram_index_engine.h" +#include "FQDN_engine.h" +#include "alignment_int64.h" +#include + +struct similar_runtime +{ + GIE_handle_t* gie_handle; + MESA_lqueue_head update_q; +}; +struct fqdn_plugin_runtime +{ + struct FQDN_engine* fqdn_engine; + struct EX_data_rt* ex_data_rt; + int changed_flag; +}; +struct plugin_runtime +{ + struct EX_data_rt* ex_data_rt; + long long acc_line_num; +}; + +struct ip_plugin_runtime +{ + struct EX_data_rt* ex_data_rt; + struct ip_matcher* ip_matcher; + int changed_flag; +}; +struct expr_runtime +{ + long long expr_rule_cnt; //expr_type=0,1,3 + long long regex_rule_cnt; //expr_type=2 +}; +struct ip_runtime +{ + long long ipv4_rule_cnt; + long long ipv6_rule_cnt; + +}; +struct group2compile_runtime +{ + long long not_flag_group; +}; +struct Maat_table_runtime +{ + enum MAAT_TABLE_TYPE table_type; + long origin_rule_num; + union + { + struct similar_runtime similar; //for digest and similarity + struct fqdn_plugin_runtime fqdn_plugin;//for fqdn_plugin and fqdn_plugin + struct plugin_runtime plugin; + struct ip_plugin_runtime ip_plugin; + struct expr_runtime expr; + struct ip_runtime ip; + struct group2compile_runtime group2compile; + void * other; + }; + mcore_long_t scan_cnt; + mcore_long_t scan_cpu_time; //nano + mcore_long_t input_bytes; + mcore_long_t stream_num; + mcore_long_t hit_cnt; + struct Maat_garbage_bin* ref_garbage_bin; +}; struct Maat_table_runtime_manager; -struct Maat_table_runtime; struct Maat_table_runtime_manager* Maat_table_runtime_manager_create(struct Maat_table_manager* table_manager, int max_thread_num, struct Maat_garbage_bin* bin); void Maat_table_rt_manager_destroy(struct Maat_table_runtime_manager* table_rt_mgr); struct Maat_table_runtime* Maat_table_runtime_get(struct Maat_table_runtime_manager* table_rt_mgr, int table_id);