diff --git a/src/entry/Maat_rule.cpp b/src/entry/Maat_rule.cpp index 9e8bbc3..11ffeba 100644 --- a/src/entry/Maat_rule.cpp +++ b/src/entry/Maat_rule.cpp @@ -2372,25 +2372,14 @@ void do_scanner_update(struct Maat_scanner* scanner, int scan_thread_num, void* break; case TABLE_TYPE_IP_PLUGIN: ret=Maat_table_runtime_ip_plugin_build_new_ip_matcher(table_rt); - if(ret) - { - old_ip_matcher=Maat_table_runtime_apply_new_ip_matcher(table_rt); - if(old_ip_matcher) - { - Maat_garbage_bagging(scanner->ref_garbage_bin, old_ip_matcher, (void (*)(void*))ip_matcher_free); - } - } break; case TABLE_TYPE_FQDN_PLUGIN: - ret=Maat_table_runtime_fqdn_plugin_build_new_fqdn_engine(table_rt); - if(ret) + if(ret<0) { - old_fqdn_engine=Maat_table_runtime_apply_new_fqdn_engine(table_rt); - if(old_fqdn_engine) - { - Maat_garbage_bagging(scanner->ref_garbage_bin, old_fqdn_engine, (void (*)(void*))FQDN_engine_free); - } + MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module, + "FQDN plugin table %s build failed.", Maat_table_runtime_get_name); + } break; default: diff --git a/src/entry/Maat_table_runtime.cpp b/src/entry/Maat_table_runtime.cpp index 004c738..0bb0f19 100644 --- a/src/entry/Maat_table_runtime.cpp +++ b/src/entry/Maat_table_runtime.cpp @@ -1,4 +1,3 @@ -#include "Maat_table_runtime.h" #include "Maat_rule.h" #include "Maat_utils.h" @@ -7,6 +6,77 @@ #include #include + +#include "IPMatcher.h" +#include "gram_index_engine.h" +#include "FQDN_engine.h" +#include "alignment_int64.h" +#include "dynamic_array.h" +#include +#include + +struct similar_runtime +{ + GIE_handle_t* gie_handle; + MESA_lqueue_head update_q; +}; +struct fqdn_plugin_runtime +{ + struct FQDN_engine* fqdn_engine; + struct FQDN_engine* new_fqdn_engine; + struct EX_data_rt* ex_data_rt; + int changed_flag; +}; +struct plugin_runtime +{ + struct EX_data_rt* ex_data_rt; + long long acc_line_num; +}; + +struct ip_plugin_runtime +{ + struct EX_data_rt* ex_data_rt; + struct ip_matcher* ip_matcher; + int changed_flag; +}; +struct expr_runtime +{ + long long expr_rule_cnt; //expr_type=0,1,3 + long long regex_rule_cnt; //expr_type=2 +}; +struct ip_runtime +{ + long long ipv4_rule_cnt; + long long ipv6_rule_cnt; + +}; +struct group2compile_runtime +{ + long long not_flag_group; +}; +struct Maat_table_runtime +{ + enum MAAT_TABLE_TYPE table_type; + long origin_rule_num; + union + { + struct similar_runtime similar; //for digest and similarity + struct fqdn_plugin_runtime fqdn_plugin;//for fqdn_plugin and fqdn_plugin + struct plugin_runtime plugin; + struct ip_plugin_runtime ip_plugin; + struct expr_runtime expr; + struct ip_runtime ip; + struct group2compile_runtime group2compile; + void * other; + }; + mcore_long_t scan_cnt; + mcore_long_t scan_cpu_time; //nano + mcore_long_t input_bytes; + mcore_long_t stream_num; + mcore_long_t hit_cnt; + struct Maat_garbage_bin* ref_garbage_bin; +}; + struct ip_rule* ip_plugin_row2ip_rule(const struct ip_plugin_table_schema* schema, const char* row) { struct ip_rule* range_rule=ALLOC(struct ip_rule, 1); @@ -167,7 +237,7 @@ static struct Maat_table_runtime* table_runtime_new(const struct Maat_table_sche default: break; } - + table_rt->ref_garbage_bin=bin; table_rt->scan_cnt=alignment_int64_array_alloc(max_thread_num); table_rt->scan_cpu_time=alignment_int64_array_alloc(max_thread_num); table_rt->input_bytes=alignment_int64_array_alloc(max_thread_num); @@ -447,15 +517,15 @@ void Maat_table_runtime_fqdn_plugin_new_row(struct Maat_table_runtime* table_rt, int Maat_table_runtime_fqdn_plugin_build_new_fqdn_engine(struct Maat_table_runtime* table_rt) { - struct FQDN_engine* new_fqdn_engine=NULL; + struct FQDN_engine* new_fqdn_engine=NULL, *old_fqdn_engine=NULL; struct fqdn_plugin_runtime* fqdn_rt=&table_rt->fqdn_plugin; assert(table_rt->table_type==TABLE_TYPE_FQDN_PLUGIN); struct EX_data_container **exc_array=NULL; struct FQDN_rule* rules=NULL; - size_t rule_cnt=0, i=0; + size_t rule_cnt=0, i=0, ret=0; if(!fqdn_rt->changed_flag) { - return 0; + return ret; } rule_cnt=EX_data_rt_list_all_ex_container(fqdn_rt->ex_data_rt, &exc_array); @@ -468,12 +538,23 @@ int Maat_table_runtime_fqdn_plugin_build_new_fqdn_engine(struct Maat_table_runti } if(rule_cnt>0) { + + old_fqdn_engine=fqdn_rt->fqdn_engine; new_fqdn_engine=FQDN_engine_new(rules, rule_cnt); + if(new_fqdn_engine) + { + fqdn_rt->fqdn_engine=new_fqdn_engine; + Maat_garbage_bagging(table_rt->ref_garbage_bin, old_fqdn_engine, (void (*)(void*))FQDN_engine_free); + } + else + { + ret=-1; + } } - fqdn_rt->new_fqdn_engine=new_fqdn_engine; free(rules); - free(exc_array); - return 1; + free(exc_array); + table_rt->fqdn_plugin.changed_flag=0; + return ret; } struct FQDN_engine* Maat_table_runtime_apply_new_fqdn_engine(struct Maat_table_runtime* table_rt) { @@ -589,7 +670,7 @@ int Maat_table_runtime_digest_batch_udpate(struct Maat_table_runtime* table_rt) int Maat_table_runtime_ip_plugin_build_new_ip_matcher(struct Maat_table_runtime* table_rt) { - struct ip_matcher* new_ip_matcher=NULL; + struct ip_matcher* new_ip_matcher=NULL, *old_ip_matcher=NULL; size_t rule_cnt=0; size_t i=0, mem_use=0; struct ip_rule *rules=NULL; @@ -612,8 +693,11 @@ int Maat_table_runtime_ip_plugin_build_new_ip_matcher(struct Maat_table_runtime* { new_ip_matcher=ip_matcher_new(rules, rule_cnt, &mem_use); assert(ip_plugin->new_ip_matcher==NULL); - ip_plugin->new_ip_matcher=new_ip_matcher; + old_ip_matcher=ip_plugin->ip_matcher; + ip_plugin->ip_matcher=new_ip_matcher; + Maat_garbage_bagging(table_rt->ref_garbage_bin, old_ip_matcher, (void (*)(void*))ip_matcher_free); } + free(rules); free(exc_array); exc_array=NULL; @@ -622,17 +706,6 @@ int Maat_table_runtime_ip_plugin_build_new_ip_matcher(struct Maat_table_runtime* } - -struct ip_matcher* Maat_table_runtime_apply_new_ip_matcher(struct Maat_table_runtime* table_rt) -{ - struct ip_matcher* old_one=table_rt->ip_plugin.ip_matcher; - table_rt->ip_plugin.ip_matcher=table_rt->ip_plugin.new_ip_matcher; - assert(table_rt->table_type==TABLE_TYPE_IP_PLUGIN); - table_rt->ip_plugin.new_ip_matcher=NULL; - table_rt->origin_rule_num=EX_data_rt_get_ex_container_count(table_rt->ip_plugin.ex_data_rt); - return old_one; -} - void Maat_table_runtime_ip_plugin_new_row(struct Maat_table_runtime* table_rt, struct Maat_table_schema* table_schema, const char* row, void *logger) { struct ip_plugin_table_schema* ip_plugin_schema=&(table_schema->ip_plugin); @@ -699,7 +772,6 @@ int Maat_table_runtime_ip_plugin_commit_ex_schema(struct Maat_table_runtime* tab } EX_data_rt_clear_row_cache(ip_plugin_rt->ex_data_rt); Maat_table_runtime_ip_plugin_build_new_ip_matcher(table_rt); - Maat_table_runtime_apply_new_ip_matcher(table_rt);//returned NULL. return 0; } diff --git a/src/inc_internal/Maat_table_runtime.h b/src/inc_internal/Maat_table_runtime.h index ff5d804..7b34551 100644 --- a/src/inc_internal/Maat_table_runtime.h +++ b/src/inc_internal/Maat_table_runtime.h @@ -1,76 +1,10 @@ #include "Maat_table.h" #include "Maat_ex_data.h" -#include "IPMatcher.h" -#include "gram_index_engine.h" -#include "FQDN_engine.h" -#include "alignment_int64.h" -#include "dynamic_array.h" -#include -#include -struct similar_runtime -{ - GIE_handle_t* gie_handle; - MESA_lqueue_head update_q; -}; -struct fqdn_plugin_runtime -{ - struct FQDN_engine* fqdn_engine; - struct FQDN_engine* new_fqdn_engine; - struct EX_data_rt* ex_data_rt; - int changed_flag; -}; -struct plugin_runtime -{ - struct EX_data_rt* ex_data_rt; - long long acc_line_num; -}; -struct ip_plugin_runtime -{ - struct EX_data_rt* ex_data_rt; - struct ip_matcher* ip_matcher; - struct ip_matcher* new_ip_matcher; - int changed_flag; -}; -struct expr_runtime -{ - long long expr_rule_cnt; //expr_type=0,1,3 - long long regex_rule_cnt; //expr_type=2 -}; -struct ip_runtime -{ - long long ipv4_rule_cnt; - long long ipv6_rule_cnt; - -}; -struct group2compile_runtime -{ - long long not_flag_group; -}; -struct Maat_table_runtime -{ - enum MAAT_TABLE_TYPE table_type; - long origin_rule_num; - union - { - struct similar_runtime similar; //for digest and similarity - struct fqdn_plugin_runtime fqdn_plugin;//for fqdn_plugin and fqdn_plugin - struct plugin_runtime plugin; - struct ip_plugin_runtime ip_plugin; - struct expr_runtime expr; - struct ip_runtime ip; - struct group2compile_runtime group2compile; - void * other; - }; - mcore_long_t scan_cnt; - mcore_long_t scan_cpu_time; //nano - mcore_long_t input_bytes; - mcore_long_t stream_num; - mcore_long_t hit_cnt; -}; struct Maat_table_runtime_manager; +struct Maat_table_runtime; struct Maat_table_runtime_manager* Maat_table_runtime_manager_create(struct Maat_table_manager* table_manager, int max_thread_num, struct Maat_garbage_bin* bin); void Maat_table_rt_manager_destroy(struct Maat_table_runtime_manager* table_rt_mgr); struct Maat_table_runtime* Maat_table_runtime_get(struct Maat_table_runtime_manager* table_rt_mgr, int table_id); diff --git a/tools/maat_debug_tool.cpp b/tools/maat_debug_tool.cpp index 5697bab..e75f02b 100644 --- a/tools/maat_debug_tool.cpp +++ b/tools/maat_debug_tool.cpp @@ -8,6 +8,7 @@ #include #include #include +#include /* for sleep*/ void debug_maat_result_print(const char* table_name, int scan_ret, struct Maat_rule_t* result) { @@ -102,9 +103,12 @@ enum tool_arg_type{ ARG_SCAN_CHARSET, ARG_INPUT_JSON, ARG_INPUT_FULL_INDEX, + ARG_REDIS_IP, + ARG_REDIS_PORT, ARG_DECRYPT_KEY, ARG_ACCEPT_TAGS, ARG_SCAN_IPv4, + ARG_SLEEP_SECONDS, __ARG_MAX }; @@ -129,9 +133,12 @@ int main(int argc, char ** argv) {"scan-charset", optional_argument, 0, 0}, {"maat-json", required_argument, 0, 0}, {"full-index", required_argument, 0, 0}, + {"redis-ip", required_argument, 0, 0}, + {"redis-port", required_argument, 0, 0}, {"decrypt-key", optional_argument, 0, 0}, {"accept-tags", optional_argument, 0, 0}, {"scan-ipv4", required_argument, 0, 0}, + {"sleep-seconds", required_argument, 0, 0}, {0, 0, 0, 0} }; @@ -157,6 +164,7 @@ int main(int argc, char ** argv) int scan_detail=0, ret=0; Maat_feather_t feather=NULL; void *g_logger=NULL; + int redis_port=6379, sleep_seconds=0; g_logger=MESA_create_runtime_log_handle(log_file, 0); @@ -185,11 +193,30 @@ int main(int argc, char ** argv) ret=Maat_set_feather_opt(feather, MAAT_OPT_FULL_CFG_DIR, arg_value[ARG_INPUT_FULL_INDEX], strlen(arg_value[ARG_INPUT_FULL_INDEX])+1); if(ret!=0) { - printf("Set %s failed, invalid maat json.\n", arg_value[ARG_INPUT_FULL_INDEX]); + printf("Set %s failed, invalid full cfg directory.\n", arg_value[ARG_INPUT_FULL_INDEX]); ret=-1; goto clean_up; } } + else if(strlen(arg_value[ARG_REDIS_IP])) + { + ret=Maat_set_feather_opt(feather, MAAT_OPT_REDIS_IP, arg_value[ARG_REDIS_IP], strlen(arg_value[ARG_REDIS_IP])+1); + if(ret!=0) + { + printf("Set %s failed, redis IP.\n", arg_value[ARG_REDIS_IP]); + ret=-1; + goto clean_up; + } + if(strlen(arg_value[ARG_REDIS_PORT])) + { + redis_port=atoi(arg_value[ARG_REDIS_PORT]); + } + else + { + redis_port=6379; + } + ret=Maat_set_feather_opt(feather, MAAT_OPT_REDIS_PORT, &redis_port, sizeof(redis_port)); + } else { printf("Error: One of --%s and --%s should be specified.\n", long_options[ARG_INPUT_JSON].name, long_options[ARG_INPUT_FULL_INDEX].name); @@ -225,6 +252,11 @@ int main(int argc, char ** argv) { debug_maat_ip_scan(feather, arg_value[ARG_TABLE_NAME], arg_value[ARG_SCAN_IPv4]); } + if(strlen(arg_value[ARG_SLEEP_SECONDS])>0) + { + sleep_seconds=atoi(arg_value[ARG_SLEEP_SECONDS]); + sleep(sleep_seconds); + } clean_up: Maat_burn_feather(feather); MESA_destroy_runtime_log_handle(g_logger);