gfwleak/tango-maat
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

未对FQDN Plugin表的changed_flag进行重置，产生不必要的FQDN Engine重建

Browse Source
This commit is contained in:
zhengchao
2021-08-04 23:18:02 +08:00
parent 96abe1d9f4
commit 7e3b746eaa
4 changed files with 132 additions and 105 deletions
Download Patch File Download Diff File Expand all files Collapse all files

19
src/entry/Maat_rule.cpp
View File

@@ -2372,25 +2372,14 @@ void do_scanner_update(struct Maat_scanner* scanner, int scan_thread_num, void*
break; break;
case TABLE_TYPE_IP_PLUGIN: case TABLE_TYPE_IP_PLUGIN:
ret=Maat_table_runtime_ip_plugin_build_new_ip_matcher(table_rt); ret=Maat_table_runtime_ip_plugin_build_new_ip_matcher(table_rt);
if(ret)
{
old_ip_matcher=Maat_table_runtime_apply_new_ip_matcher(table_rt);
if(old_ip_matcher)
{
Maat_garbage_bagging(scanner->ref_garbage_bin, old_ip_matcher, (void (*)(void*))ip_matcher_free);
}
}
break; break;
case TABLE_TYPE_FQDN_PLUGIN: case TABLE_TYPE_FQDN_PLUGIN:
ret=Maat_table_runtime_fqdn_plugin_build_new_fqdn_engine(table_rt); ret=Maat_table_runtime_fqdn_plugin_build_new_fqdn_engine(table_rt);
if(ret) if(ret<0)
{ {
old_fqdn_engine=Maat_table_runtime_apply_new_fqdn_engine(table_rt); MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_module,
if(old_fqdn_engine) "FQDN plugin table %s build failed.", Maat_table_runtime_get_name);
{
Maat_garbage_bagging(scanner->ref_garbage_bin, old_fqdn_engine, (void (*)(void*))FQDN_engine_free);
}
} }
break; break;
default: default:

114
src/entry/Maat_table_runtime.cpp
View File

@@ -1,4 +1,3 @@
#include "Maat_table_runtime.h"
#include "Maat_rule.h" #include "Maat_rule.h"
#include "Maat_utils.h" #include "Maat_utils.h"
@@ -7,6 +6,77 @@
#include <assert.h> #include <assert.h>
#include <sys/queue.h> #include <sys/queue.h>
#include "IPMatcher.h"
#include "gram_index_engine.h"
#include "FQDN_engine.h"
#include "alignment_int64.h"
#include "dynamic_array.h"
#include <MESA/MESA_htable.h>
#include <MESA/MESA_list_queue.h>
struct similar_runtime
{
GIE_handle_t* gie_handle;
MESA_lqueue_head update_q;
};
struct fqdn_plugin_runtime
{
struct FQDN_engine* fqdn_engine;
struct FQDN_engine* new_fqdn_engine;
struct EX_data_rt* ex_data_rt;
int changed_flag;
};
struct plugin_runtime
{
struct EX_data_rt* ex_data_rt;
long long acc_line_num;
};
struct ip_plugin_runtime
{
struct EX_data_rt* ex_data_rt;
struct ip_matcher* ip_matcher;
int changed_flag;
};
struct expr_runtime
{
long long expr_rule_cnt; //expr_type=0,1,3
long long regex_rule_cnt; //expr_type=2
};
struct ip_runtime
{
long long ipv4_rule_cnt;
long long ipv6_rule_cnt;
};
struct group2compile_runtime
{
long long not_flag_group;
};
struct Maat_table_runtime
{
enum MAAT_TABLE_TYPE table_type;
long origin_rule_num;
union
{
struct similar_runtime similar; //for digest and similarity
struct fqdn_plugin_runtime fqdn_plugin;//for fqdn_plugin and fqdn_plugin
struct plugin_runtime plugin;
struct ip_plugin_runtime ip_plugin;
struct expr_runtime expr;
struct ip_runtime ip;
struct group2compile_runtime group2compile;
void * other;
};
mcore_long_t scan_cnt;
mcore_long_t scan_cpu_time; //nano
mcore_long_t input_bytes;
mcore_long_t stream_num;
mcore_long_t hit_cnt;
struct Maat_garbage_bin* ref_garbage_bin;
};
struct ip_rule* ip_plugin_row2ip_rule(const struct ip_plugin_table_schema* schema, const char* row) struct ip_rule* ip_plugin_row2ip_rule(const struct ip_plugin_table_schema* schema, const char* row)
{ {
struct ip_rule* range_rule=ALLOC(struct ip_rule, 1); struct ip_rule* range_rule=ALLOC(struct ip_rule, 1);
@@ -167,7 +237,7 @@ static struct Maat_table_runtime* table_runtime_new(const struct Maat_table_sche
default: default:
break; break;
} }
table_rt->ref_garbage_bin=bin;
table_rt->scan_cnt=alignment_int64_array_alloc(max_thread_num); table_rt->scan_cnt=alignment_int64_array_alloc(max_thread_num);
table_rt->scan_cpu_time=alignment_int64_array_alloc(max_thread_num); table_rt->scan_cpu_time=alignment_int64_array_alloc(max_thread_num);
table_rt->input_bytes=alignment_int64_array_alloc(max_thread_num); table_rt->input_bytes=alignment_int64_array_alloc(max_thread_num);
@@ -447,15 +517,15 @@ void Maat_table_runtime_fqdn_plugin_new_row(struct Maat_table_runtime* table_rt,
int Maat_table_runtime_fqdn_plugin_build_new_fqdn_engine(struct Maat_table_runtime* table_rt) int Maat_table_runtime_fqdn_plugin_build_new_fqdn_engine(struct Maat_table_runtime* table_rt)
{ {
struct FQDN_engine* new_fqdn_engine=NULL; struct FQDN_engine* new_fqdn_engine=NULL, *old_fqdn_engine=NULL;
struct fqdn_plugin_runtime* fqdn_rt=&table_rt->fqdn_plugin; struct fqdn_plugin_runtime* fqdn_rt=&table_rt->fqdn_plugin;
assert(table_rt->table_type==TABLE_TYPE_FQDN_PLUGIN); assert(table_rt->table_type==TABLE_TYPE_FQDN_PLUGIN);
struct EX_data_container **exc_array=NULL; struct EX_data_container **exc_array=NULL;
struct FQDN_rule* rules=NULL; struct FQDN_rule* rules=NULL;
size_t rule_cnt=0, i=0; size_t rule_cnt=0, i=0, ret=0;
if(!fqdn_rt->changed_flag) if(!fqdn_rt->changed_flag)
{ {
return 0; return ret;
} }
rule_cnt=EX_data_rt_list_all_ex_container(fqdn_rt->ex_data_rt, &exc_array); rule_cnt=EX_data_rt_list_all_ex_container(fqdn_rt->ex_data_rt, &exc_array);
@@ -468,12 +538,23 @@ int Maat_table_runtime_fqdn_plugin_build_new_fqdn_engine(struct Maat_table_runti
} }
if(rule_cnt>0) if(rule_cnt>0)
{ {
old_fqdn_engine=fqdn_rt->fqdn_engine;
new_fqdn_engine=FQDN_engine_new(rules, rule_cnt); new_fqdn_engine=FQDN_engine_new(rules, rule_cnt);
if(new_fqdn_engine)
{
fqdn_rt->fqdn_engine=new_fqdn_engine;
Maat_garbage_bagging(table_rt->ref_garbage_bin, old_fqdn_engine, (void (*)(void*))FQDN_engine_free);
}
else
{
ret=-1;
}
} }
fqdn_rt->new_fqdn_engine=new_fqdn_engine;
free(rules); free(rules);
free(exc_array); free(exc_array);
return 1; table_rt->fqdn_plugin.changed_flag=0;
return ret;
} }
struct FQDN_engine* Maat_table_runtime_apply_new_fqdn_engine(struct Maat_table_runtime* table_rt) struct FQDN_engine* Maat_table_runtime_apply_new_fqdn_engine(struct Maat_table_runtime* table_rt)
{ {
@@ -589,7 +670,7 @@ int Maat_table_runtime_digest_batch_udpate(struct Maat_table_runtime* table_rt)
int Maat_table_runtime_ip_plugin_build_new_ip_matcher(struct Maat_table_runtime* table_rt) int Maat_table_runtime_ip_plugin_build_new_ip_matcher(struct Maat_table_runtime* table_rt)
{ {
struct ip_matcher* new_ip_matcher=NULL; struct ip_matcher* new_ip_matcher=NULL, *old_ip_matcher=NULL;
size_t rule_cnt=0; size_t rule_cnt=0;
size_t i=0, mem_use=0; size_t i=0, mem_use=0;
struct ip_rule *rules=NULL; struct ip_rule *rules=NULL;
@@ -612,8 +693,11 @@ int Maat_table_runtime_ip_plugin_build_new_ip_matcher(struct Maat_table_runtime*
{ {
new_ip_matcher=ip_matcher_new(rules, rule_cnt, &mem_use); new_ip_matcher=ip_matcher_new(rules, rule_cnt, &mem_use);
assert(ip_plugin->new_ip_matcher==NULL); assert(ip_plugin->new_ip_matcher==NULL);
ip_plugin->new_ip_matcher=new_ip_matcher; old_ip_matcher=ip_plugin->ip_matcher;
ip_plugin->ip_matcher=new_ip_matcher;
Maat_garbage_bagging(table_rt->ref_garbage_bin, old_ip_matcher, (void (*)(void*))ip_matcher_free);
} }
free(rules); free(rules);
free(exc_array); free(exc_array);
exc_array=NULL; exc_array=NULL;
@@ -622,17 +706,6 @@ int Maat_table_runtime_ip_plugin_build_new_ip_matcher(struct Maat_table_runtime*
} }
struct ip_matcher* Maat_table_runtime_apply_new_ip_matcher(struct Maat_table_runtime* table_rt)
{
struct ip_matcher* old_one=table_rt->ip_plugin.ip_matcher;
table_rt->ip_plugin.ip_matcher=table_rt->ip_plugin.new_ip_matcher;
assert(table_rt->table_type==TABLE_TYPE_IP_PLUGIN);
table_rt->ip_plugin.new_ip_matcher=NULL;
table_rt->origin_rule_num=EX_data_rt_get_ex_container_count(table_rt->ip_plugin.ex_data_rt);
return old_one;
}
void Maat_table_runtime_ip_plugin_new_row(struct Maat_table_runtime* table_rt, struct Maat_table_schema* table_schema, const char* row, void *logger) void Maat_table_runtime_ip_plugin_new_row(struct Maat_table_runtime* table_rt, struct Maat_table_schema* table_schema, const char* row, void *logger)
{ {
struct ip_plugin_table_schema* ip_plugin_schema=&(table_schema->ip_plugin); struct ip_plugin_table_schema* ip_plugin_schema=&(table_schema->ip_plugin);
@@ -699,7 +772,6 @@ int Maat_table_runtime_ip_plugin_commit_ex_schema(struct Maat_table_runtime* tab
} }
EX_data_rt_clear_row_cache(ip_plugin_rt->ex_data_rt); EX_data_rt_clear_row_cache(ip_plugin_rt->ex_data_rt);
Maat_table_runtime_ip_plugin_build_new_ip_matcher(table_rt); Maat_table_runtime_ip_plugin_build_new_ip_matcher(table_rt);
Maat_table_runtime_apply_new_ip_matcher(table_rt);//returned NULL.
return 0; return 0;
} }

68
src/inc_internal/Maat_table_runtime.h
View File

@@ -1,76 +1,10 @@
#include "Maat_table.h" #include "Maat_table.h"
#include "Maat_ex_data.h" #include "Maat_ex_data.h"
#include "IPMatcher.h"
#include "gram_index_engine.h"
#include "FQDN_engine.h"
#include "alignment_int64.h"
#include "dynamic_array.h"
#include <MESA/MESA_htable.h>
#include <MESA/MESA_list_queue.h>
struct similar_runtime
{
GIE_handle_t* gie_handle;
MESA_lqueue_head update_q;
};
struct fqdn_plugin_runtime
{
struct FQDN_engine* fqdn_engine;
struct FQDN_engine* new_fqdn_engine;
struct EX_data_rt* ex_data_rt;
int changed_flag;
};
struct plugin_runtime
{
struct EX_data_rt* ex_data_rt;
long long acc_line_num;
};
struct ip_plugin_runtime
{
struct EX_data_rt* ex_data_rt;
struct ip_matcher* ip_matcher;
struct ip_matcher* new_ip_matcher;
int changed_flag;
};
struct expr_runtime
{
long long expr_rule_cnt; //expr_type=0,1,3
long long regex_rule_cnt; //expr_type=2
};
struct ip_runtime
{
long long ipv4_rule_cnt;
long long ipv6_rule_cnt;
};
struct group2compile_runtime
{
long long not_flag_group;
};
struct Maat_table_runtime
{
enum MAAT_TABLE_TYPE table_type;
long origin_rule_num;
union
{
struct similar_runtime similar; //for digest and similarity
struct fqdn_plugin_runtime fqdn_plugin;//for fqdn_plugin and fqdn_plugin
struct plugin_runtime plugin;
struct ip_plugin_runtime ip_plugin;
struct expr_runtime expr;
struct ip_runtime ip;
struct group2compile_runtime group2compile;
void * other;
};
mcore_long_t scan_cnt;
mcore_long_t scan_cpu_time; //nano
mcore_long_t input_bytes;
mcore_long_t stream_num;
mcore_long_t hit_cnt;
};
struct Maat_table_runtime_manager; struct Maat_table_runtime_manager;
struct Maat_table_runtime;
struct Maat_table_runtime_manager* Maat_table_runtime_manager_create(struct Maat_table_manager* table_manager, int max_thread_num, struct Maat_garbage_bin* bin); struct Maat_table_runtime_manager* Maat_table_runtime_manager_create(struct Maat_table_manager* table_manager, int max_thread_num, struct Maat_garbage_bin* bin);
void Maat_table_rt_manager_destroy(struct Maat_table_runtime_manager* table_rt_mgr); void Maat_table_rt_manager_destroy(struct Maat_table_runtime_manager* table_rt_mgr);
struct Maat_table_runtime* Maat_table_runtime_get(struct Maat_table_runtime_manager* table_rt_mgr, int table_id); struct Maat_table_runtime* Maat_table_runtime_get(struct Maat_table_runtime_manager* table_rt_mgr, int table_id);

34
tools/maat_debug_tool.cpp
View File

@@ -8,6 +8,7 @@
#include <getopt.h> #include <getopt.h>
#include <sys/stat.h> #include <sys/stat.h>
#include <arpa/inet.h> #include <arpa/inet.h>
#include <unistd.h> /* for sleep*/
void debug_maat_result_print(const char* table_name, int scan_ret, struct Maat_rule_t* result) void debug_maat_result_print(const char* table_name, int scan_ret, struct Maat_rule_t* result)
{ {
@@ -102,9 +103,12 @@ enum tool_arg_type{
ARG_SCAN_CHARSET, ARG_SCAN_CHARSET,
ARG_INPUT_JSON, ARG_INPUT_JSON,
ARG_INPUT_FULL_INDEX, ARG_INPUT_FULL_INDEX,
ARG_REDIS_IP,
ARG_REDIS_PORT,
ARG_DECRYPT_KEY, ARG_DECRYPT_KEY,
ARG_ACCEPT_TAGS, ARG_ACCEPT_TAGS,
ARG_SCAN_IPv4, ARG_SCAN_IPv4,
ARG_SLEEP_SECONDS,
__ARG_MAX __ARG_MAX
}; };
@@ -129,9 +133,12 @@ int main(int argc, char ** argv)
{"scan-charset", optional_argument, 0, 0}, {"scan-charset", optional_argument, 0, 0},
{"maat-json", required_argument, 0, 0}, {"maat-json", required_argument, 0, 0},
{"full-index", required_argument, 0, 0}, {"full-index", required_argument, 0, 0},
{"redis-ip", required_argument, 0, 0},
{"redis-port", required_argument, 0, 0},
{"decrypt-key", optional_argument, 0, 0}, {"decrypt-key", optional_argument, 0, 0},
{"accept-tags", optional_argument, 0, 0}, {"accept-tags", optional_argument, 0, 0},
{"scan-ipv4", required_argument, 0, 0}, {"scan-ipv4", required_argument, 0, 0},
{"sleep-seconds", required_argument, 0, 0},
{0, 0, 0, 0} {0, 0, 0, 0}
}; };
@@ -157,6 +164,7 @@ int main(int argc, char ** argv)
int scan_detail=0, ret=0; int scan_detail=0, ret=0;
Maat_feather_t feather=NULL; Maat_feather_t feather=NULL;
void *g_logger=NULL; void *g_logger=NULL;
int redis_port=6379, sleep_seconds=0;
g_logger=MESA_create_runtime_log_handle(log_file, 0); g_logger=MESA_create_runtime_log_handle(log_file, 0);
@@ -185,11 +193,30 @@ int main(int argc, char ** argv)
ret=Maat_set_feather_opt(feather, MAAT_OPT_FULL_CFG_DIR, arg_value[ARG_INPUT_FULL_INDEX], strlen(arg_value[ARG_INPUT_FULL_INDEX])+1); ret=Maat_set_feather_opt(feather, MAAT_OPT_FULL_CFG_DIR, arg_value[ARG_INPUT_FULL_INDEX], strlen(arg_value[ARG_INPUT_FULL_INDEX])+1);
if(ret!=0) if(ret!=0)
{ {
printf("Set %s failed, invalid maat json.\n", arg_value[ARG_INPUT_FULL_INDEX]); printf("Set %s failed, invalid full cfg directory.\n", arg_value[ARG_INPUT_FULL_INDEX]);
ret=-1; ret=-1;
goto clean_up; goto clean_up;
} }
} }
else if(strlen(arg_value[ARG_REDIS_IP]))
{
ret=Maat_set_feather_opt(feather, MAAT_OPT_REDIS_IP, arg_value[ARG_REDIS_IP], strlen(arg_value[ARG_REDIS_IP])+1);
if(ret!=0)
{
printf("Set %s failed, redis IP.\n", arg_value[ARG_REDIS_IP]);
ret=-1;
goto clean_up;
}
if(strlen(arg_value[ARG_REDIS_PORT]))
{
redis_port=atoi(arg_value[ARG_REDIS_PORT]);
}
else
{
redis_port=6379;
}
ret=Maat_set_feather_opt(feather, MAAT_OPT_REDIS_PORT, &redis_port, sizeof(redis_port));
}
else else
{ {
printf("Error: One of --%s and --%s should be specified.\n", long_options[ARG_INPUT_JSON].name, long_options[ARG_INPUT_FULL_INDEX].name); printf("Error: One of --%s and --%s should be specified.\n", long_options[ARG_INPUT_JSON].name, long_options[ARG_INPUT_FULL_INDEX].name);
@@ -225,6 +252,11 @@ int main(int argc, char ** argv)
{ {
debug_maat_ip_scan(feather, arg_value[ARG_TABLE_NAME], arg_value[ARG_SCAN_IPv4]); debug_maat_ip_scan(feather, arg_value[ARG_TABLE_NAME], arg_value[ARG_SCAN_IPv4]);
} }
if(strlen(arg_value[ARG_SLEEP_SECONDS])>0)
{
sleep_seconds=atoi(arg_value[ARG_SLEEP_SECONDS]);
sleep(sleep_seconds);
}
clean_up: clean_up:
Maat_burn_feather(feather); Maat_burn_feather(feather);
MESA_destroy_runtime_log_handle(g_logger); MESA_destroy_runtime_log_handle(g_logger);