gfwleak/tango-maat
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[BUGFIX]fix illegal clause index

Browse Source
This commit is contained in:
刘文坛
2023-10-11 06:53:03 +00:00
parent 461d43c6b7
commit e49427974f
13 changed files with 153 additions and 46 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
src/maat_bool_plugin.c
View File

@@ -304,7 +304,7 @@ bool_plugin_expr_new(struct bool_plugin_schema *schema, const char *table_name,
size_t column_offset = 0;
size_t column_len = 0;
size_t n_item = 0;
char expr_buffer[BUFSIZ] = {0};
char expr_buffer[BUFSIZ + 1] = {0};
unsigned long long items[MAX_ITEMS_PER_BOOL_EXPR] = {0};
char *token = NULL, *sub_token = NULL, *saveptr;
struct bool_expr *bool_expr = ALLOC(struct bool_expr, 1);
@@ -467,7 +467,13 @@ int bool_plugin_runtime_commit(void *bool_plugin_runtime, const char *table_name
struct bool_matcher *old_bool_matcher = NULL;
if (rule_cnt > 0) {
struct timespec start, end;
clock_gettime(CLOCK_MONOTONIC, &start);
new_bool_matcher = bool_matcher_new(rules, rule_cnt, &mem_used);
clock_gettime(CLOCK_MONOTONIC, &end);
long long time_elapse_ms = (end.tv_sec - start.tv_sec) * 1000 +
(end.tv_nsec - start.tv_nsec) / 1000000;
if (NULL == new_bool_matcher) {
log_error(bool_plugin_rt->logger, MODULE_BOOL_PLUGIN,
"[%s:%d] table[%s] rebuild bool_matcher engine failed when "
@@ -477,7 +483,8 @@ int bool_plugin_runtime_commit(void *bool_plugin_runtime, const char *table_name
} else {
log_info(bool_plugin_rt->logger, MODULE_BOOL_PLUGIN,
"table[%s] commit %zu bool_plugin rules and rebuild bool_matcher"
" completed, version:%lld", table_name, rule_cnt, maat_rt_version);
" completed, version:%lld, consume:%lldms", table_name, rule_cnt,
maat_rt_version, time_elapse_ms);
}
}

43
src/maat_compile.c
View File

@@ -29,6 +29,12 @@
#define MODULE_COMPILE module_name_str("maat.compile")
#define DEFAULT_GC_TIMEOUT_S 10
#define MAX_SUPER_GROUP_CNT 128
#define MAX_NOT_CLAUSE_NUM 8
enum clause_not_flag {
CLAUSE_NOT_FLAG_UNSET = 0,
CLAUSE_NOT_FLAG_SET
};
struct compile_schema {
int compile_id_column;
@@ -275,7 +281,16 @@ compile_item_new(const char *table_line, struct compile_schema *schema,
__FUNCTION__, __LINE__, table_name, table_line);
goto error;
}
compile_item->declared_clause_num = atoi(table_line + column_offset);
if (compile_item->declared_clause_num < 0 ||
compile_item->declared_clause_num > MAX_NOT_CLAUSE_NUM) {
log_error(logger, MODULE_COMPILE,
"[%s:%d] table: <%s> clause_num:%d exceed maximum:%d in line:%s",
__FUNCTION__, __LINE__, table_name, compile_item->declared_clause_num,
MAX_NOT_CLAUSE_NUM, table_line);
goto error;
}
compile_item->ref_schema = schema;
compile_item->ex_data = ALLOC(void *, 1);
@@ -308,7 +323,7 @@ static void compile_item_free(struct compile_item *item)
FREE(item->ex_data);
}
item->declared_clause_num = -1;
item->declared_clause_num = 0;
if (item->table_line != NULL) {
FREE(item->table_line);
@@ -722,7 +737,7 @@ group2compile_item_new(const char *line, struct group2compile_schema *g2c_schema
{
size_t column_offset = 0;
size_t column_len = 0;
char vtable_name[MAX_NAME_STR_LEN] = {0};
char vtable_name[MAX_NAME_STR_LEN + 1] = {0};
struct group2compile_item *g2c_item = ALLOC(struct group2compile_item, 1);
int ret = get_column_pos(line, g2c_schema->group_id_column, &column_offset,
@@ -753,7 +768,15 @@ group2compile_item_new(const char *line, struct group2compile_schema *g2c_schema
__FUNCTION__, __LINE__, table_name, line);
goto error;
}
g2c_item->not_flag = atoi(line + column_offset);
if (g2c_item->not_flag != CLAUSE_NOT_FLAG_SET &&
g2c_item->not_flag != CLAUSE_NOT_FLAG_UNSET) {
log_error(logger, MODULE_COMPILE,
"[%s:%d] g2c table:<%s> NOT_flag:%d is illegal in line:%s ",
__FUNCTION__, __LINE__, table_name, g2c_item->not_flag, line);
goto error;
}
ret = get_column_pos(line, g2c_schema->vtable_name_column, &column_offset,
&column_len);
@@ -795,6 +818,13 @@ group2compile_item_new(const char *line, struct group2compile_schema *g2c_schema
}
g2c_item->clause_index = atoi(line + column_offset);
if (g2c_item->clause_index < 0 || g2c_item->clause_index >= MAX_NOT_CLAUSE_NUM) {
log_error(logger, MODULE_COMPILE,
"[%s:%d] g2c table:<%s> clause_index:%d exceed maximum:%d in line:%s",
__FUNCTION__, __LINE__, table_name, g2c_item->clause_index,
MAX_NOT_CLAUSE_NUM, line);
goto error;
}
return g2c_item;
error:
@@ -1947,7 +1977,13 @@ int compile_runtime_commit(void *compile_runtime, const char *table_name,
struct bool_matcher *old_bool_matcher = NULL;
struct bool_matcher *new_bool_matcher = NULL;
struct timespec start, end;
clock_gettime(CLOCK_MONOTONIC, &start);
new_bool_matcher = maat_compile_bool_matcher_new(compile_rt, &compile_cnt);
clock_gettime(CLOCK_MONOTONIC, &end);
long long time_elapse_ms = (end.tv_sec - start.tv_sec) * 1000 +
(end.tv_nsec - start.tv_nsec) / 1000000;
if (NULL == new_bool_matcher) {
log_error(compile_rt->logger, MODULE_COMPILE,
"[%s:%d] table[%s] rebuild compile bool_matcher failed, compile"
@@ -1956,7 +1992,8 @@ int compile_runtime_commit(void *compile_runtime, const char *table_name,
} else {
log_info(compile_rt->logger, MODULE_COMPILE,
"table[%s] commit %zu compile rules and rebuild compile bool_matcher"
" completed, version:%lld", table_name, compile_cnt, maat_rt_version);
" completed, version:%lld, consume:%lldms", table_name, compile_cnt,
maat_rt_version, time_elapse_ms);
}
struct literal_clause *old_literal2clause = NULL;

10
src/maat_expr.c
View File

@@ -877,8 +877,14 @@ int expr_runtime_commit(void *expr_runtime, const char *table_name,
engine_type = EXPR_ENGINE_TYPE_RS;
}
struct timespec start, end;
clock_gettime(CLOCK_MONOTONIC, &start);
new_matcher = expr_matcher_new(rules, real_rule_cnt, engine_type,
expr_rt->n_worker_thread, expr_rt->logger);
clock_gettime(CLOCK_MONOTONIC, &end);
long long time_elapse_ms = (end.tv_sec - start.tv_sec) * 1000 +
(end.tv_nsec - start.tv_nsec) / 1000000;
if (NULL == new_matcher) {
log_error(expr_rt->logger, MODULE_EXPR,
"[%s:%d] table[%s] rebuild expr_matcher failed when update"
@@ -887,8 +893,8 @@ int expr_runtime_commit(void *expr_runtime, const char *table_name,
} else {
log_info(expr_rt->logger, MODULE_EXPR,
"table[%s] has %zu rules, commit %zu expr rules(regex rules:%zu) "
"and rebuild adapter_hs completed, version:%lld", table_name, rule_cnt,
real_rule_cnt, real_regex_rule_cnt, maat_rt_version);
"and rebuild adapter_hs completed, version:%lld, consume:%lldms", table_name, rule_cnt,
real_rule_cnt, real_regex_rule_cnt, maat_rt_version, time_elapse_ms);
}
}

11
src/maat_flag.c
View File

@@ -360,7 +360,7 @@ flag_item_new(struct flag_schema *schema, const char *table_name,
item->flag = strtoull(line + column_offset, NULL, 0);
ret = get_column_pos(line, schema->flag_mask_column, &column_offset, &column_len);
ret = get_column_pos(line, schema->flag_mask_column, &column_offset, &column_len);
if (ret < 0) {
log_error(flag_rt->logger, MODULE_FLAG,
"[%s:%d] flag table:<%s> has no flag_mask in line:%s",
@@ -490,7 +490,13 @@ int flag_runtime_commit(void *flag_runtime, const char *table_name,
struct flag_matcher *old_flag_matcher = NULL;
if (rule_cnt > 0) {
struct timespec start, end;
clock_gettime(CLOCK_MONOTONIC, &start);
new_flag_matcher = flag_matcher_new(rules, rule_cnt);
clock_gettime(CLOCK_MONOTONIC, &end);
long long time_elapse_ms = (end.tv_sec - start.tv_sec) * 1000 +
(end.tv_nsec - start.tv_nsec) / 1000000;
if (NULL == new_flag_matcher) {
log_error(flag_rt->logger, MODULE_FLAG,
"[%s:%d] table[%s] rebuild flag_matcher engine failed "
@@ -500,7 +506,8 @@ int flag_runtime_commit(void *flag_runtime, const char *table_name,
} else {
log_info(flag_rt->logger, MODULE_FLAG,
"table[%s] commit %zu flag rules and rebuild flag_matcher completed,"
" version:%lld", table_name, rule_cnt, maat_rt_version);
" version:%lld, consume:%lldms", table_name, rule_cnt, maat_rt_version,
time_elapse_ms);
}
}

23
src/maat_fqdn_plugin.c
View File

@@ -21,7 +21,7 @@
struct fqdn_plugin_schema {
int item_id_column;
int suffix_flag_column;
int suffix_match_method_column;
int fqdn_column;
int rule_tag_column;
int gc_timeout_s;
@@ -79,7 +79,7 @@ void *fqdn_plugin_schema_new(cJSON *json, struct table_manager *tbl_mgr,
custom_item = cJSON_GetObjectItem(item, "suffix_match_method");
if (custom_item != NULL && custom_item->type == cJSON_Number) {
schema->suffix_flag_column = custom_item->valueint;
schema->suffix_match_method_column = custom_item->valueint;
} else {
log_error(logger, MODULE_FQDN_PLUGIN,
"[%s:%d] fqdn_plugin table:<%s> schema has no suffix_match_method column",
@@ -287,14 +287,22 @@ fqdn_plugin_rule_new(const char *line, struct fqdn_plugin_schema *schema,
}
fqdn_plugin_rule->id = atoi(line + column_offset);
ret = get_column_pos(line, schema->suffix_flag_column, &column_offset, &column_len);
ret = get_column_pos(line, schema->suffix_match_method_column, &column_offset, &column_len);
if (ret < 0) {
log_error(logger, MODULE_FQDN_PLUGIN,
"[%s:%d] fqdn_plugin table:<%s> has no suffix_match_method in line:%s",
__FUNCTION__, __LINE__, table_name, line);
goto error;
}
fqdn_plugin_rule->is_suffix_match = atoi(line + column_offset);
if (fqdn_plugin_rule->is_suffix_match != 0 &&
fqdn_plugin_rule->is_suffix_match != 1) {
log_error(logger, MODULE_FQDN_PLUGIN,
"[%s:%d] fqdn_plugin table:<%s> suffix_match_method:%d is illegal in line:%s",
__FUNCTION__, __LINE__, table_name, fqdn_plugin_rule->is_suffix_match, line);
goto error;
}
ret = get_column_pos(line, schema->fqdn_column, &column_offset, &column_len);
if (ret < 0) {
@@ -473,7 +481,13 @@ int fqdn_plugin_runtime_commit(void *fqdn_plugin_runtime, const char *table_name
struct FQDN_engine *old_fqdn_engine = NULL;
if (rule_cnt > 0) {
struct timespec start, end;
clock_gettime(CLOCK_MONOTONIC, &start);
new_fqdn_engine = FQDN_engine_new(rules, rule_cnt);
clock_gettime(CLOCK_MONOTONIC, &end);
long long time_elapse_ms = (end.tv_sec - start.tv_sec) * 1000 +
(end.tv_nsec - start.tv_nsec) / 1000000;
if (NULL == new_fqdn_engine) {
log_error(fqdn_plugin_rt->logger, MODULE_FQDN_PLUGIN,
"[%s:%d] table[%s] rebuild FQDN engine failed when update"
@@ -483,7 +497,8 @@ int fqdn_plugin_runtime_commit(void *fqdn_plugin_runtime, const char *table_name
} else {
log_info(fqdn_plugin_rt->logger, MODULE_FQDN_PLUGIN,
"table[%s] commit %zu fqdn_plugin rules and rebuild FQDN engine"
" completed, version:%lld", table_name, rule_cnt, maat_rt_version);
" completed, version:%lld, consume:%lldms", table_name, rule_cnt,
maat_rt_version, time_elapse_ms);
}
}

16
src/maat_group.c
View File

@@ -354,7 +354,14 @@ group2group_item_new(const char *line, struct group2group_schema *g2g_schema,
__FUNCTION__, __LINE__, table_name, line);
goto error;
}
g2g_item->is_exclude = atoi(line + column_offset);
if (g2g_item->is_exclude != 0 && g2g_item->is_exclude != 1) {
log_error(logger, MODULE_GROUP,
"[%s:%d] g2g table:<%s> is_exclude:%d is illegal in line:%s",
__FUNCTION__, __LINE__, table_name, g2g_item->is_exclude, line);
goto error;
}
return g2g_item;
error:
@@ -768,7 +775,13 @@ int group2group_runtime_commit(void *g2g_runtime, const char *table_name,
return 0;
}
struct timespec start, end;
clock_gettime(CLOCK_MONOTONIC, &start);
int ret = group_topology_build_super_groups(g2g_rt->updating_group_topo);
clock_gettime(CLOCK_MONOTONIC, &end);
long long time_elapse_ms = (end.tv_sec - start.tv_sec) * 1000 +
(end.tv_nsec - start.tv_nsec) / 1000000;
if (ret < 0) {
log_error(g2g_rt->logger, MODULE_GROUP,
"[%s:%d] table[%s] group2group runtime commit failed",
@@ -786,7 +799,8 @@ int group2group_runtime_commit(void *g2g_runtime, const char *table_name,
log_info(g2g_rt->logger, MODULE_GROUP,
"table[%s] commit %zu g2g rules and rebuild super_groups completed,"
" version:%lld", table_name, g2g_rt->rule_num, maat_rt_version);
" version:%lld, consume:%lldms", table_name, g2g_rt->rule_num,
maat_rt_version, time_elapse_ms);
return 0;
}

9
src/maat_interval.c
View File

@@ -491,7 +491,13 @@ int interval_runtime_commit(void *interval_runtime, const char *table_name,
struct interval_matcher *old_interval_matcher = NULL;
if (rule_cnt > 0) {
struct timespec start, end;
clock_gettime(CLOCK_MONOTONIC, &start);
new_interval_matcher = interval_matcher_new(rules, rule_cnt);
clock_gettime(CLOCK_MONOTONIC, &end);
long long time_elapse_ms = (end.tv_sec - start.tv_sec) * 1000 +
(end.tv_nsec - start.tv_nsec) / 1000000;
if (NULL == new_interval_matcher) {
log_error(interval_rt->logger, MODULE_INTERVAL,
"[%s:%d] table[%s]rebuild interval_matcher engine failed "
@@ -501,7 +507,8 @@ int interval_runtime_commit(void *interval_runtime, const char *table_name,
} else {
log_info(interval_rt->logger, MODULE_INTERVAL,
"table[%s] commit %zu interval rules and rebuild interval_matcher "
"completed, version:%lld", table_name, rule_cnt, maat_rt_version);
"completed, version:%lld, consume:%lldms", table_name, rule_cnt,
maat_rt_version, time_elapse_ms);
}
}

22
src/maat_ip.c
View File

@@ -24,6 +24,11 @@
#define MODULE_IP module_name_str("maat.ip")
#define IP_PROTO_ANY -1
#define IP_PROTO_ICMP 1
#define IP_PROTO_TCP 6
#define IP_PROTO_UDP 17
struct ip_schema {
int item_id_column;
int group_id_column;
@@ -370,7 +375,15 @@ ip_item_new(struct ip_schema *ip_schema, const char *table_name,
__FUNCTION__, __LINE__, table_name, line);
goto error;
}
ip_item->proto = atoi(line + column_offset);
if (ip_item->proto != IP_PROTO_ANY && ip_item->proto != IP_PROTO_ICMP &&
ip_item->proto != IP_PROTO_TCP && ip_item->proto != IP_PROTO_UDP) {
log_error(logger, MODULE_IP,
"[%s:%d] ip table:<%s> protocol:%d is illegal in line:%s",
__FUNCTION__, __LINE__, table_name, ip_item->proto, line);
goto error;
}
return ip_item;
error:
@@ -596,7 +609,13 @@ int ip_runtime_commit(void *ip_runtime, const char *table_name,
struct ip_matcher *old_ip_matcher = NULL;
if (rule_cnt > 0) {
struct timespec start, end;
clock_gettime(CLOCK_MONOTONIC, &start);
new_ip_matcher = ip_matcher_new(rules, rule_cnt, &mem_used);
clock_gettime(CLOCK_MONOTONIC, &end);
long long time_elapse_ms = (end.tv_sec - start.tv_sec) * 1000 +
(end.tv_nsec - start.tv_nsec) / 1000000;
if (NULL == new_ip_matcher) {
log_error(ip_rt->logger, MODULE_IP,
"[%s:%d] table[%s] rebuild ip_matcher engine failed "
@@ -606,7 +625,8 @@ int ip_runtime_commit(void *ip_runtime, const char *table_name,
} else {
log_info(ip_rt->logger, MODULE_IP,
"table[%s] commit %zu ip rules and rebuild ip_matcher completed"
", version:%lld", table_name, rule_cnt, maat_rt_version);
", version:%lld, consume:%lldms", table_name, rule_cnt,
maat_rt_version, time_elapse_ms);
}
}

22
src/maat_ip_plugin.c
View File

@@ -26,7 +26,6 @@ struct ip_plugin_schema {
int ip_type_column;
int start_ip_column;
int end_ip_column;
int addr_format_column;
int rule_tag_column;
int gc_timeout_s;
int table_id; //ugly
@@ -111,18 +110,6 @@ void *ip_plugin_schema_new(cJSON *json, struct table_manager *tbl_mgr,
goto error;
}
custom_item = cJSON_GetObjectItem(item, "addr_format");
if (custom_item != NULL && custom_item->type == cJSON_Number) {
schema->addr_format_column = custom_item->valueint;
}
//TODO: just because test table has no addr_format
// else {
// log_error(logger, MODULE_IP_PLUGIN,
// "[%s:%d] table: <%s> schema has no addr_format column",
// __FUNCTION__, __LINE__, table_name);
// goto error;
// }
// rule_tag is optional
custom_item = cJSON_GetObjectItem(item, "rule_tag");
if (custom_item != NULL && custom_item->type == cJSON_Number) {
@@ -491,7 +478,13 @@ int ip_plugin_runtime_commit(void *ip_plugin_runtime, const char *table_name,
struct ip_matcher *old_ip_matcher = NULL;
if (rule_cnt > 0) {
struct timespec start, end;
clock_gettime(CLOCK_MONOTONIC, &start);
new_ip_matcher = ip_matcher_new(rules, rule_cnt, &mem_used);
clock_gettime(CLOCK_MONOTONIC, &end);
long long time_elapse_ms = (end.tv_sec - start.tv_sec) * 1000 +
(end.tv_nsec - start.tv_nsec) / 1000000;
if (NULL == new_ip_matcher) {
log_error(ip_plugin_rt->logger, MODULE_IP_PLUGIN,
"[%s:%d] ip_plugin table[%s] rebuild ip_matcher failed when "
@@ -500,7 +493,8 @@ int ip_plugin_runtime_commit(void *ip_plugin_runtime, const char *table_name,
} else {
log_info(ip_plugin_rt->logger, MODULE_IP_PLUGIN,
"table[%s] commit %zu ip_plugin rules and rebuild ip_matcher "
"completed, version:%lld", table_name, rule_cnt, maat_rt_version);
"completed, version:%lld, consume:%lldms", table_name, rule_cnt,
maat_rt_version, time_elapse_ms);
}
}

3
src/maat_ipport_plugin.c
View File

@@ -500,7 +500,8 @@ int ipport_plugin_runtime_commit(void *ipport_plugin_runtime, const char *table_
clock_gettime(CLOCK_MONOTONIC, &start);
new_matcher = ipport_matcher_new(rules, rule_cnt);
clock_gettime(CLOCK_MONOTONIC, &end);
long long time_elapse_ms = (end.tv_sec - start.tv_sec) * 1000 + (end.tv_nsec - start.tv_nsec) / 1000000;
long long time_elapse_ms = (end.tv_sec - start.tv_sec) * 1000 +
(end.tv_nsec - start.tv_nsec) / 1000000;
if (NULL == new_matcher) {
log_error(ipport_plugin_rt->logger, MODULE_IPPORT_PLUGIN,
"[%s:%d] ipport_plugin table[%s] rebuild ipport_matcher failed when "