gfwleak/tango-maat
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

compile table support conjunction, ip_plugin support cidr

Browse Source
This commit is contained in:
liuwentan
2023-02-20 10:57:40 +08:00
parent be5d157733
commit bbed56db80
30 changed files with 1030 additions and 523 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
src/inc_internal/maat_compile.h
View File

@@ -74,7 +74,7 @@ size_t compile_runtime_get_hit_paths(struct compile_runtime *compile_rt,
struct group2group_runtime *g2g_rt,
struct maat_compile_state *compile_state,
struct maat_hit_path *hit_paths,
size_t n_hit_path);
size_t hit_path_index, size_t n_hit_path);
/* group2compile runtime API */
void *group2compile_runtime_new(void *g2c_schema, int max_thread_num,
struct maat_garbage_bin *garbage_bin,
@@ -92,7 +92,7 @@ void maat_compile_state_free(struct maat_compile_state *compile_state);
int maat_compile_state_update(struct maat_item *item_hash, int vtable_id,
int *hit_item_ids, size_t hit_item_cnt,
int *group_ids, size_t group_ids_size,
int *group_ids, int hit_group_index, size_t group_ids_size,
size_t *n_hit_group_id, struct maat_state *state);
int maat_compile_state_has_NOT_clause(struct maat_compile_state *compile_state);

1
src/inc_internal/maat_config_monitor.h
View File

@@ -17,6 +17,7 @@ extern "C"
#endif
#include <stdint.h>
#include "log/log.h"
void config_monitor_traverse(long long version, const char *idx_dir,
void (*start_fn)(long long, int, void *),

7
src/inc_internal/maat_ex_data.h
View File

@@ -82,9 +82,12 @@ int ex_data_runtime_del_ex_container(struct ex_data_runtime *ex_data_rt,
size_t ex_data_runtime_list_updating_ex_container(struct ex_data_runtime *ex_data_rt,
struct ex_data_container ***ex_container);
void *ex_data_runtime_get_ex_data(struct ex_data_runtime *ex_data_rt,
const char *key, size_t key_len);
void *ex_data_runtime_get_ex_data_by_key(struct ex_data_runtime *ex_data_rt,
const char *key, size_t key_len);
void *ex_data_runtime_get_ex_data_by_container(struct ex_data_runtime *ex_data_rt,
struct ex_data_container *ex_container);
void *ex_data_runtime_get_custom_data(struct ex_data_runtime *ex_data_rt,
const char *key, size_t key_len);

11
src/inc_internal/maat_expr.h
View File

@@ -44,14 +44,13 @@ int expr_runtime_commit(void *expr_runtime, const char *table_name);
*
* @retval the num of hit group_id
*/
int expr_runtime_scan(struct expr_runtime *expr_rt, int thread_id,
const char *data, size_t data_len,
int *group_ids, size_t group_ids_size,
int vtable_ids, struct maat_state *state);
int expr_runtime_scan(struct expr_runtime *expr_rt, int thread_id, const char *data,
size_t data_len, int *group_ids, int hit_group_index,
size_t group_ids_size, int vtable_ids, struct maat_state *state);
void expr_runtime_stream_open(struct expr_runtime *expr_rt, int thread_id);
int expr_runtime_stream_scan(struct expr_runtime *expr_rt, const char *data,
size_t data_len, int *group_ids, size_t group_ids_size,
int expr_runtime_stream_scan(struct expr_runtime *expr_rt, const char *data, size_t data_len,
int *group_ids, int hit_group_index, size_t group_ids_size,
int vtable_id, struct maat_state *state);
void expr_runtime_stream_close(struct expr_runtime *expr_rt);

4
src/inc_internal/maat_flag.h
View File

@@ -46,8 +46,8 @@ int flag_runtime_commit(void *flag_runtime, const char *table_name);
*
* @retval the num of hit group_id
*/
int flag_runtime_scan(struct flag_runtime *flag_rt, int thread_id,
uint64_t flag, int *group_ids, size_t group_ids_size,
int flag_runtime_scan(struct flag_runtime *flag_rt, int thread_id, uint64_t flag,
int *group_ids, int hit_group_index,size_t group_ids_size,
int vtable_id, struct maat_state *state);
void flag_runtime_scan_hit_inc(struct flag_runtime *flag_rt, int thread_id);

4
src/inc_internal/maat_interval.h
View File

@@ -47,8 +47,8 @@ int interval_runtime_commit(void *interval_runtime, const char *table_name);
* @retval the num of hit group_id
*/
int interval_runtime_scan(struct interval_runtime *interval_rt, int thread_id,
uint64_t integer, int *group_ids, size_t group_ids_size,
int vtable_id, struct maat_state *state);
uint64_t integer, int *group_ids, int hit_group_index,
size_t group_ids_size, int vtable_id, struct maat_state *state);
void interval_runtime_scan_hit_inc(struct interval_runtime *interval_rt, int thread_id);
long long interval_runtime_scan_hit_sum(struct interval_runtime *interval_rt, int n_thread);

4
src/inc_internal/maat_ip.h
View File

@@ -41,8 +41,8 @@ struct ex_data_runtime *ip_runtime_get_ex_data_rt(struct ip_runtime *ip_rt);
/* ip runtime scan API */
int ip_runtime_scan(struct ip_runtime *ip_rt, int thread_id, int ip_type,
uint8_t *ip_addr, int *group_ids, size_t group_id_size,
int vtable_id, struct maat_state *state);
uint8_t *ip_addr, int *group_ids, int hit_group_index,
size_t group_id_size, int vtable_id, struct maat_state *state);
void ip_runtime_scan_hit_inc(struct ip_runtime *ip_rt, int thread_id);
long long ip_runtime_scan_hit_sum(struct ip_runtime *ip_rt, int n_thread);

5
src/inc_internal/maat_ip_plugin.h
View File

@@ -32,7 +32,7 @@ void ip_plugin_schema_free(void *ip_plugin_schema);
/* ip plugin table ex data API */
struct ex_data_schema *ip_plugin_table_get_ex_data_schema(void *ip_plugin_schema);
int ip_plugin_table_set_ex_data_schema(void *ip_plugin_schema,
void ip_plugin_table_set_ex_data_schema(void *ip_plugin_schema,
maat_plugin_ex_new_func_t *new_func,
maat_plugin_ex_free_func_t *free_func,
maat_plugin_ex_dup_func_t *dup_func,
@@ -51,6 +51,9 @@ int ip_plugin_runtime_commit(void *ip_plugin_runtime, const char *table_name);
struct ex_data_runtime *ip_plugin_runtime_get_ex_data_rt(void *ip_plugin_runtime);
int ip_plugin_runtime_get_ex_data(void *ip_plugin_runtime, const struct ip_addr *ip_addr,
void **ex_data_array, size_t n_ex_data_array);
#ifdef __cplusplus
}
#endif

2
src/inc_internal/maat_plugin.h
View File

@@ -40,7 +40,7 @@ void plugin_table_all_callback_finish(struct plugin_schema *plugin_schema);
int plugin_table_get_foreign_column(struct plugin_schema *plugin_schema, int *foreign_columns);
/* plugin table ex data API */
int plugin_table_set_ex_data_schema(void *plugin_schema,
void plugin_table_set_ex_data_schema(void *plugin_schema,
maat_plugin_ex_new_func_t *new_func,
maat_plugin_ex_free_func_t *free_func,
maat_plugin_ex_dup_func_t *dup_func,

9
src/inc_internal/maat_rule.h
View File

@@ -36,6 +36,8 @@ extern "C"
#include "maat_virtual.h"
#define MAX_TABLE_NUM 256
#define MAX_COMPILE_TABLE_NUM 16
#define MAX_PHYSICAL_TABLE_NUM 16
#define MAAT_UPDATE_TYPE_NONE 0
#define MAAT_UPDATE_TYPE_FULL 1
@@ -60,10 +62,6 @@ enum tag_match {
struct maat_rule_head {
int config_id;
int service_id;
char do_log;
char do_blacklist;
char action;
char resevered;
int serv_def_len;
};
@@ -241,7 +239,8 @@ struct maat {
struct maat_state {
struct maat *maat_instance;
int16_t thread_id;
int compile_table_id; //caller can select compile table to scan
size_t n_compile_table;
char compile_tables[MAX_COMPILE_TABLE_NUM][NAME_MAX]; //caller can select compile table to scan
unsigned char is_set_district;
unsigned char is_last_scan;
int district_id; //-1: Any District; -2: Unkonwn District;

3
src/inc_internal/maat_virtual.h
View File

@@ -40,7 +40,8 @@ void *virtual_schema_new(cJSON *json, struct table_manager *tbl_mgr,
void virtual_schema_free(void *virtual_schema);
int virtual_table_get_physical_table_id(void *virtual_schema, enum scan_type type);
int virtual_table_get_id(void *virtual_schema);
size_t virtual_table_get_physical_table_id(void *virtual_schema, int physical_table_ids[]);
#ifdef __cplusplus
}

3
src/json2iris.c
View File

@@ -1024,11 +1024,12 @@ void write_table_idx(struct iris_description *p_iris, struct iris_table *table)
memcpy(buff, line_cnt_str, strlen(line_cnt_str));
memcpy(buff + strlen(line_cnt_str), table->buff, table->write_pos);
UNUSED int ret = 0;
FILE *table_fp = fopen(table->table_path, "w");
if (p_iris->encrypt_key) {
unsigned char *encrypt_buff = NULL;
size_t encrypt_buff_sz = 0;
int ret = crypt_memory(buff, table_file_sz, &encrypt_buff,
ret = crypt_memory(buff, table_file_sz, &encrypt_buff,
&encrypt_buff_sz, p_iris->encrypt_key,
p_iris->encrypt_algo, 1, err_str, sizeof(err_str));
assert(ret == 0);

701
src/maat_api.c
View File

File diff suppressed because it is too large Load Diff

133
src/maat_compile.c
View File

@@ -10,6 +10,7 @@
#include <assert.h>
#include <pthread.h>
#include <limits.h>
#include "maat_utils.h"
#include "log/log.h"
@@ -35,10 +36,6 @@ enum user_region_encode {
struct compile_schema {
int compile_id_column;
int service_id_column;
int action_column;
int do_blacklist_column;
int do_log_column;
int tags_column;
int user_region_column;
int clause_num_column;
@@ -64,10 +61,6 @@ struct group2compile_schema {
struct compile_item {
int compile_id;
int service_id;
int action;
int do_blacklist;
int do_log;
char user_region[MAX_TABLE_LINE_SIZE];
int clause_num;
int evaluation_order;
@@ -274,30 +267,6 @@ void *compile_schema_new(cJSON *json, struct table_manager *tbl_mgr,
read_cnt++;
}
custom_item = cJSON_GetObjectItem(item, "service_id");
if (custom_item != NULL && custom_item->type == cJSON_Number) {
compile_schema->service_id_column = custom_item->valueint;
read_cnt++;
}
custom_item = cJSON_GetObjectItem(item, "action");
if (custom_item != NULL && custom_item->type == cJSON_Number) {
compile_schema->action_column = custom_item->valueint;
read_cnt++;
}
custom_item = cJSON_GetObjectItem(item, "do_blacklist");
if (custom_item != NULL && custom_item->type == cJSON_Number) {
compile_schema->do_blacklist_column = custom_item->valueint;
read_cnt++;
}
custom_item = cJSON_GetObjectItem(item, "do_log");
if (custom_item != NULL && custom_item->type == cJSON_Number) {
compile_schema->do_log_column = custom_item->valueint;
read_cnt++;
}
custom_item = cJSON_GetObjectItem(item, "tags");
if (custom_item != NULL && custom_item->type == cJSON_Number) {
compile_schema->tags_column = custom_item->valueint;
@@ -324,7 +293,7 @@ void *compile_schema_new(cJSON *json, struct table_manager *tbl_mgr,
compile_schema->ref_tbl_mgr = tbl_mgr;
if (read_cnt < 10) {
if (read_cnt < 6) {
goto error;
}
@@ -438,46 +407,6 @@ compile_item_new(const char *line, struct compile_schema *compile_schema,
}
compile_item->compile_id = atoi(line + column_offset);
ret = get_column_pos(line, compile_schema->service_id_column,
&column_offset, &column_len);
if (ret < 0) {
log_error(logger, MODULE_COMPILE,
"compile table(table_id:%d) line:%s has no service_id",
compile_schema->table_id, line);
goto error;
}
compile_item->service_id = atoi(line + column_offset);
ret = get_column_pos(line, compile_schema->action_column,
&column_offset, &column_len);
if (ret < 0) {
log_error(logger, MODULE_COMPILE,
"compile table(table_id:%d) line:%s has no action",
compile_schema->table_id, line);
goto error;
}
compile_item->action = atoi(line + column_offset);
ret = get_column_pos(line, compile_schema->do_blacklist_column,
&column_offset, &column_len);
if (ret < 0) {
log_error(logger, MODULE_COMPILE,
"compile table(table_id:%d) line:%s has no do_blacklist",
compile_schema->table_id, line);
goto error;
}
compile_item->do_blacklist = atoi(line + column_offset);
ret = get_column_pos(line, compile_schema->do_log_column,
&column_offset, &column_len);
if (ret < 0) {
log_error(logger, MODULE_COMPILE,
"compile table(table_id:%d) line:%s has no do_log",
compile_schema->table_id, line);
goto error;
}
compile_item->do_log = atoi(line + column_offset);
ret = get_column_pos(line, compile_schema->tags_column,
&column_offset, &column_len);
if (ret < 0) {
@@ -1296,7 +1225,7 @@ static int maat_compile_has_literal(struct maat_compile* compile,
return 0;
}
static int maat_compile_is_hit_path_existed(const struct maat_hit_path *hit_paths,
static int maat_compile_is_hit_path_existed(const struct maat_hit_path *hit_paths,
size_t n_path, const struct maat_hit_path *find)
{
for (size_t i = 0; i < n_path; i++) {
@@ -1312,11 +1241,11 @@ size_t compile_runtime_get_hit_paths(struct compile_runtime *compile_rt,
struct group2group_runtime *g2g_rt,
struct maat_compile_state *compile_state,
struct maat_hit_path *hit_paths,
size_t hit_path_size)
size_t hit_path_index, size_t hit_path_size)
{
size_t i = 0, j = 0;
size_t i = 0, j = 0;
struct maat_internal_hit_path *internal_path = NULL;
size_t hit_path_cnt = 0;
size_t hit_path_cnt = hit_path_index;
size_t new_hit_path_cnt = 0;
for (i = 0; i < utarray_len(compile_state->internal_hit_paths); i++) {
@@ -1502,10 +1431,6 @@ void compile_item_to_compile_rule(struct compile_item *compile_item,
{
struct maat_rule_head rule_head;
rule_head.config_id = compile_item->compile_id;
rule_head.service_id = compile_item->service_id;
rule_head.action = compile_item->action;
rule_head.do_blacklist = compile_item->do_blacklist;
rule_head.do_log = compile_item->do_log;
compile_rule->magic_num = COMPILE_RULE_MAGIC;
compile_rule->head = rule_head;
@@ -1792,11 +1717,11 @@ int compile_runtime_match(struct compile_runtime *compile_rt,
int maat_compile_state_update(struct maat_item *item_hash, int vtable_id,
int *hit_item_ids, size_t hit_item_cnt,
int *group_ids, size_t group_ids_size,
int *group_ids, int hit_group_index, size_t group_ids_size,
size_t *n_hit_group_id, struct maat_state *state)
{
struct maat_item *item = NULL;
size_t hit_group_cnt = 0;
size_t hit_group_cnt = hit_group_index;
void *g2g_rt = table_manager_get_runtime(state->maat_instance->tbl_mgr,
state->maat_instance->g2g_table_id);
@@ -1827,31 +1752,29 @@ int maat_compile_state_update(struct maat_item *item_hash, int vtable_id,
*n_hit_group_id = hit_group_cnt;
/* update hit clause */
int compile_table_id = -1;
if (state->compile_table_id == -1) {
compile_table_id = state->maat_instance->default_compile_table_id;
int compile_table_ids[MAX_COMPILE_TABLE_NUM] = {0};
size_t compile_table_cnt = 0;
if (0 == state->n_compile_table) {
compile_table_ids[0] = state->maat_instance->default_compile_table_id;
compile_table_cnt = 1;
} else {
compile_table_id = state->compile_table_id;
for (size_t i = 0; i < state->n_compile_table; i++) {
compile_table_ids[i] = maat_table_get_id(state->maat_instance, state->compile_tables[i]);
}
compile_table_cnt = state->n_compile_table;
}
void *compile_rt = table_manager_get_runtime(state->maat_instance->tbl_mgr,
compile_table_id);
enum table_type table_type = table_manager_get_table_type(state->maat_instance->tbl_mgr,
compile_table_id);
assert(table_type == TABLE_TYPE_COMPILE);
for (size_t i = 0; i < hit_group_cnt; i++) {
int top_group_ids[MAX_SCANNER_HIT_GROUP_NUM] = {-1};
int top_group_cnt = group2group_runtime_get_top_groups(g2g_rt, &group_ids[i],
1, top_group_ids);
// if (0 == top_group_cnt) {
// maat_compile_state_update_hit_clause(state->compile_state, compile_rt,
// group_ids[i], vtable_id);
// }
for (int j = 0; j < top_group_cnt; j++) {
maat_compile_state_update_hit_clause(state->compile_state, compile_rt,
top_group_ids[j], vtable_id);
for (size_t idx = 0; idx < compile_table_cnt; idx++) {
void *compile_rt = table_manager_get_runtime(state->maat_instance->tbl_mgr,
compile_table_ids[idx]);
for (size_t i = 0; i < hit_group_cnt; i++) {
int top_group_ids[MAX_SCANNER_HIT_GROUP_NUM] = {-1};
int top_group_cnt = group2group_runtime_get_top_groups(g2g_rt, &group_ids[i],
1, top_group_ids);
for (int j = 0; j < top_group_cnt; j++) {
maat_compile_state_update_hit_clause(state->compile_state, compile_rt,
top_group_ids[j], vtable_id);
}
}
}

15
src/maat_ex_data.c
View File

@@ -250,8 +250,8 @@ int ex_data_runtime_del_ex_container(struct ex_data_runtime *ex_data_rt,
return 0;
}
void *ex_data_runtime_get_ex_data(struct ex_data_runtime *ex_data_rt,
const char *key, size_t key_len)
void *ex_data_runtime_get_ex_data_by_key(struct ex_data_runtime *ex_data_rt,
const char *key, size_t key_len)
{
struct ex_data_container *ex_container = NULL;
ex_container = (struct ex_data_container *)rcu_hash_find(ex_data_rt->htable,
@@ -268,6 +268,17 @@ void *ex_data_runtime_get_ex_data(struct ex_data_runtime *ex_data_rt,
return dup_ex_data;
}
void *ex_data_runtime_get_ex_data_by_container(struct ex_data_runtime *ex_data_rt,
struct ex_data_container *ex_container)
{
void *dup_ex_data = NULL;
ex_data_rt->ex_schema->dup_func(ex_data_rt->table_id, &dup_ex_data,
&(ex_container->ex_data),
ex_data_rt->ex_schema->argl,
ex_data_rt->ex_schema->argp);
return dup_ex_data;
}
void *ex_data_runtime_get_custom_data(struct ex_data_runtime *ex_data_rt,
const char *key, size_t key_len)
{

21
src/maat_expr.c
View File

@@ -840,10 +840,9 @@ int expr_runtime_commit(void *expr_runtime, const char *table_name)
return ret;
}
int expr_runtime_scan(struct expr_runtime *expr_rt, int thread_id,
const char *data, size_t data_len,
int *group_ids, size_t group_ids_size,
int vtable_id, struct maat_state *state)
int expr_runtime_scan(struct expr_runtime *expr_rt, int thread_id, const char *data,
size_t data_len, int *group_ids, int hit_group_index,
size_t group_ids_size, int vtable_ids, struct maat_state *state)
{
size_t n_hit_item = 0;
struct hs_scan_result hit_results[MAX_SCANNER_HIT_ITEM_NUM] = {0};
@@ -881,9 +880,9 @@ int expr_runtime_scan(struct expr_runtime *expr_rt, int thread_id,
}
size_t group_hit_cnt = 0;
ret = maat_compile_state_update(expr_rt->item_hash, vtable_id, hit_item_ids,
real_hit_item_cnt, group_ids, group_ids_size,
&group_hit_cnt, state);
ret = maat_compile_state_update(expr_rt->item_hash, vtable_ids, hit_item_ids,
real_hit_item_cnt, group_ids, hit_group_index,
group_ids_size, &group_hit_cnt, state);
if (ret < 0) {
return -1;
}
@@ -901,8 +900,8 @@ void expr_runtime_stream_open(struct expr_runtime *expr_rt, int thread_id)
expr_rt->hs_stream = hs_stream;
}
int expr_runtime_stream_scan(struct expr_runtime *expr_rt, const char *data,
size_t data_len, int *group_ids, size_t group_ids_size,
int expr_runtime_stream_scan(struct expr_runtime *expr_rt, const char *data, size_t data_len,
int *group_ids, int hit_group_index, size_t group_ids_size,
int vtable_id, struct maat_state *state)
{
if (NULL == expr_rt) {
@@ -936,8 +935,8 @@ int expr_runtime_stream_scan(struct expr_runtime *expr_rt, const char *data,
size_t group_hit_cnt = 0;
ret = maat_compile_state_update(expr_rt->item_hash, vtable_id, hit_item_ids,
n_hit_item, group_ids, group_ids_size,
&group_hit_cnt, state);
n_hit_item, group_ids, hit_group_index,
group_ids_size, &group_hit_cnt, state);
if (ret < 0) {
return -1;
}

6
src/maat_flag.c
View File

@@ -411,8 +411,8 @@ int flag_runtime_commit(void *flag_runtime, const char *table_name)
}
int flag_runtime_scan(struct flag_runtime *flag_rt, int thread_id,
uint64_t flag, int *group_ids, size_t group_ids_size,
int vtable_id, struct maat_state *state)
uint64_t flag, int *group_ids, int hit_group_index,
size_t group_ids_size, int vtable_id, struct maat_state *state)
{
struct flag_result hit_results[MAX_SCANNER_HIT_ITEM_NUM] = {0};
@@ -438,7 +438,7 @@ int flag_runtime_scan(struct flag_runtime *flag_rt, int thread_id,
size_t group_hit_cnt = 0;
int ret = maat_compile_state_update(flag_rt->item_hash, vtable_id, hit_item_ids,
n_hit_item, group_ids, group_ids_size,
n_hit_item, group_ids, hit_group_index, group_ids_size,
&group_hit_cnt, state);
if (ret < 0) {
return -1;

7
src/maat_interval.c
View File

@@ -440,9 +440,8 @@ int interval_runtime_commit(void *interval_runtime, const char *table_name)
}
int interval_runtime_scan(struct interval_runtime *interval_rt, int thread_id,
uint64_t integer,
int *group_ids, size_t group_ids_size,
int vtable_id, struct maat_state *state)
uint64_t integer, int *group_ids, int hit_group_index,
size_t group_ids_size, int vtable_id, struct maat_state *state)
{
struct interval_result hit_results[MAX_SCANNER_HIT_ITEM_NUM] = {0};
@@ -475,7 +474,7 @@ int interval_runtime_scan(struct interval_runtime *interval_rt, int thread_id,
size_t group_hit_cnt = 0;
int ret = maat_compile_state_update(interval_rt->item_hash, vtable_id, hit_item_ids,
n_hit_item, group_ids, group_ids_size,
n_hit_item, group_ids, hit_group_index, group_ids_size,
&group_hit_cnt, state);
if (ret < 0) {
return -1;

26
src/maat_ip.c
View File

@@ -453,17 +453,15 @@ int ip_runtime_commit(void *ip_runtime, const char *table_name)
struct ip_matcher *old_ip_matcher = NULL;
size_t mem_used = 0;
if (rule_cnt > 0) {
log_info(ip_rt->logger, MODULE_IP,
"table[%s] committing %zu ip rules for rebuilding ip_matcher engine",
table_name, rule_cnt);
new_ip_matcher = ip_matcher_new(rules, rule_cnt, &mem_used);
if (NULL == new_ip_matcher) {
log_error(ip_rt->logger, MODULE_IP,
"table[%s] rebuild ip_matcher engine failed when update %zu ip rules",
table_name, rule_cnt);
ret = -1;
}
log_info(ip_rt->logger, MODULE_IP,
"table[%s] committing %zu ip rules for rebuilding ip_matcher engine",
table_name, rule_cnt);
new_ip_matcher = ip_matcher_new(rules, rule_cnt, &mem_used);
if (NULL == new_ip_matcher) {
log_error(ip_rt->logger, MODULE_IP,
"table[%s] rebuild ip_matcher engine failed when update %zu ip rules",
table_name, rule_cnt);
ret = -1;
}
old_ip_matcher = ip_rt->ip_matcher;
@@ -479,8 +477,8 @@ int ip_runtime_commit(void *ip_runtime, const char *table_name)
}
int ip_runtime_scan(struct ip_runtime *ip_rt, int thread_id, int ip_type,
uint8_t *ip_addr, int *group_ids, size_t group_id_size,
int vtable_id, struct maat_state *state)
uint8_t *ip_addr, int *group_ids, int hit_group_index,
size_t group_id_size, int vtable_id, struct maat_state *state)
{
int n_hit_item = 0;
struct scan_result scan_results[MAX_SCANNER_HIT_ITEM_NUM] = {0};
@@ -518,7 +516,7 @@ int ip_runtime_scan(struct ip_runtime *ip_rt, int thread_id, int ip_type,
size_t group_hit_cnt = 0;
int ret = maat_compile_state_update(ip_rt->item_hash, vtable_id, hit_item_ids, n_hit_item,
group_ids, group_id_size, &group_hit_cnt, state);
group_ids, hit_group_index, group_id_size, &group_hit_cnt, state);
if (ret < 0) {
return -1;
}

163
src/maat_ip_plugin.c
View File

@@ -16,16 +16,29 @@
#include "maat_ex_data.h"
#include "IPMatcher.h"
#include "maat_rule.h"
#include "maat.h"
#include "maat_garbage_collection.h"
#define MODULE_IP_PLUGIN module_name_str("maat.ip_plugin")
#define MAX_IP_STR 128
struct ipv4_item_rule {
uint32_t min_sip; /* 源地址下界；0表示忽略本字段 */
uint32_t max_sip; /* 源地址上界；0表示固定IP=min_saddr */
};
struct ipv6_item_rule {
uint32_t min_sip[4]; /* 源地址下界；全0表示忽略本字段 */
uint32_t max_sip[4]; /* 源地址上界；全0表示固定IP=min_saddr */
};
struct ip_plugin_item {
int item_id;
int ip_type;
char start_ip[MAX_IP_STR];
char end_ip[MAX_IP_STR];
union {
struct ipv4_item_rule ipv4;
struct ipv6_item_rule ipv6;
};
int rule_tag;
};
@@ -34,6 +47,7 @@ struct ip_plugin_schema {
int ip_type_column;
int start_ip_column;
int end_ip_column;
int addr_format_column;
int rule_tag_column;
struct ex_data_schema *ex_schema;
int table_id; //ugly
@@ -98,6 +112,12 @@ void *ip_plugin_schema_new(cJSON *json, struct table_manager *tbl_mgr,
read_cnt++;
}
custom_item = cJSON_GetObjectItem(item, "addr_format");
if (custom_item != NULL && custom_item->type == cJSON_Number) {
schema->addr_format_column = custom_item->valueint;
read_cnt++;
}
// rule_tag is optional
custom_item = cJSON_GetObjectItem(item, "rule_tag");
if (custom_item != NULL && custom_item->type == cJSON_Number) {
@@ -106,7 +126,7 @@ void *ip_plugin_schema_new(cJSON *json, struct table_manager *tbl_mgr,
schema->ref_tbl_mgr = tbl_mgr;
if (read_cnt < 5) {
if (read_cnt < 6) {
goto error;
}
@@ -193,7 +213,11 @@ ip_plugin_item_new(const char *line, struct ip_plugin_schema *schema,
size_t column_offset = 0;
size_t column_len = 0;
char addr_format[16] = {0};
char start_ip_str[40] = {0};
char end_ip_str[40] = {0};
struct ip_plugin_item *ip_plugin_item = ALLOC(struct ip_plugin_item, 1);
ret = get_column_pos(line, schema->item_id_column, &column_offset, &column_len);
if (ret < 0) {
log_error(logger, MODULE_IP_PLUGIN,
@@ -211,13 +235,29 @@ ip_plugin_item_new(const char *line, struct ip_plugin_schema *schema,
goto error;
}
ip_plugin_item->ip_type = atoi(line + column_offset);
if (ip_plugin_item->ip_type != 4 && ip_plugin_item->ip_type != 6) {
if (ip_plugin_item->ip_type != IPv4 && ip_plugin_item->ip_type != IPv6) {
log_error(logger, MODULE_IP_PLUGIN,
"ip_plugin table(table_id:%d) line:%s ip_type[%d] invalid",
schema->table_id, line, ip_plugin_item->ip_type);
goto error;
}
ret = get_column_pos(line, schema->addr_format_column, &column_offset, &column_len);
if (ret < 0) {
log_error(logger, MODULE_IP_PLUGIN,
"ip_plugin table(table_id:%d) line:%s has no addr_format column",
schema->table_id, line);
goto error;
}
memcpy(addr_format, (line + column_offset), column_len);
if (IP_FORMAT_UNKNOWN == ip_format_str2int(addr_format)) {
log_error(logger, MODULE_IP_PLUGIN,
"ip_plugin table(table_id:%d) line:%s has invalid addr_format, should be range/CIDR",
schema->table_id, line);
goto error;
}
ret = get_column_pos(line, schema->start_ip_column, &column_offset, &column_len);
if (ret < 0) {
log_error(logger, MODULE_IP_PLUGIN,
@@ -225,8 +265,7 @@ ip_plugin_item_new(const char *line, struct ip_plugin_schema *schema,
schema->table_id, line);
goto error;
}
strncpy(ip_plugin_item->start_ip, line + column_offset,
MIN(column_len, sizeof(ip_plugin_item->start_ip)));
strncpy(start_ip_str, line + column_offset, column_len);
ret = get_column_pos(line, schema->end_ip_column, &column_offset, &column_len);
if (ret < 0) {
@@ -235,8 +274,28 @@ ip_plugin_item_new(const char *line, struct ip_plugin_schema *schema,
schema->table_id, line);
goto error;
}
strncpy(ip_plugin_item->end_ip, line + column_offset,
MIN(column_len, sizeof(ip_plugin_item->end_ip)));
strncpy(end_ip_str, line + column_offset, column_len);
if (IPv4 == ip_plugin_item->ip_type) {
ret = ip_format2range(ip_plugin_item->ip_type, ip_format_str2int(addr_format), start_ip_str, end_ip_str,
&ip_plugin_item->ipv4.min_sip, &ip_plugin_item->ipv4.max_sip);
if (ret < 0) {
log_error(logger, MODULE_IP_PLUGIN,
"ip_plugin table(table_id:%d) line:%s ip_format2range(ip4) failed",
schema->table_id, line);
goto error;
}
} else {
//ipv6
ret = ip_format2range(ip_plugin_item->ip_type, ip_format_str2int(addr_format), start_ip_str, end_ip_str,
ip_plugin_item->ipv6.min_sip, ip_plugin_item->ipv6.max_sip);
if (ret < 0) {
log_error(logger, MODULE_IP_PLUGIN,
"ip_plugin table(table_id:%d) line:%s ip_format2range(ip6) failed",
schema->table_id, line);
goto error;
}
}
return ip_plugin_item;
error:
@@ -249,33 +308,16 @@ void ip_plugin_item_free(struct ip_plugin_item *item)
FREE(item);
}
int ip_plugin_table_ex_data_schema_flag(struct ip_plugin_schema *ip_plugin_schema)
{
return 0;
}
int ip_plugin_table_set_ex_data_schema(void *ip_plugin_schema,
void ip_plugin_table_set_ex_data_schema(void *ip_plugin_schema,
maat_plugin_ex_new_func_t *new_func,
maat_plugin_ex_free_func_t *free_func,
maat_plugin_ex_dup_func_t *dup_func,
long argl, void *argp,
struct log_handle *logger)
{
if (NULL == ip_plugin_schema) {
return -1;
}
struct ip_plugin_schema *schema = (struct ip_plugin_schema *)ip_plugin_schema;
if (schema->ex_schema != NULL) {
assert(0);
log_error(logger, MODULE_IP_PLUGIN,
"Error: %s, EX data schema already registed", __FUNCTION__);
return -1;
}
schema->ex_schema = ex_data_schema_new(new_func, free_func, dup_func, argl, argp);
return 0;
}
int ip_plugin_runtime_update_row(struct ip_plugin_runtime *rt, struct ip_plugin_schema *schema,
@@ -284,9 +326,9 @@ int ip_plugin_runtime_update_row(struct ip_plugin_runtime *rt, struct ip_plugin_
{
int ret = -1;
struct ex_data_runtime *ex_data_rt = rt->ex_data_rt;
int set_flag = ip_plugin_table_ex_data_schema_flag(schema);
struct ex_data_schema *ex_schema = schema->ex_schema;
if (1 == set_flag) {
if (ex_schema != NULL) {
if (0 == is_valid) {
//delete
ret = ex_data_runtime_del_ex_container(ex_data_rt, key, key_len);
@@ -350,14 +392,16 @@ void ip_plugin_runtime_free(void *ip_plugin_runtime)
void ip_plugin_item_to_ip_rule(struct ip_plugin_item *item, struct ip_rule *rule)
{
if (4 == item->ip_type) {
if (IPv4 == item->ip_type) {
rule->type = IPv4;
ip_format2range(item->ip_type, IP_FORMAT_RANGE, item->start_ip, item->end_ip,
&(rule->ipv4_rule.start_ip), &(rule->ipv4_rule.end_ip));
rule->ipv4_rule.start_ip = item->ipv4.min_sip;
rule->ipv4_rule.end_ip = item->ipv4.max_sip;
} else {
rule->type = IPv6;
ip_format2range(item->ip_type, IP_FORMAT_RANGE, item->start_ip, item->end_ip,
rule->ipv6_rule.start_ip, rule->ipv6_rule.end_ip);
memcpy(rule->ipv6_rule.start_ip, item->ipv6.min_sip,
sizeof(item->ipv6.min_sip));
memcpy(rule->ipv6_rule.end_ip, item->ipv6.max_sip,
sizeof(item->ipv6.max_sip));
}
rule->rule_id = item->item_id;
@@ -436,23 +480,22 @@ int ip_plugin_runtime_commit(void *ip_plugin_runtime, const char *table_name)
for (size_t i = 0; i < rule_cnt; i++) {
struct ip_plugin_item *item = (struct ip_plugin_item *)ex_container[i]->custom_data;
ip_plugin_item_to_ip_rule(item, &rules[i]);
rules[i].user_tag = ex_container[i];
}
struct ip_matcher *new_ip_matcher = NULL;
struct ip_matcher *old_ip_matcher = NULL;
size_t mem_used = 0;
if (rule_cnt > 0) {
log_info(ip_plugin_rt->logger, MODULE_IP_PLUGIN,
"table[%s] committing %zu ip_plugin rules for rebuilding ip_matcher engine",
table_name, rule_cnt);
new_ip_matcher = ip_matcher_new(rules, rule_cnt, &mem_used);
if (NULL == new_ip_matcher) {
log_error(ip_plugin_rt->logger, MODULE_IP_PLUGIN,
"table[%s] rebuild ip_matcher engine failed when update %zu ip_plugin rules",
table_name, rule_cnt);
ret = -1;
}
log_info(ip_plugin_rt->logger, MODULE_IP_PLUGIN,
"table[%s] committing %zu ip_plugin rules for rebuilding ip_matcher engine",
table_name, rule_cnt);
new_ip_matcher = ip_matcher_new(rules, rule_cnt, &mem_used);
if (NULL == new_ip_matcher) {
log_error(ip_plugin_rt->logger, MODULE_IP_PLUGIN,
"table[%s] rebuild ip_matcher engine failed when update %zu ip_plugin rules",
table_name, rule_cnt);
ret = -1;
}
old_ip_matcher = ip_plugin_rt->ip_matcher;
@@ -476,4 +519,36 @@ struct ex_data_runtime *ip_plugin_runtime_get_ex_data_rt(void *ip_plugin_runtime
struct ip_plugin_runtime *ip_plugin_rt = (struct ip_plugin_runtime *)ip_plugin_runtime;
return ip_plugin_rt->ex_data_rt;
}
int ip_plugin_runtime_get_ex_data(void *ip_plugin_runtime, const struct ip_addr *ip_addr,
void **ex_data_array, size_t n_ex_data)
{
if (NULL == ip_plugin_runtime) {
return -1;
}
struct ip_plugin_runtime *ip_plugin_rt = (struct ip_plugin_runtime *)ip_plugin_runtime;
if (NULL == ip_plugin_rt->ip_matcher) {
log_info(ip_plugin_rt->logger, MODULE_IP_PLUGIN,
"ip_matcher is NULL, can't get ex data");
return 0;
}
struct scan_result results[n_ex_data];
memset(results, 0, sizeof(results));
struct ip_data ip_data = *(const struct ip_data *)ip_addr;
if (ip_data.type == IPv4) {
ip_data.ipv4 = ntohl(ip_data.ipv4);
} else {
ipv6_ntoh(ip_data.ipv6);
}
int n_result = ip_matcher_match(ip_plugin_rt->ip_matcher, &ip_data, results, n_ex_data);
for (int i = 0; i < n_result; i++) {
ex_data_array[i] = ex_data_runtime_get_ex_data_by_container(ip_plugin_rt->ex_data_rt,
(struct ex_data_container *)results[i].tag);
}
return n_result;
}

16
src/maat_plugin.c
View File

@@ -205,28 +205,16 @@ int plugin_table_get_foreign_column(struct plugin_schema *plugin_schema,
return n_foreign;
}
int plugin_table_set_ex_data_schema(void *plugin_schema,
void plugin_table_set_ex_data_schema(void *plugin_schema,
maat_plugin_ex_new_func_t *new_func,
maat_plugin_ex_free_func_t *free_func,
maat_plugin_ex_dup_func_t *dup_func,
long argl, void *argp,
struct log_handle *logger)
{
if (NULL == plugin_schema) {
return -1;
}
struct plugin_schema *schema = (struct plugin_schema *)plugin_schema;
if (schema->ex_schema != NULL) {
assert(0);
log_error(logger, MODULE_PLUGIN,
"Error: %s, EX data schema already registed",
__FUNCTION__);
return -1;
}
schema->ex_schema = ex_data_schema_new(new_func, free_func, dup_func, argl, argp);
return 0;
}
struct ex_data_schema *plugin_table_get_ex_data_schema(void *plugin_schema)

2
src/maat_rule.c
View File

@@ -418,8 +418,6 @@ void *rule_monitor_loop(void *arg)
while (maat_instance->is_running) {
usleep(maat_instance->rule_update_checking_interval_ms * 1000);
if (0 == pthread_mutex_trylock(&(maat_instance->background_update_mutex))) {
log_info(maat_instance->logger, MODULE_MAAT_RULE,
"rule_monitor_loop.................%d", maat_instance->rule_update_checking_interval_ms * 1000);
switch (maat_instance->input_mode) {
case DATA_SOURCE_REDIS:
redis_monitor_traverse(maat_instance->maat_version,

2
src/maat_table.c
View File

@@ -325,6 +325,7 @@ struct maat_table *maat_table_new(cJSON *json, struct maat_kv_store *reserved_wo
item = cJSON_GetObjectItem(json, "table_type");
if (NULL == item || item->type != cJSON_String) {
log_error(logger, MODULE_TABLE, "table:%s has no table_type column", ptable->table_name);
goto error;
}
@@ -337,6 +338,7 @@ struct maat_table *maat_table_new(cJSON *json, struct maat_kv_store *reserved_wo
item = cJSON_GetObjectItem(json, "valid_column");
if (NULL == item || item->type != cJSON_Number) {
if (ptable->table_type != TABLE_TYPE_VIRTUAL) {
log_error(logger, MODULE_TABLE, "table:%s has no valid column", ptable->table_name);
goto error;
}
} else {

63
src/maat_virtual.c
View File

@@ -20,7 +20,8 @@
#define MODULE_VIRTUAL module_name_str("maat.virtual")
struct virtual_schema {
int physical_table_id[SCAN_TYPE_MAX];
char physical_tables[MAX_PHYSICAL_TABLE_NUM][NAME_MAX];
size_t n_physical_table;
int table_id;
struct table_manager *ref_tbl_mgr;
};
@@ -28,38 +29,38 @@ struct virtual_schema {
void *virtual_schema_new(cJSON *json, struct table_manager *tbl_mgr,
const char *table_name, struct log_handle *logger)
{
cJSON *item = cJSON_GetObjectItem(json, "physical_table");
struct virtual_schema *schema = ALLOC(struct virtual_schema, 1);
schema->ref_tbl_mgr = tbl_mgr;
cJSON *item = cJSON_GetObjectItem(json, "table_id");
if (NULL == item || item->type != cJSON_Number) {
log_error(logger, MODULE_VIRTUAL,
"virtual table %s has no table_id column", table_name);
goto error;
}
schema->table_id = item->valueint;
item = cJSON_GetObjectItem(json, "physical_table");
if (NULL == item || item->type != cJSON_Array) {
log_error(logger, MODULE_VIRTUAL,
"virtual table %s has no physical_table column", table_name);
return NULL;
goto error;
}
struct virtual_schema *vt_schema = ALLOC(struct virtual_schema, 1);
vt_schema->ref_tbl_mgr = tbl_mgr;
int cnt = cJSON_GetArraySize(item);
for (int i = 0; i < cnt; i++) {
cJSON *tmp_item = cJSON_GetArrayItem(item, i);
if (tmp_item != NULL && tmp_item->type == cJSON_String) {
int table_id = table_manager_get_table_id(tbl_mgr, tmp_item->valuestring);
/* physical table should already exist */
if (table_id < 0) {
log_error(logger, MODULE_VIRTUAL, "table:%s is not registered",
tmp_item->valuestring);
FREE(vt_schema);
return NULL;
}
enum table_type table_type = table_manager_get_table_type(tbl_mgr, table_id);
assert(table_type != TABLE_TYPE_INVALID);
enum scan_type scan_type = maat_table_get_scan_type(table_type);
assert(scan_type != SCAN_TYPE_INVALID);
vt_schema->physical_table_id[scan_type]= table_id;
memcpy(schema->physical_tables[i], tmp_item->valuestring,
strlen(tmp_item->valuestring));
}
}
schema->n_physical_table = cnt;
return vt_schema;
return schema;
error:
FREE(schema);
return NULL;
}
void virtual_schema_free(void *virtual_schema)
@@ -67,12 +68,28 @@ void virtual_schema_free(void *virtual_schema)
FREE(virtual_schema);
}
int virtual_table_get_physical_table_id(void *virtual_schema, enum scan_type type)
int virtual_table_get_id(void *virtual_schema)
{
if (NULL == virtual_schema) {
return -1;
}
struct virtual_schema *schema = (struct virtual_schema *)virtual_schema;
return schema->physical_table_id[type];
return schema->table_id;
}
size_t virtual_table_get_physical_table_id(void *virtual_schema, int physical_table_ids[])
{
if (NULL == virtual_schema) {
return -1;
}
struct virtual_schema *schema = (struct virtual_schema *)virtual_schema;
for (size_t i = 0; i < schema->n_physical_table; i++) {
int table_id = table_manager_get_table_id(schema->ref_tbl_mgr, schema->physical_tables[i]);
physical_table_ids[i] = table_id;
}
return schema->n_physical_table;
}