gfwleak/tango-maat
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

delete set_iris evaluation_order

Browse Source
This commit is contained in:
liuwentan
2023-04-03 19:01:26 +08:00
parent 3ed1f3dfd4
commit 3515b7f1f7
20 changed files with 107 additions and 211 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
src/CMakeLists.txt
View File

@@ -13,7 +13,7 @@ add_definitions(-D_GNU_SOURCE)
add_definitions(-fPIC)
set(MAAT_SRC alignment.c json2iris.c maat_api.c rcu_hash.c maat_garbage_collection.c maat_config_monitor.c
maat_rule.c maat_kv.c maat_ex_data.c maat_utils.c maat_command.c maat_redis_monitor.c maat_table.c
maat_compile.c maat_group.c maat_ip.c maat_flag.c maat_interval.c maat_expr.c maat_fqdn.c maat_plugin.c
maat_compile.c maat_group.c maat_ip.c maat_flag.c maat_interval.c maat_expr.c maat_plugin.c
maat_ip_plugin.c maat_bool_plugin.c maat_fqdn_plugin.c maat_virtual.c)
set(LIB_SOURCE_FILES

1
src/inc_internal/maat_ex_data.h
View File

@@ -35,7 +35,6 @@ struct ex_data_schema {
struct ex_container_schema {
int table_id;
void (*custom_data_free)(void *custom_data);
struct ex_data_schema *ex_schema;
};

54
src/inc_internal/maat_fqdn.h
View File

@@ -1,54 +0,0 @@
/*
**********************************************************************************************
* File: maat_fqdn.h
* Description:
* Authors: Liu WenTan <liuwentan@geedgenetworks.com>
* Date: 2022-10-31
* Copyright: (c) 2018-2022 Geedge Networks, Inc. All rights reserved.
***********************************************************************************************
*/
#ifndef _MAAT_FQDN_H_
#define _MAAT_FQDN_H_
#ifdef __cplusplus
extern "C"
{
#endif
#include "maat_rule.h"
#include "cJSON/cJSON.h"
#include "maat_table.h"
struct fqdn_runtime;
void *fqdn_schema_new(cJSON *json, struct table_manager *tbl_mgr,
const char *table_name, struct log_handle *logger);
void fqdn_schema_free(void *fqdn_schema);
/* fqdn runtime API */
void *fqdn_runtime_new(void *fqdn_schema, int max_thread_num,
struct maat_garbage_bin *garbage_bin,
struct log_handle *logger);
void fqdn_runtime_free(void *fqdn_runtime);
int fqdn_runtime_update(void *fqdn_runtime, void *fqdn_schema,
const char *table_name, const char *line,
int valid_column);
int fqdn_runtime_commit(void *fqdn_runtime, const char *table_name);
struct ex_data_runtime *fqdn_runtime_get_ex_data_rt(struct fqdn_runtime *fqdn_rt);
/* fqdn runtime scan API */
int fqdn_runtime_scan(struct fqdn_runtime *fqdn_rt, int thread_id, int port,
int proto, int *group_ids, size_t group_id_size,
int vtable_id, struct maat_state *state);
void fqdn_runtime_scan_hit_inc(struct fqdn_runtime *fqdn_rt, int thread_id);
long long fqdn_runtime_scan_hit_sum(struct fqdn_runtime *fqdn_rt, int n_thread);
#ifdef __cplusplus
}
#endif
#endif

2
src/inc_internal/maat_rule.h
View File

@@ -78,10 +78,8 @@ struct compile_rule {
char *table_line;
size_t table_line_len;
int declared_clause_num;
double evaluation_order;
struct compile_schema *ref_table;
void **ex_data;
pthread_rwlock_t rwlock;
};
struct group2group_rule {

6
src/json2iris.c
View File

@@ -896,12 +896,6 @@ int write_compile_line(cJSON *compile, struct iris_description *p_iris,
compile_cmd[cmd_cnt].json_type = cJSON_Number;
cmd_cnt++;
compile_cmd[cmd_cnt].json_string = "evaluation_order";
compile_cmd[cmd_cnt].json_type = cJSON_String;
compile_cmd[cmd_cnt].empty_allowed = 1;
compile_cmd[cmd_cnt].default_string = "0.0";
cmd_cnt++;
struct iris_table *table_info = NULL;
item = cJSON_GetObjectItem(compile,"compile_table_name");
if (NULL == item || item->type != cJSON_String) {

51
src/maat_compile.c
View File

@@ -54,7 +54,6 @@ struct group2compile_schema {
struct compile_item {
long long compile_id;
int declared_clause_num;
double evaluation_order;
};
struct group2compile_item {
@@ -108,7 +107,6 @@ struct maat_clause {
};
struct compile_sort_para {
double evaluation_order;
int declared_clause_num;
long long compile_id;
void *user;
@@ -121,7 +119,6 @@ struct maat_compile {
char table_name[NAME_MAX];
int actual_clause_num;
int declared_clause_num;
double evaluation_order;
int not_clause_cnt;
void *user_data;
void (*user_data_free)(void *);
@@ -299,6 +296,12 @@ error:
void compile_schema_free(void *compile_schema)
{
struct compile_schema *schema = (struct compile_schema *)compile_schema;
if (schema->ex_schema != NULL) {
FREE(schema->ex_schema);
}
FREE(compile_schema);
}
@@ -759,8 +762,8 @@ struct maat_compile *maat_compile_new(long long compile_id)
}
int maat_compile_set(struct maat_compile *compile, const char *table_name,
double evaluation_order, int declared_clause_num,
void *user_data, void (*user_data_free)(void *))
int declared_clause_num, void *user_data,
void (*user_data_free)(void *))
{
if (user_data != NULL && NULL == user_data_free) {
return -1;
@@ -768,7 +771,6 @@ int maat_compile_set(struct maat_compile *compile, const char *table_name,
memset(compile->table_name, 0, sizeof(compile->table_name));
memcpy(compile->table_name, table_name, sizeof(compile->table_name));
compile->evaluation_order = evaluation_order;
compile->declared_clause_num = declared_clause_num;
compile->user_data = user_data;
compile->user_data_free = user_data_free;
@@ -813,10 +815,8 @@ void maat_compile_hash_set(struct maat_compile **compile_hash, long long compile
assert(tmp_compile != NULL);
assert(tmp_compile->user_data == NULL);
maat_compile_set(tmp_compile, table_name, compile->evaluation_order,
compile->declared_clause_num, compile->user_data,
compile->user_data_free);
maat_compile_set(tmp_compile, table_name, compile->declared_clause_num,
compile->user_data, compile->user_data_free);
}
int maat_compile_hash_remove(struct maat_compile **compile_hash, struct maat_compile *compile,
@@ -1443,12 +1443,6 @@ void maat_compile_state_update_hit_clause(struct maat_compile_state *compile_sta
utarray_push_back(compile_state->all_hit_clauses, clause_id);
}
utarray_sort(compile_state->all_hit_clauses, compare_clause_id);
// printf("<update_hit_clause> all_hit_clause:");
// for (i = 0; i < utarray_len(compile_state->all_hit_clauses); i++) {
// long long *tmp_clause_id = (long long *)utarray_eltptr(compile_state->all_hit_clauses, i);
// printf(" %lld ", *tmp_clause_id);
// }
// printf("\n");
}
}
}
@@ -1467,7 +1461,6 @@ void compile_item_to_compile_rule(struct compile_item *compile_item,
const char *table_line)
{
compile_rule->magic_num = COMPILE_RULE_MAGIC;
compile_rule->evaluation_order = compile_item->evaluation_order;
compile_rule->declared_clause_num = compile_item->declared_clause_num;
compile_rule->ref_table = compile_schema;
compile_rule->ex_data = ALLOC(void *, 1);
@@ -1482,10 +1475,9 @@ void compile_item_to_compile_rule(struct compile_item *compile_item,
}
compile_rule->compile_id = compile_item->compile_id;
pthread_rwlock_init(&compile_rule->rwlock, NULL);
}
void destroy_compile_rule(struct compile_rule *compile_rule)
void compile_rule_free(struct compile_rule *compile_rule)
{
struct compile_schema *schema = compile_rule->ref_table;
assert(compile_rule->magic_num==COMPILE_RULE_MAGIC);
@@ -1595,7 +1587,7 @@ int compile_runtime_update(void *compile_runtime, void *compile_schema,
compile = maat_compile_new(compile_rule->compile_id);
if (NULL == compile) {
destroy_compile_rule(compile_rule);
compile_rule_free(compile_rule);
pthread_rwlock_unlock(&compile_rt->rwlock);
log_error(compile_rt->logger, MODULE_COMPILE,
"[%s:%d] maat_compile_new failed, compile_table:%s compile_id:%d",
@@ -1603,9 +1595,8 @@ int compile_runtime_update(void *compile_runtime, void *compile_schema,
return -1;
}
maat_compile_set(compile, table_name, compile_rule->evaluation_order,
compile_rule->declared_clause_num, compile_rule,
(void (*)(void *))destroy_compile_rule);
maat_compile_set(compile, table_name, compile_rule->declared_clause_num,
compile_rule, (void (*)(void *))compile_rule_free);
struct maat_compile *tmp_compile = maat_compile_hash_find(&(compile_rt->compile_hash), compile_id);
if (tmp_compile != NULL) {
maat_compile_hash_set(&(compile_rt->compile_hash), compile_id, table_name, compile);
@@ -1653,6 +1644,7 @@ int group2compile_runtime_update(void *g2c_runtime, void *g2c_schema,
log_error(compile_rt->logger, MODULE_COMPILE,
"[%s:%d] Remove group %d from compile %d failed, group is not exisited.",
__FUNCTION__, __LINE__, g2c_item->group_id, g2c_item->compile_id);
group2compile_item_free(g2c_item);
return -1;
}
@@ -1741,18 +1733,6 @@ int compile_runtime_commit(void *compile_runtime, const char *table_name)
static int compile_sort_para_compare(const struct compile_sort_para *a,
const struct compile_sort_para *b)
{
//If both of compile rule's evaluation order are specified, compile rule with small evaluation order is priority.
if (a->evaluation_order != 0 && b->evaluation_order != 0) {
if (a->evaluation_order - b->evaluation_order < 0) {
return -1;
} else if(a->evaluation_order - b->evaluation_order > 0) {
return 1;
}
} else if(a->evaluation_order + b->evaluation_order!= 0) {
//If one of compile rule's evaluation order is zero, compile rule with big evaluation order is priority.
return (a->evaluation_order - b->evaluation_order > 0) ? -1 : 1;
}
//If compile rule's execute sequences are not specified or equal.
if (a->declared_clause_num != b->declared_clause_num) {
return (a->declared_clause_num - b->declared_clause_num);
@@ -1766,7 +1746,6 @@ static void compile_sort_para_set(struct compile_sort_para *para,
void *user)
{
para->compile_id = compile_relation->compile_id;
para->evaluation_order = compile_relation->evaluation_order;
para->declared_clause_num = compile_relation->declared_clause_num;
para->user = user;
}

5
src/maat_ex_data.c
View File

@@ -218,9 +218,8 @@ void ex_container_free(void *schema, void *data)
&(ex_container->ex_data), argl, argp);
}
if (ex_container->custom_data != NULL
&& container_schema->custom_data_free != NULL) {
container_schema->custom_data_free(ex_container->custom_data);
if (ex_container->custom_data != NULL) {
FREE(ex_container->custom_data);
}
FREE(ex_container);

7
src/maat_expr.c
View File

@@ -838,7 +838,8 @@ int expr_runtime_scan(struct expr_runtime *expr_rt, int thread_id, const char *d
}
size_t n_hit_item = 0;
struct hs_scan_result hit_results[MAX_SCANNER_HIT_ITEM_NUM] = {0};
struct hs_scan_result hit_results[MAX_SCANNER_HIT_ITEM_NUM];
memset(hit_results, 0, sizeof(hit_results));
int ret = adapter_hs_scan(expr_rt->hs, thread_id, data, data_len,
hit_results, MAX_SCANNER_HIT_ITEM_NUM,
@@ -901,7 +902,9 @@ int expr_runtime_stream_scan(struct expr_runtime *expr_rt, struct adapter_hs_str
}
size_t n_hit_item = 0;
struct hs_scan_result hit_results[MAX_SCANNER_HIT_ITEM_NUM] = {0};
struct hs_scan_result hit_results[MAX_SCANNER_HIT_ITEM_NUM];
memset(hit_results, 0, sizeof(hit_results));
int ret = adapter_hs_scan_stream(s_handle, data, data_len, hit_results, MAX_SCANNER_HIT_ITEM_NUM, &n_hit_item);
if (ret < 0) {
return -1;

3
src/maat_flag.c
View File

@@ -480,7 +480,8 @@ int flag_runtime_scan(struct flag_runtime *flag_rt, int thread_id,
return 0;
}
struct flag_result hit_results[MAX_SCANNER_HIT_ITEM_NUM] = {0};
struct flag_result hit_results[MAX_SCANNER_HIT_ITEM_NUM];
memset(hit_results, 0, sizeof(hit_results));
int n_hit_item = flag_matcher_match(flag_rt->matcher, flag,
hit_results, MAX_SCANNER_HIT_ITEM_NUM);

95
src/maat_fqdn.c
View File

@@ -1,95 +0,0 @@
/*
**********************************************************************************************
* File: maat_fqdn.cpp
* Description:
* Authors: Liu WenTan <liuwentan@geedgenetworks.com>
* Date: 2022-10-31
* Copyright: (c) 2018-2022 Geedge Networks, Inc. All rights reserved.
***********************************************************************************************
*/
#include "maat_fqdn.h"
#include "log/log.h"
#include "fqdn_engine.h"
struct fqdn_schema {
int item_id_column;
int group_id_column;
int fqdn_column;
int match_method_column;
int table_id;
struct table_manager *ref_tbl_mgr;
};
struct fqdn_runtime {
struct FQDN_engine *engine;
struct rcu_hash_table *htable;
uint32_t rule_num;
struct maat_item *item_hash;
void (*item_user_data_free)(void *);
struct maat_garbage_bin *ref_garbage_bin;
struct log_handle *logger;
long long *scan_cnt;
long long *hit_cnt;
};
void *fqdn_schema_new(cJSON *json, struct table_manager *tbl_mgr,
const char *table_name, struct log_handle *logger)
{
return NULL;
}
void fqdn_schema_free(void *fqdn_schema)
{
}
void *fqdn_runtime_new(void *fqdn_schema, int max_thread_num,
struct maat_garbage_bin *garbage_bin,
struct log_handle *logger)
{
return NULL;
}
void fqdn_runtime_free(void *fqdn_runtime)
{
}
int fqdn_runtime_update(void *fqdn_runtime, void *fqdn_schema,
const char *table_name, const char *line,
int valid_column)
{
return 0;
}
int fqdn_runtime_commit(void *fqdn_runtime, const char *table_name)
{
return 0;
}
struct ex_data_runtime *fqdn_runtime_get_ex_data_rt(struct fqdn_runtime *fqdn_rt)
{
return NULL;
}
int fqdn_runtime_scan(struct fqdn_runtime *fqdn_rt, int thread_id, int port,
int proto, int *group_ids, size_t group_id_size,
int vtable_id, struct maat_state *state)
{
return 0;
}
void fqdn_runtime_scan_hit_inc(struct fqdn_runtime *fqdn_rt, int thread_id)
{
}
long long fqdn_runtime_scan_hit_sum(struct fqdn_runtime *fqdn_rt, int n_thread)
{
return 0;
}

1
src/maat_garbage_collection.c
View File

@@ -72,6 +72,7 @@ void maat_garbage_bagging(struct maat_garbage_bin* bin, void* garbage, void *arg
bag->create_time = time(NULL);
bag->timeout = bin->timeout_seconds;
bag->garbage = garbage;
bag->arg = arg;
bag->garbage_free = func;
TAILQ_INSERT_TAIL(&bin->garbage_q, bag, entries);
bin->bag_cnt++;

3
src/maat_interval.c
View File

@@ -476,7 +476,8 @@ int interval_runtime_scan(struct interval_runtime *interval_rt, int thread_id,
return 0;
}
struct interval_result hit_results[MAX_SCANNER_HIT_ITEM_NUM] = {0};
struct interval_result hit_results[MAX_SCANNER_HIT_ITEM_NUM];
memset(hit_results, 0, sizeof(hit_results));
int n_hit_item = interval_matcher_match(interval_rt->matcher, integer,
hit_results, MAX_SCANNER_HIT_ITEM_NUM);

6
src/maat_ip.c
View File

@@ -664,7 +664,8 @@ int ip_runtime_scan(struct ip_runtime *ip_rt, int thread_id, int ip_type,
return 0;
}
struct scan_result ip_results[MAX_SCANNER_HIT_ITEM_NUM] = {0};
struct scan_result ip_results[MAX_SCANNER_HIT_ITEM_NUM];
memset(ip_results, 0, sizeof(ip_results));
/* if ip_addr = "0.0.0.0" means any ip */
int any_ip_flag = 0;
@@ -695,7 +696,8 @@ int ip_runtime_scan(struct ip_runtime *ip_rt, int thread_id, int ip_type,
// any ip, so scan port+proto
if (1 == any_ip_flag) {
struct interval_result port_results[MAX_SCANNER_HIT_ITEM_NUM] = {0};
struct interval_result port_results[MAX_SCANNER_HIT_ITEM_NUM];
memset(port_results, 0, sizeof(port_results));
uint16_t host_port = ntohs(port);
int n_hit_port_item = interval_matcher_match(ip_rt->intval_matcher, host_port,
port_results, MAX_SCANNER_HIT_ITEM_NUM);

9
src/maat_ip_plugin.c
View File

@@ -298,6 +298,12 @@ void ip_plugin_rule_free(struct ip_rule *rule)
FREE(rule);
}
void garbage_ip_plugin_rule_free(void *ip_rule, void *arg)
{
struct ip_rule *rule = (struct ip_rule *)ip_rule;
ip_plugin_rule_free(rule);
}
void ip_plugin_table_set_ex_data_schema(void *ip_plugin_schema,
maat_ex_new_func_t *new_func,
maat_ex_free_func_t *free_func,
@@ -451,12 +457,13 @@ int ip_plugin_runtime_commit(void *ip_plugin_runtime, const char *table_name)
ex_data_runtime_commit(ex_data_rt);
size_t i = 0;
struct ip_rule *rules = NULL;
struct ex_container **ex_container = NULL;
size_t rule_cnt = ex_data_runtime_list_ex_container(ex_data_rt, &ex_container);
if (rule_cnt > 0) {
rules = ALLOC(struct ip_rule, rule_cnt);
for (size_t i = 0; i < rule_cnt; i++) {
for (i = 0; i < rule_cnt; i++) {
rules[i] = *(struct ip_rule *)ex_container[i]->custom_data;
assert(rules[i].user_tag == ex_container[i] || rules[i].user_tag == NULL);
rules[i].user_tag = ex_container[i];

2
src/maat_rule.c
View File

@@ -562,8 +562,8 @@ void *rule_monitor_loop(void *arg)
}
maat_runtime_destroy(maat_instance->maat_rt);
maat_garbage_bin_free(maat_instance->garbage_bin);
table_manager_destroy(maat_instance->tbl_mgr);
maat_garbage_bin_free(maat_instance->garbage_bin);
alignment_int64_array_free(maat_instance->thread_call_cnt);
alignment_int64_array_free(maat_instance->hit_cnt);

8
src/maat_table.c
View File

@@ -643,7 +643,15 @@ void table_manager_runtime_destroy(struct table_manager *tbl_mgr)
}
/* free district map */
if (tbl_mgr->district_map != NULL) {
maat_kv_store_free(tbl_mgr->district_map);
tbl_mgr->district_map = NULL;
}
if (tbl_mgr->tmp_district_map != NULL) {
maat_kv_store_free(tbl_mgr->tmp_district_map);
tbl_mgr->tmp_district_map = NULL;
}
}
void table_manager_destroy(struct table_manager *tbl_mgr)

3
test/CMakeLists.txt
View File

@@ -26,6 +26,9 @@ target_link_libraries(adapter_hs_gtest maat_frame_static gtest_static)
add_executable(maat_ex_data_gtest maat_ex_data_gtest.cpp)
target_link_libraries(maat_ex_data_gtest maat_frame_static gtest_static)
add_executable(maat_garbage_collection_gtest maat_garbage_collection_gtest.cpp)
target_link_libraries(maat_garbage_collection_gtest maat_frame_static gtest_static)
file(COPY table_info.conf DESTINATION ./)
file(COPY file_test_tableinfo.conf DESTINATION ./)
file(COPY literal_expr.conf DESTINATION ./)

3
test/file_test_tableinfo.conf
View File

@@ -8,8 +8,7 @@
"custom": {
"compile_id":1,
"tags":6,
"clause_num":9,
"evaluation_order":10
"clause_num":9
}
},
{

3
test/maat_ex_data_gtest.cpp
View File

@@ -133,10 +133,13 @@ int main(int argc, char ** argv)
maat_options_set_json_file(opts, json_path);
g_maat_instance = maat_new(opts, table_info_path);
maat_options_free(opts);
EXPECT_TRUE(g_maat_instance != NULL);
ret=RUN_ALL_TESTS();
log_handle_destroy(g_logger);
maat_free(g_maat_instance);
g_maat_instance = NULL;
return ret;
}

48
test/maat_garbage_collection_gtest.cpp Normal file
View File

@@ -0,0 +1,48 @@
#include <limits.h>
#include "maat_utils.h"
#include "maat_garbage_collection.h"
#include <gtest/gtest.h>
struct user_info {
char user_name[NAME_MAX];
int user_id;
};
struct user_info *user_info_new(const char *user_name, int user_id)
{
struct user_info *info = ALLOC(struct user_info, 1);
memcpy(info->user_name, user_name, sizeof(info->user_name));
info->user_id = user_id;
return info;
}
void user_info_free(struct user_info *user_info)
{
free(user_info);
}
void garbage_user_info_free(void *user_info, void *arg)
{
struct user_info *info = (struct user_info *)user_info;
user_info_free(info);
}
TEST(garbage_collection, basic)
{
struct maat_garbage_bin *garbage_bin = maat_garbage_bin_new(2);
struct user_info *info = user_info_new("zhangsan", 101);
maat_garbage_bagging(garbage_bin, info, NULL, garbage_user_info_free);
maat_garbage_collect_routine(garbage_bin);
maat_garbage_bin_free(garbage_bin);
}
int main(int argc, char ** argv)
{
int ret=0;
::testing::InitGoogleTest(&argc, argv);
ret=RUN_ALL_TESTS();
return ret;
}