gfwleak/tango-maat
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[BUGFIX]fix xx_plugin user_tag lifecycle

Browse Source
This commit is contained in:
liuwentan
2023-10-09 15:15:05 +08:00
parent 01c290e509
commit f09aa3e1fe
12 changed files with 67 additions and 156 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
src/inc_internal/maat_ex_data.h
View File

@@ -77,8 +77,8 @@ int ex_data_runtime_add_ex_container(struct ex_data_runtime *ex_data_rt,
int ex_data_runtime_del_ex_container(struct ex_data_runtime *ex_data_rt,
const char *key, size_t key_len);
size_t ex_data_runtime_list_ex_container(struct ex_data_runtime *ex_data_rt,
struct ex_container ***ex_container);
size_t ex_data_runtime_list_updating_ex_container(struct ex_data_runtime *ex_data_rt,
struct ex_container ***ex_container);
size_t ex_data_runtime_ex_container_count(struct ex_data_runtime *ex_data_rt);

8
src/maat_bool_plugin.c
View File

@@ -33,11 +33,11 @@ struct bool_plugin_schema {
struct bool_plugin_runtime {
struct bool_matcher *matcher;
struct ex_data_runtime *ex_data_rt;
long long rule_num;
size_t n_worker_thread;
struct maat_garbage_bin *ref_garbage_bin;
struct log_handle *logger;
long long rule_num;
long long update_err_cnt;
};
@@ -449,11 +449,9 @@ int bool_plugin_runtime_commit(void *bool_plugin_runtime, const char *table_name
return 0;
}
ex_data_runtime_commit(ex_data_rt);
struct bool_expr *rules = NULL;
struct ex_container **ex_container = NULL;
size_t rule_cnt = ex_data_runtime_list_ex_container(ex_data_rt, &ex_container);
size_t rule_cnt = ex_data_runtime_list_updating_ex_container(ex_data_rt, &ex_container);
if (rule_cnt > 0) {
rules = ALLOC(struct bool_expr, rule_cnt);
for (size_t i = 0; i < rule_cnt; i++) {
@@ -485,6 +483,8 @@ int bool_plugin_runtime_commit(void *bool_plugin_runtime, const char *table_name
old_bool_matcher = bool_plugin_rt->matcher;
bool_plugin_rt->matcher = new_bool_matcher;
ex_data_runtime_commit(ex_data_rt);
if (old_bool_matcher != NULL) {
maat_garbage_bagging(bool_plugin_rt->ref_garbage_bin, old_bool_matcher, NULL,
garbage_bool_matcher_free);

6
src/maat_ex_data.c
View File

@@ -280,10 +280,10 @@ int ex_data_runtime_is_updating(struct ex_data_runtime *ex_data_rt)
return rcu_hash_is_updating(ex_data_rt->htable);
}
size_t ex_data_runtime_list_ex_container(struct ex_data_runtime *ex_data_rt,
struct ex_container ***ex_container)
size_t ex_data_runtime_list_updating_ex_container(struct ex_data_runtime *ex_data_rt,
struct ex_container ***ex_container)
{
return rcu_hash_list(ex_data_rt->htable, (void ***)ex_container);
return rcu_updating_hash_list(ex_data_rt->htable, (void ***)ex_container);
}
void ex_data_runtime_garbage_collect_routine(struct ex_data_runtime *ex_data_rt)

11
src/maat_fqdn_plugin.c
View File

@@ -34,11 +34,12 @@ struct fqdn_plugin_schema {
struct fqdn_plugin_runtime {
struct FQDN_engine *engine;
struct ex_data_runtime *ex_data_rt;
long long rule_num;
long long update_err_cnt;
size_t n_worker_thread;
struct maat_garbage_bin *ref_garbage_bin;
struct log_handle *logger;
long long rule_num;
long long update_err_cnt;
};
void *fqdn_plugin_schema_new(cJSON *json, struct table_manager *tbl_mgr,
@@ -455,11 +456,9 @@ int fqdn_plugin_runtime_commit(void *fqdn_plugin_runtime, const char *table_name
return 0;
}
ex_data_runtime_commit(ex_data_rt);
struct FQDN_rule *rules = NULL;
struct ex_container **ex_container = NULL;
size_t rule_cnt = ex_data_runtime_list_ex_container(ex_data_rt, &ex_container);
size_t rule_cnt = ex_data_runtime_list_updating_ex_container(ex_data_rt, &ex_container);
if (rule_cnt > 0) {
rules = ALLOC(struct FQDN_rule, rule_cnt);
for (size_t i = 0; i < rule_cnt; i++) {
@@ -490,6 +489,8 @@ int fqdn_plugin_runtime_commit(void *fqdn_plugin_runtime, const char *table_name
old_fqdn_engine = fqdn_plugin_rt->engine;
fqdn_plugin_rt->engine = new_fqdn_engine;
ex_data_runtime_commit(ex_data_rt);
if (old_fqdn_engine != NULL) {
maat_garbage_bagging(fqdn_plugin_rt->ref_garbage_bin, old_fqdn_engine, NULL,
garbage_fqdn_engine_free);

25
src/maat_ip.c
View File

@@ -66,7 +66,6 @@ struct ip_item {
struct ip_runtime {
struct ip_matcher *ip_matcher;
struct interval_matcher *intval_matcher;
struct rcu_hash_table *item_hash; // <item_id, struct ip_item>
long long rule_num;
@@ -428,11 +427,6 @@ void ip_runtime_free(void *ip_runtime)
ip_rt->ip_matcher = NULL;
}
if (ip_rt->intval_matcher != NULL) {
interval_matcher_free(ip_rt->intval_matcher);
ip_rt->intval_matcher = NULL;
}
if (ip_rt->item_hash != NULL) {
rcu_hash_free(ip_rt->item_hash);
ip_rt->item_hash = NULL;
@@ -601,9 +595,6 @@ int ip_runtime_commit(void *ip_runtime, const char *table_name,
struct ip_matcher *new_ip_matcher = NULL;
struct ip_matcher *old_ip_matcher = NULL;
struct interval_matcher *new_intval_matcher = NULL;
struct interval_matcher *old_intval_matcher = NULL;
if (rule_cnt > 0) {
new_ip_matcher = ip_matcher_new(rules, rule_cnt, &mem_used);
if (NULL == new_ip_matcher) {
@@ -617,15 +608,6 @@ int ip_runtime_commit(void *ip_runtime, const char *table_name,
"table[%s] commit %zu ip rules and rebuild ip_matcher completed"
", version:%lld", table_name, rule_cnt, maat_rt_version);
}
new_intval_matcher = interval_matcher_new(intval_rules, rule_cnt);
if (NULL == new_intval_matcher) {
log_error(ip_rt->logger, MODULE_IP,
"[%s:%d] table[%s] rebuild interval_matcher engine "
"failed when update %zu ip rules", __FUNCTION__, __LINE__,
table_name, rule_cnt);
ret = -1;
}
}
old_ip_matcher = ip_rt->ip_matcher;
@@ -637,13 +619,6 @@ int ip_runtime_commit(void *ip_runtime, const char *table_name,
garbage_ip_matcher_free);
}
old_intval_matcher = ip_rt->intval_matcher;
ip_rt->intval_matcher = new_intval_matcher;
if (old_intval_matcher != NULL) {
maat_garbage_bagging(ip_rt->ref_garbage_bin, old_intval_matcher, NULL,
garbage_interval_matcher_free);
}
ip_rt->rule_num = rule_cnt;
if (rules != NULL) {

11
src/maat_ip_plugin.c
View File

@@ -38,11 +38,12 @@ struct ip_plugin_schema {
struct ip_plugin_runtime {
struct ip_matcher *ip_matcher;
struct ex_data_runtime *ex_data_rt;
long long rule_num;
long long update_err_cnt;
size_t n_worker_thread;
struct maat_garbage_bin *ref_garbage_bin;
struct log_handle *logger;
long long rule_num;
long long update_err_cnt;
};
void *ip_plugin_schema_new(cJSON *json, struct table_manager *tbl_mgr,
@@ -494,11 +495,9 @@ int ip_plugin_runtime_commit(void *ip_plugin_runtime, const char *table_name,
return 0;
}
ex_data_runtime_commit(ex_data_rt);
struct ip_rule *rules = NULL;
struct ex_container **ex_container = NULL;
size_t rule_cnt = ex_data_runtime_list_ex_container(ex_data_rt, &ex_container);
size_t rule_cnt = ex_data_runtime_list_updating_ex_container(ex_data_rt, &ex_container);
if (rule_cnt > 0) {
rules = ALLOC(struct ip_rule, rule_cnt);
for (size_t i = 0; i < rule_cnt; i++) {
@@ -529,6 +528,8 @@ int ip_plugin_runtime_commit(void *ip_plugin_runtime, const char *table_name,
old_ip_matcher = ip_plugin_rt->ip_matcher;
ip_plugin_rt->ip_matcher = new_ip_matcher;
ex_data_runtime_commit(ex_data_rt);
if (old_ip_matcher != NULL) {
maat_garbage_bagging(ip_plugin_rt->ref_garbage_bin, old_ip_matcher, NULL,
garbage_ip_matcher_free);

18
src/maat_ipport_plugin.c
View File

@@ -57,15 +57,14 @@ struct ipport_item {
struct ipport_plugin_runtime {
struct ip_matcher *ip_matcher;
struct interval_matcher *intval_matcher;
struct rcu_hash_table *item_hash; //<item_id, struct ipport_item>
long long rule_num;
long long update_err_cnt;
struct ex_data_runtime *ex_data_rt;
size_t n_worker_thread;
struct maat_garbage_bin *ref_garbage_bin;
struct log_handle *logger;
long long rule_num;
long long update_err_cnt;
};
void *ipport_plugin_schema_new(cJSON *json, struct table_manager *tbl_mgr,
@@ -239,11 +238,6 @@ void ipport_plugin_runtime_free(void *ipport_plugin_runtime)
ipport_plugin_rt->ip_matcher = NULL;
}
if (ipport_plugin_rt->intval_matcher != NULL) {
interval_matcher_free(ipport_plugin_rt->intval_matcher);
ipport_plugin_rt->intval_matcher = NULL;
}
if (ipport_plugin_rt->ex_data_rt != NULL) {
ex_data_runtime_free(ipport_plugin_rt->ex_data_rt);
ipport_plugin_rt->ex_data_rt = NULL;
@@ -477,11 +471,9 @@ int ipport_plugin_runtime_commit(void *ipport_plugin_runtime, const char *table_
return 0;
}
ex_data_runtime_commit(ex_data_rt);
struct ip_rule *rules = NULL;
struct ex_container **ex_container = NULL;
size_t rule_cnt = ex_data_runtime_list_ex_container(ex_data_rt, &ex_container);
size_t rule_cnt = ex_data_runtime_list_updating_ex_container(ex_data_rt, &ex_container);
if (rule_cnt > 0) {
rules = ALLOC(struct ip_rule, rule_cnt);
for (size_t i = 0; i < rule_cnt; i++) {
@@ -512,6 +504,8 @@ int ipport_plugin_runtime_commit(void *ipport_plugin_runtime, const char *table_
old_ip_matcher = ipport_plugin_rt->ip_matcher;
ipport_plugin_rt->ip_matcher = new_ip_matcher;
ex_data_runtime_commit(ex_data_rt);
if (old_ip_matcher != NULL) {
maat_garbage_bagging(ipport_plugin_rt->ref_garbage_bin, old_ip_matcher, NULL,
garbage_ip_matcher_free);

7
src/maat_plugin.c
View File

@@ -32,12 +32,13 @@ struct plugin_callback_schema {
};
struct plugin_runtime {
long long acc_line_num;
struct ex_data_runtime *ex_data_rt;
long long rule_num;
long long update_err_cnt;
struct maat_garbage_bin *ref_garbage_bin;
struct log_handle *logger;
long long rule_num;
long long acc_line_num;
long long update_err_cnt;
};
enum plugin_key_type {

2
src/maat_utils.c
View File

@@ -318,7 +318,7 @@ int crypt_memory(const unsigned char *inbuf, size_t inlen, unsigned char **pp_ou
int ret = EVP_BytesToKey(cipher, dgst, salt, (unsigned char *)key,
strlen((const char *)key), 1, cipher_key, cipher_iv);
if(0 == ret) {
snprintf(err_str, err_str_sz, "Key and IV generatioin failed.");
snprintf(err_str, err_str_sz, "Key and IV generation failed.");
return 0;
}

57
test/ipport_plugin/ipport_plugin_gtest.cpp
View File

@@ -15,36 +15,22 @@
const char *table_info_path = "./ipport_plugin_table_info.conf";
const char *log_file = "./ipport_plugin_gtest.log";
const char *g_ip_str = "192.0.2.0";
class IPPortPluginTable : public testing::Test
{
protected:
static void SetUpTestCase() {
logger = log_handle_create(log_file, 0);
// char iris_idx_path[PATH_MAX] = {0};
// snprintf(iris_idx_path, sizeof(iris_idx_path), "test_data/index");
// struct maat_options *opts = maat_options_new();
// maat_options_set_iris(opts, iris_idx_path, iris_idx_path);
// maat_options_set_logger(opts, log_file, LOG_LEVEL_INFO);
// maat_options_set_stat_file(opts, "./stat.log");
// struct maat *_shared_maat_inst = maat_new(opts, table_info_path);
// assert(_shared_maat_inst);
// maat_options_free(opts);
}
static void TearDownTestCase() {
log_handle_destroy(logger);
// maat_free(_shared_maat_inst);
}
static struct log_handle *logger;
// static struct maat *_shared_maat_inst;
};
// struct maat *IPPortPluginTable::_shared_maat_inst;
struct log_handle *IPPortPluginTable::logger;
int count_line_num_cb(const char *table_name, const char *line, void *u_para)
@@ -207,7 +193,7 @@ void *ipport_plugin_scan_thread(void *arg)
struct ip_addr ipv4;
ipv4.ip_type = 4;
int ret = inet_pton(AF_INET, "192.0.2.0", &ipv4.ipv4);
int ret = inet_pton(AF_INET, g_ip_str, &ipv4.ipv4);
EXPECT_EQ(ret, 1);
uint16_t port = htons(200);
@@ -236,7 +222,7 @@ void *ipport_plugin_scan_thread(void *arg)
}
static void test_add_ipport_plugin_command(struct maat *maat_inst, const char *table_name,
const char *ip_str, long long item_id)
long long item_id, const char *ip_str, int port1, int port2)
{
int table_id = maat_get_table_id(maat_inst, table_name);
assert(table_id >= 0);
@@ -246,7 +232,29 @@ static void test_add_ipport_plugin_command(struct maat *maat_inst, const char *t
char table_line[1024] = {0};
sprintf(table_line, "%lld\t4\t%s\t100\t200\t1", item_id, ip_str);
sprintf(table_line, "%lld\t4\t%s\t%d\t%d\t1", item_id, ip_str, port1, port2);
struct maat_cmd_line line_rule;
line_rule.rule_id = item_id;
line_rule.table_line = table_line;
line_rule.table_name = table_name;
line_rule.expire_after = 0;
maat_cmd_set_line(maat_inst, &line_rule);
}
static void test_del_ipport_plugin_command(struct maat *maat_inst, const char *table_name,
long long item_id, const char *ip_str, int port1, int port2)
{
int table_id = maat_get_table_id(maat_inst, table_name);
assert(table_id >= 0);
enum table_type table_type = table_manager_get_table_type(maat_inst->tbl_mgr, table_id);
assert(table_type == TABLE_TYPE_IPPORT_PLUGIN);
char table_line[1024] = {0};
sprintf(table_line, "%lld\t4\t%s\t%d\t%d\t0", item_id, ip_str, port1, port2);
struct maat_cmd_line line_rule;
line_rule.rule_id = item_id;
@@ -262,17 +270,22 @@ void *ipport_plugin_update_thread(void *arg)
struct thread_param *param = (struct thread_param *)arg;
struct maat *maat_inst = param->maat_inst;
const char *table_name = param->table_name;
const int CMD_EXPR_NUM = 100;
char ip_buf[128] = {0};
const int CMD_EXPR_NUM = 1024;
long long item_id = 9000000;
for (int i = 0; i < CMD_EXPR_NUM; i++) {
sprintf(ip_buf, "172.16.1.%d", i);
test_add_ipport_plugin_command(maat_inst, table_name, ip_buf, item_id);
test_add_ipport_plugin_command(maat_inst, table_name, item_id, g_ip_str, i+201, i+201);
item_id++;
usleep(100 * 1000);
}
item_id = 9000000;
for (int i = 0; i < CMD_EXPR_NUM; i++) {
test_del_ipport_plugin_command(maat_inst, table_name, item_id, g_ip_str, i+201, i+201);
usleep(100 * 1000);
item_id++;
}
int *is_all_hit = ALLOC(int, 1);
*is_all_hit = 1;

1
test/ipport_plugin/ipport_plugin_table_info.conf
View File

@@ -5,7 +5,6 @@
"table_type":"ipport_plugin",
"valid_column":6,
"custom": {
"gc_timeout_s": 3,
"item_id":1,
"ip_type":2,
"ip_addr":3,

73
test/maat_framework_gtest.cpp
View File

@@ -322,79 +322,6 @@ static void random_keyword_generate(char *keyword_buf, size_t sz)
keyword_buf[i]='\0';
return;
}
#if 0
class MaatIris : public testing::Test
{
protected:
static void SetUpTestCase() {
logger = log_handle_create("./maat_framework_gtest.log", 0);
struct maat_options *opts = maat_options_new();
maat_options_set_iris(opts, "./redis_dump", "./redis_dump");
maat_options_set_stat_file(opts, "./stat.log");
maat_options_set_logger(opts, "./maat_framework_gtest.log", LOG_LEVEL_INFO);
const char *table_info_path = "./verify_table_info.conf";
_shared_maat_inst = maat_new(opts, table_info_path);
maat_options_free(opts);
if (NULL == _shared_maat_inst) {
log_error(logger, MODULE_FRAMEWORK_GTEST,
"[%s:%d] create maat instance in MaatIris failed.",
__FUNCTION__, __LINE__);
}
}
static void TearDownTestCase() {
maat_free(_shared_maat_inst);
log_handle_destroy(logger);
}
static struct log_handle *logger;
static struct maat *_shared_maat_inst;
};
struct maat *MaatIris::_shared_maat_inst;
struct log_handle *MaatIris::logger;
TEST_F(MaatIris, basic) {
struct maat *maat_inst = MaatIris::_shared_maat_inst;
struct log_handle *logger = MaatIris::logger;
long long results[ARRAY_SIZE] = {0};
size_t n_hit_result = 0;
int thread_id = 0;
struct maat_state *state = maat_state_new(maat_inst, thread_id);
uint32_t sip_addr;
inet_pton(AF_INET, "100.64.1.1", &sip_addr);
uint16_t sport = htons(80);
int table_id = maat_get_table_id(maat_inst, "TSG_SECURITY_SOURCE_ADDR");
ASSERT_GT(table_id, 0);
int ret = maat_scan_ipv4(maat_inst, table_id, sip_addr, sport, 6,
results, ARRAY_SIZE, &n_hit_result, state);
EXPECT_EQ(ret, MAAT_SCAN_HALF_HIT);
table_id = maat_get_table_id(maat_inst, "TSG_OBJ_APP_ID");
ASSERT_GT(table_id, 0);
ret = maat_scan_integer(maat_inst, table_id, 105, results, ARRAY_SIZE,
&n_hit_result, state);
EXPECT_EQ(ret, MAAT_SCAN_HALF_HIT);
table_id = maat_get_table_id(maat_inst, "TSG_FIELD_HTTP_URL");
ASSERT_GT(table_id, 0);
const char *scan_data = "www.luis.com";
ret = maat_scan_string(maat_inst, table_id, scan_data, strlen(scan_data),
results, ARRAY_SIZE, &n_hit_result, state);
EXPECT_EQ(ret, MAAT_SCAN_HIT);
EXPECT_EQ(n_hit_result, 1);
EXPECT_EQ(results[0], 1267067);
maat_state_free(state);
}
#endif
const char* watched_json="./json_update/maat.json";
const char* old_json="./json_update/old.json";