gfwleak/tango-maat
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Feature/scan ip port together support endpoint object

Browse Source
This commit is contained in:
郑超
2024-03-27 11:19:39 +00:00
committed by liuwentan
parent 35d60d06b5
commit 99f98abbcd
30 changed files with 942 additions and 1064 deletions
Download Patch File Download Diff File Expand all files Collapse all files

355
test/maat_framework_perf_gtest.cpp
View File

@@ -1,5 +1,6 @@
#include <gtest/gtest.h>
#include "test_utils.h"
#include "log/log.h"
#include "maat.h"
#include "maat_rule.h"
@@ -17,8 +18,8 @@
#define PERF_THREAD_NUM 5
#define PERF_SCAN_TIMES 1000 * 1000
const char *table_info_path = "./table_info.conf";
const char *json_filename = "maat_json.json";
const char *g_table_info_path = "./table_info.conf";
const char *g_json_filename = "maat_json.json";
struct thread_param {
int thread_id;
@@ -40,282 +41,7 @@ void random_keyword_generate(char *keyword_buf, size_t sz)
keyword_buf[i] = '\0';
}
int count_line_num_cb(const char *table_name, const char *line, void *u_para)
{
(*((unsigned int *)u_para))++;
return 0;
}
int line_idx = 0;
long long absolute_expire_time=0;
int make_serial_rule(const char *table_name, const char *line, void *u_para)
{
struct serial_rule *s_rule=(struct serial_rule *)u_para;
long long rule_id = 0;
char *buff = ALLOC(char, strlen(line) + 1);
memcpy(buff, line, strlen(line) + 1);
while (buff[strlen(buff) - 1] == '\n' || buff[strlen(buff) - 1] == '\t') {
buff[strlen(buff) - 1] = '\0';
}
int j = 0;
char *str1 = NULL;
char *token = NULL;
char *saveptr1 = NULL;
for (j = 0,str1 = buff; ; j++, str1 = NULL) {
token = strtok_r(str1, "\t ", &saveptr1);
if (token == NULL)
break;
if (j == 0) {
sscanf(token,"%lld", &rule_id);
}
}
memcpy(buff, line, strlen(line)+1);
while(buff[strlen(buff)-1]=='\n'||buff[strlen(buff)-1]=='\t') {
buff[strlen(buff)-1]='\0';
}
maat_set_serial_rule(s_rule + line_idx, MAAT_OP_ADD, rule_id, table_name,
buff, absolute_expire_time);
line_idx++;
FREE(buff);
return 0;
}
static int
write_config_to_redis(const char *json_iris_path,
char *redis_ip, int redis_port,
int redis_db, struct log_handle *logger)
{
redisContext *c =
maat_connect_redis(redis_ip, redis_port, redis_db, logger);
if (NULL == c) {
return -1;
}
redisReply *reply = maat_wrap_redis_command(c, logger, "flushdb");
if (NULL == reply) {
return -1;
} else {
freeReplyObject(reply);
reply = NULL;
}
if (access(json_iris_path, F_OK) < 0) {
char tmp_iris_path[128] = {0};
char *json_buff = NULL;
size_t json_buff_sz = 0;
int ret = load_file_to_memory(json_filename,
(unsigned char **)&json_buff,
&json_buff_sz);
if (ret < 0) {
return -1;
}
ret = json2iris(json_buff, json_filename, c, tmp_iris_path,
sizeof(tmp_iris_path), NULL, NULL, logger);
FREE(json_buff);
if (ret < 0) {
return -1;
}
}
size_t total_line_cnt = 0;
char tmp_iris_full_idx_path[PATH_MAX] = {0};
snprintf(tmp_iris_full_idx_path, sizeof(tmp_iris_full_idx_path),
"%s/index", json_iris_path);
config_monitor_traverse(0, tmp_iris_full_idx_path, NULL,
count_line_num_cb, NULL, &total_line_cnt,
NULL, logger);
struct serial_rule *s_rule = ALLOC(struct serial_rule, total_line_cnt);
long long server_time = maat_redis_server_time_s(c);
if (server_time < 0) {
return -1;
}
//absolute_expire_time = server_time + 300;
config_monitor_traverse(0, tmp_iris_full_idx_path, NULL,
make_serial_rule, NULL, s_rule,
NULL, logger);
line_idx = 0;
absolute_expire_time = 0;
int success_cnt = 0;
do {
success_cnt = maat_cmd_write_rule(c, s_rule, total_line_cnt,
server_time, logger);
} while (success_cnt < 0);
EXPECT_EQ(success_cnt, (int)total_line_cnt);
for (size_t i = 0; i < total_line_cnt; i++) {
maat_clear_rule_cache(s_rule + i);
}
FREE(s_rule);
redisFree(c);
return 0;
}
static int
compile_table_set_line(struct maat *maat_inst, const char *table_name,
enum maat_operation op, long long compile_id,
const char *user_region, int clause_num,
int expire_after)
{
char table_line[1024 * 16] = {0};
sprintf(table_line, "%lld\t0\t0\t0\t0\t0\t%s\t%d\t%d\t0.0",
compile_id, user_region, clause_num, op);
struct maat_cmd_line line_rule;
line_rule.rule_id = compile_id;
line_rule.table_line = table_line;
line_rule.table_name = table_name;
line_rule.expire_after = expire_after;
return maat_cmd_set_line(maat_inst, &line_rule);
}
static int
group2compile_table_set_line(struct maat *maat_inst, const char *table_name,
enum maat_operation op, long long group_id,
long long compile_id, int not_flag,
const char *vtable_name, int clause_index,
int expire_after)
{
char table_line[128] = {0};
sprintf(table_line, "%lld\t%lld\t%d\t%s\t%d\t%d",
group_id, compile_id, not_flag, vtable_name, clause_index, op);
struct maat_cmd_line line_rule;
line_rule.rule_id = group_id;
line_rule.table_line = table_line;
line_rule.table_name = table_name;
line_rule.expire_after = expire_after;
return maat_cmd_set_line(maat_inst, &line_rule);
}
static int
expr_table_set_line(struct maat *maat_inst, const char *table_name,
enum maat_operation op, long long item_id,
long long group_id, const char *keywords,
const char *district, int expr_type,
int match_method, int is_hexbin, int expire_after)
{
char table_line[1024] = {0};
int table_id = maat_get_table_id(maat_inst, table_name);
if (table_id < 0) {
return 0;
}
enum table_type table_type =
table_manager_get_table_type(maat_inst->tbl_mgr, table_id);
assert(table_type == TABLE_TYPE_EXPR || table_type == TABLE_TYPE_EXPR_PLUS);
if (table_type == TABLE_TYPE_EXPR_PLUS) {
sprintf(table_line, "%lld\t%lld\t%s\t%s\t%d\t%d\t%d\t%d",
item_id, group_id, district, keywords, expr_type,
match_method, is_hexbin, op);
} else {
sprintf(table_line, "%lld\t%lld\t%s\t%d\t%d\t%d\t%d",
item_id, group_id, keywords, expr_type,
match_method, is_hexbin, op);
}
struct maat_cmd_line line_rule;
line_rule.rule_id = item_id;
line_rule.table_line = table_line;
line_rule.table_name = table_name;
line_rule.expire_after = expire_after;
return maat_cmd_set_line(maat_inst, &line_rule);
}
static int
ip_table_set_line(struct maat *maat_inst, const char *table_name,
enum maat_operation op, long long item_id,
long long group_id, enum IP_TYPE type,
const char *ip1, const char *ip2, int expire_after)
{
char table_line[1024] = {0};
int table_id = maat_get_table_id(maat_inst, table_name);
if (table_id < 0) {
return 0;
}
int ip_type = IPV4;
if (type == IPv6) {
ip_type = IPV6;
}
sprintf(table_line, "%lld\t%lld\t%d\t%s\t%s\t%s\t%d",
item_id, group_id, ip_type, "range", ip1, ip2, op);
struct maat_cmd_line line_rule;
line_rule.rule_id = item_id;
line_rule.table_line = table_line;
line_rule.table_name = table_name;
line_rule.expire_after = expire_after;
return maat_cmd_set_line(maat_inst, &line_rule);
}
static int
integer_table_set_line(struct maat *maat_inst, const char *table_name,
enum maat_operation op, long long item_id,
long long group_id, int low_boundary,
int up_boundary, int expire_after)
{
char table_line[1024] = {0};
int table_id = maat_get_table_id(maat_inst, table_name);
if (table_id < 0) {
return 0;
}
sprintf(table_line, "%lld\t%lld\t%d\t%d\t%d",
item_id, group_id, low_boundary, up_boundary, op);
struct maat_cmd_line line_rule;
line_rule.rule_id = item_id;
line_rule.table_line = table_line;
line_rule.table_name = table_name;
line_rule.expire_after = expire_after;
return maat_cmd_set_line(maat_inst, &line_rule);
}
static int
flag_table_set_line(struct maat *maat_inst, const char *table_name,
enum maat_operation op, long long item_id,
long long group_id, long long flag,
long long flag_mask, int expire_after)
{
char table_line[1024] = {0};
int table_id = maat_get_table_id(maat_inst, table_name);
if (table_id < 0) {
return 0;
}
sprintf(table_line, "%lld\t%lld\t%lld\t%lld\t%d",
item_id, group_id, flag, flag_mask, op);
struct maat_cmd_line line_rule;
line_rule.rule_id = item_id;
line_rule.table_line = table_line;
line_rule.table_name = table_name;
line_rule.expire_after = expire_after;
return maat_cmd_set_line(maat_inst, &line_rule);
}
static void
test_add_expr_command(struct maat *maat_inst, const char *table_name,
@@ -378,8 +104,8 @@ test_add_integer_command(struct maat *maat_inst, const char *table_name,
EXPECT_EQ(ret, 1);
long long item_id = maat_cmd_incrby(maat_inst, "SEQUENCE_REGION", 1);
ret = integer_table_set_line(maat_inst, table_name, MAAT_OP_ADD, item_id,
group_id, low_boundary, up_boundary, 0);
ret = interval_table_set_line(maat_inst, table_name, MAAT_OP_ADD, item_id,
group_id, low_boundary, up_boundary, NULL, 0);
EXPECT_EQ(ret, 1);
}
@@ -415,12 +141,9 @@ protected:
char redis_ip[64] = "127.0.0.1";
int redis_port = 6379;
int redis_db = 0;
char json_iris_path[512] = {0};
snprintf(json_iris_path, sizeof(json_iris_path),
"./%s_iris_tmp", json_filename);
logger = log_handle_create("./maat_framework_perf_gtest.log", 0);
int ret = write_config_to_redis(json_iris_path, redis_ip, redis_port,
int ret = write_json_to_redis(g_json_filename, redis_ip, redis_port,
redis_db, logger);
if (ret < 0) {
log_fatal(logger, MODULE_FRAMEWORK_PERF_GTEST,
@@ -437,7 +160,7 @@ protected:
maat_options_set_accept_tags(opts, accept_tags);
maat_options_set_caller_thread_number(opts, 5);
_shared_maat_inst = maat_new(opts, table_info_path);
_shared_maat_inst = maat_new(opts, g_table_info_path);
maat_options_free(opts);
if (NULL == _shared_maat_inst) {
log_fatal(logger, MODULE_FRAMEWORK_PERF_GTEST,
@@ -573,12 +296,9 @@ protected:
char redis_ip[64] = "127.0.0.1";
int redis_port = 6379;
int redis_db = 0;
char json_iris_path[512] = {0};
snprintf(json_iris_path, sizeof(json_iris_path),
"./%s_iris_tmp", json_filename);
logger = log_handle_create("./maat_framework_perf_gtest.log", 0);
int ret = write_config_to_redis(json_iris_path, redis_ip, redis_port,
int ret = write_json_to_redis(g_json_filename, redis_ip, redis_port,
redis_db, logger);
if (ret < 0) {
log_fatal(logger, MODULE_FRAMEWORK_PERF_GTEST,
@@ -595,7 +315,7 @@ protected:
maat_options_set_accept_tags(opts, accept_tags);
maat_options_set_caller_thread_number(opts, 5);
_shared_maat_inst = maat_new(opts, table_info_path);
_shared_maat_inst = maat_new(opts, g_table_info_path);
maat_options_free(opts);
if (NULL == _shared_maat_inst) {
log_fatal(logger, MODULE_FRAMEWORK_PERF_GTEST,
@@ -786,12 +506,9 @@ protected:
char redis_ip[64] = "127.0.0.1";
int redis_port = 6379;
int redis_db = 0;
char json_iris_path[512] = {0};
snprintf(json_iris_path, sizeof(json_iris_path),
"./%s_iris_tmp", json_filename);
\
logger = log_handle_create("./maat_framework_perf_gtest.log", 0);
int ret = write_config_to_redis(json_iris_path, redis_ip, redis_port,
int ret = write_json_to_redis(g_json_filename, redis_ip, redis_port,
redis_db, logger);
if (ret < 0) {
log_fatal(logger, MODULE_FRAMEWORK_PERF_GTEST,
@@ -808,7 +525,7 @@ protected:
maat_options_set_accept_tags(opts, accept_tags);
maat_options_set_caller_thread_number(opts, 5);
_shared_maat_inst = maat_new(opts, table_info_path);
_shared_maat_inst = maat_new(opts, g_table_info_path);
maat_options_free(opts);
if (NULL == _shared_maat_inst) {
log_fatal(logger, MODULE_FRAMEWORK_PERF_GTEST,
@@ -924,12 +641,9 @@ protected:
char redis_ip[64] = "127.0.0.1";
int redis_port = 6379;
int redis_db = 0;
char json_iris_path[512] = {0};
snprintf(json_iris_path, sizeof(json_iris_path),
"./%s_iris_tmp", json_filename);
logger = log_handle_create("./maat_framework_perf_gtest.log", 0);
int ret = write_config_to_redis(json_iris_path, redis_ip, redis_port,
int ret = write_json_to_redis(g_json_filename, redis_ip, redis_port,
redis_db, logger);
if (ret < 0) {
log_fatal(logger, MODULE_FRAMEWORK_PERF_GTEST,
@@ -946,7 +660,7 @@ protected:
maat_options_set_accept_tags(opts, accept_tags);
maat_options_set_caller_thread_number(opts, 5);
_shared_maat_inst = maat_new(opts, table_info_path);
_shared_maat_inst = maat_new(opts, g_table_info_path);
maat_options_free(opts);
if (NULL == _shared_maat_inst) {
log_fatal(logger, MODULE_FRAMEWORK_PERF_GTEST,
@@ -1096,12 +810,9 @@ protected:
char redis_ip[64] = "127.0.0.1";
int redis_port = 6379;
int redis_db = 0;
char json_iris_path[512] = {0};
snprintf(json_iris_path, sizeof(json_iris_path),
"./%s_iris_tmp", json_filename);
logger = log_handle_create("./maat_framework_perf_gtest.log", 0);
int ret = write_config_to_redis(json_iris_path, redis_ip, redis_port,
int ret = write_json_to_redis(g_json_filename, redis_ip, redis_port,
redis_db, logger);
if (ret < 0) {
log_fatal(logger, MODULE_FRAMEWORK_PERF_GTEST,
@@ -1118,7 +829,7 @@ protected:
maat_options_set_accept_tags(opts, accept_tags);
maat_options_set_caller_thread_number(opts, 5);
_shared_maat_inst = maat_new(opts, table_info_path);
_shared_maat_inst = maat_new(opts, g_table_info_path);
maat_options_free(opts);
if (NULL == _shared_maat_inst) {
log_fatal(logger, MODULE_FRAMEWORK_PERF_GTEST,
@@ -1198,12 +909,9 @@ protected:
char redis_ip[64] = "127.0.0.1";
int redis_port = 6379;
int redis_db = 0;
char json_iris_path[512] = {0};
snprintf(json_iris_path, sizeof(json_iris_path),
"./%s_iris_tmp", json_filename);
logger = log_handle_create("./maat_framework_perf_gtest.log", 0);
int ret = write_config_to_redis(json_iris_path, redis_ip, redis_port,
int ret = write_json_to_redis(g_json_filename, redis_ip, redis_port,
redis_db, logger);
if (ret < 0) {
log_fatal(logger, MODULE_FRAMEWORK_PERF_GTEST,
@@ -1220,7 +928,7 @@ protected:
maat_options_set_accept_tags(opts, accept_tags);
maat_options_set_caller_thread_number(opts, 5);
_shared_maat_inst = maat_new(opts, table_info_path);
_shared_maat_inst = maat_new(opts, g_table_info_path);
maat_options_free(opts);
if (NULL == _shared_maat_inst) {
log_fatal(logger, MODULE_FRAMEWORK_PERF_GTEST,
@@ -1353,12 +1061,9 @@ protected:
char redis_ip[64] = "127.0.0.1";
int redis_port = 6379;
int redis_db = 0;
char json_iris_path[512] = {0};
snprintf(json_iris_path, sizeof(json_iris_path),
"./%s_iris_tmp", json_filename);
logger = log_handle_create("./maat_framework_perf_gtest.log", 0);
int ret = write_config_to_redis(json_iris_path, redis_ip, redis_port,
int ret = write_json_to_redis(g_json_filename, redis_ip, redis_port,
redis_db, logger);
if (ret < 0) {
log_fatal(logger, MODULE_FRAMEWORK_PERF_GTEST,
@@ -1375,7 +1080,7 @@ protected:
maat_options_set_accept_tags(opts, accept_tags);
maat_options_set_caller_thread_number(opts, 5);
_shared_maat_inst = maat_new(opts, table_info_path);
_shared_maat_inst = maat_new(opts, g_table_info_path);
maat_options_free(opts);
if (NULL == _shared_maat_inst) {
log_fatal(logger, MODULE_FRAMEWORK_PERF_GTEST,
@@ -1591,12 +1296,9 @@ protected:
char redis_ip[64] = "127.0.0.1";
int redis_port = 6379;
int redis_db = 0;
char json_iris_path[512] = {0};
snprintf(json_iris_path, sizeof(json_iris_path),
"./%s_iris_tmp", json_filename);
logger = log_handle_create("./maat_framework_perf_gtest.log", 0);
int ret = write_config_to_redis(json_iris_path, redis_ip, redis_port,
int ret = write_json_to_redis(g_json_filename, redis_ip, redis_port,
redis_db, logger);
if (ret < 0) {
log_fatal(logger, MODULE_FRAMEWORK_PERF_GTEST,
@@ -1613,7 +1315,7 @@ protected:
maat_options_set_accept_tags(opts, accept_tags);
maat_options_set_caller_thread_number(opts, 5);
_shared_maat_inst = maat_new(opts, table_info_path);
_shared_maat_inst = maat_new(opts, g_table_info_path);
maat_options_free(opts);
if (NULL == _shared_maat_inst) {
log_fatal(logger, MODULE_FRAMEWORK_PERF_GTEST,
@@ -1949,12 +1651,10 @@ protected:
char redis_ip[64] = "127.0.0.1";
int redis_port = 6379;
int redis_db = 0;
char json_iris_path[512] = {0};
const char *tsg_table_info = "./tsg_table_info.conf";
snprintf(json_iris_path, sizeof(json_iris_path), "./tsgrule");
logger = log_handle_create("./maat_framework_perf_gtest.log", 0);
int ret = write_config_to_redis(json_iris_path, redis_ip, redis_port,
int ret = write_iris_to_redis("./tsgrule", redis_ip, redis_port,
redis_db, logger);
if (ret < 0) {
log_fatal(logger, MODULE_FRAMEWORK_PERF_GTEST,
@@ -2026,12 +1726,9 @@ protected:
char redis_ip[64] = "127.0.0.1";
int redis_port = 6379;
int redis_db = 0;
char json_iris_path[512] = {0};
snprintf(json_iris_path, sizeof(json_iris_path),
"./%s_iris_tmp", json_filename);
logger = log_handle_create("./maat_framework_perf_gtest.log", 0);
int ret = write_config_to_redis(json_iris_path, redis_ip, redis_port,
int ret = write_json_to_redis(g_json_filename, redis_ip, redis_port,
redis_db, logger);
if (ret < 0) {
log_fatal(logger, MODULE_FRAMEWORK_PERF_GTEST,
@@ -2048,7 +1745,7 @@ protected:
maat_options_set_accept_tags(opts, accept_tags);
maat_options_set_caller_thread_number(opts, 5);
_shared_maat_inst = maat_new(opts, table_info_path);
_shared_maat_inst = maat_new(opts, g_table_info_path);
maat_options_free(opts);
if (NULL == _shared_maat_inst) {
log_fatal(logger, MODULE_FRAMEWORK_PERF_GTEST,