gfwleak/tango-maat
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix coredump because of no compile/group2compile/group2group field in maat_json.json file

Browse Source
This commit is contained in:
liuwentan
2023-03-30 20:49:59 +08:00
parent 4bbd0ebdc4
commit 18881c5bc3
8 changed files with 726 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
src/json2iris.c
View File

@@ -1094,16 +1094,31 @@ int json2iris(const char *json_buff, const char *json_filename,
tmp_obj = cJSON_GetObjectItem(json, "compile_table");
if (tmp_obj) {
compile_tbl_name = tmp_obj->valuestring;
} else {
log_error(logger, MODULE_JSON2IRIS,
"[%s:%d] json file:%s has no [compile_table] field",
__FUNCTION__, __LINE__, json_filename);
goto error_out;
}
tmp_obj = cJSON_GetObjectItem(json, "group2compile_table");
if (tmp_obj) {
group2compile_tbl_name = tmp_obj->valuestring;
} else {
log_error(logger, MODULE_JSON2IRIS,
"[%s:%d] json file:%s has no [group2compile_table] field",
__FUNCTION__, __LINE__, json_filename);
goto error_out;
}
tmp_obj = cJSON_GetObjectItem(json, "group2group_table");
if (tmp_obj) {
group2group_tbl_name = tmp_obj->valuestring;
} else {
log_error(logger, MODULE_JSON2IRIS,
"[%s:%d] json file:%s has no [group2group_table] field",
__FUNCTION__, __LINE__, json_filename);
goto error_out;
}
ret = set_iris_descriptor(json_filename, json, encrypt_key, encrypt_algo,

13
src/maat_api.c
View File

@@ -1185,13 +1185,17 @@ size_t group_to_compile(struct maat *maat_instance, long long *results, size_t n
{
int compile_table_id = -1;
if (state->compile_table_id != 0) {
if (state->compile_table_id > 0) {
compile_table_id = state->compile_table_id;
} else {
compile_table_id = maat_instance->default_compile_table_id;
}
void *compile_rt = table_manager_get_runtime(maat_instance->tbl_mgr, compile_table_id);
if (NULL == compile_rt) {
return 0;
}
size_t n_hit_compile = hit_group_to_compile(compile_rt, results, n_result, state);
assert(state->is_last_scan < LAST_SCAN_FINISHED);
@@ -1800,20 +1804,21 @@ int maat_state_get_hit_paths(struct maat_state *state, struct maat_hit_path *pat
}
int compile_table_id = -1;
if (state->compile_table_id != 0) {
if (state->compile_table_id > 0) {
compile_table_id = state->compile_table_id;
} else {
compile_table_id = maat_instance->default_compile_table_id;
}
void *g2g_runtime = table_manager_get_runtime(maat_instance->tbl_mgr, maat_instance->g2g_table_id);
assert(g2g_runtime != NULL);
size_t internal_hit_path_cnt = maat_compile_state_get_internal_hit_paths(state->compile_state,
(struct group2group_runtime *)g2g_runtime,
paths, n_path);
void *compile_rt = table_manager_get_runtime(maat_instance->tbl_mgr, compile_table_id);
assert(NULL != compile_rt);
if (NULL == compile_rt) {
return internal_hit_path_cnt;
}
return compile_runtime_get_hit_paths((struct compile_runtime *)compile_rt,
state->compile_state, paths, n_path,

10
src/maat_compile.c
View File

@@ -1244,10 +1244,6 @@ static inline int compare_clause_id(const void *a, const void *b)
struct maat_compile_state *maat_compile_state_new(int thread_id)
{
if (thread_id < 0) {
return NULL;
}
struct maat_compile_state *compile_state = ALLOC(struct maat_compile_state, 1);
compile_state->thread_id = thread_id;
@@ -1858,7 +1854,7 @@ int maat_compile_state_update(struct rcu_hash_table *item_htable, int vtable_id,
/* update hit clause */
int compile_table_id = -1;
if (state->compile_table_id != 0) {
if (state->compile_table_id > 0) {
compile_table_id = state->compile_table_id;
} else {
compile_table_id = state->maat_instance->default_compile_table_id;
@@ -1866,7 +1862,9 @@ int maat_compile_state_update(struct rcu_hash_table *item_htable, int vtable_id,
void *compile_rt = table_manager_get_runtime(state->maat_instance->tbl_mgr,
compile_table_id);
assert(compile_rt != NULL);
if (NULL == compile_rt) {
return 0;
}
long long top_group_ids[MAX_SCANNER_HIT_GROUP_NUM];
memset(top_group_ids, -1, sizeof(top_group_ids));

8
src/maat_table.c
View File

@@ -488,8 +488,8 @@ table_manager_create(const char *table_info_path, const char *accept_tags,
}
}
int default_compile_table_id = MAX_TABLE_NUM;
int g2g_table_id = MAX_TABLE_NUM;
int default_compile_table_id = -1;
int g2g_table_id = -1;
struct maat_kv_store *reserved_word_map = maat_kv_store_new();
register_reserved_word(reserved_word_map);
@@ -516,7 +516,9 @@ table_manager_create(const char *table_info_path, const char *accept_tags,
maat_tbl->table_name, maat_tbl->table_id);
if (maat_tbl->table_type == TABLE_TYPE_COMPILE) {
if (maat_tbl->table_id < default_compile_table_id) {
if (default_compile_table_id < 0) {
default_compile_table_id = maat_tbl->table_id;
} else if (maat_tbl->table_id < default_compile_table_id) {
default_compile_table_id = maat_tbl->table_id;
}
}

3
test/CMakeLists.txt
View File

@@ -17,6 +17,9 @@ target_link_libraries(maat_input_mode_gtest maat_frame_static gtest_static)
add_executable(maat_framework_gtest maat_framework_gtest.cpp)
target_link_libraries(maat_framework_gtest maat_frame_static gtest_static)
add_executable(maat_framework_perf_gtest maat_framework_perf_gtest.cpp)
target_link_libraries(maat_framework_perf_gtest maat_frame_static gtest_static)
add_executable(adapter_hs_gtest adapter_hs_gtest.cpp)
target_link_libraries(adapter_hs_gtest maat_frame_static gtest_static)

4
test/maat_framework_gtest.cpp
View File

@@ -4875,7 +4875,7 @@ TEST_F(MaatCmdTest, CompileDelete_TSG6548) {
EXPECT_LE(hit_cnt, miss_cnt);
maat_state_free(state);
}
#if 0
#if 1
TEST_F(MaatCmdTest, UpdateDeadLockDetection) {
const char* g2c_table_name = "GROUP2COMPILE";
const char* compile_table_name = "COMPILE";
@@ -4901,7 +4901,7 @@ TEST_F(MaatCmdTest, UpdateDeadLockDetection) {
"part-1", NULL, 0, 0, 0, 0); /*EXPR_TYPE_STRING MATCH_METHOD_SUB*/
EXPECT_EQ(ret, 1);
sleep(WAIT_FOR_EFFECTIVE_S);
sleep(WAIT_FOR_EFFECTIVE_S * 2);
const char* scan_data1 = "scan string part-1.";
const char* scan_data2 = "scan string part-2.";

602
test/maat_framework_perf_gtest.cpp Normal file
View File

@@ -0,0 +1,602 @@
#include <gtest/gtest.h>
#include "log/log.h"
#include "maat.h"
#include "maat_rule.h"
#include "maat_utils.h"
#include "ip_matcher.h"
#include "maat_command.h"
#include "json2iris.h"
#include "maat_config_monitor.h"
#define MODULE_FRAMEWORK_PERF_GTEST module_name_str("maat.framework_perf_gtest")
#define ARRAY_SIZE 10
#define WAIT_FOR_EFFECTIVE_S 1
#define PERF_THREAD_NUM 5
const char *table_info_path = "./table_info.conf";
const char *json_filename = "maat_json.json";
struct thread_param {
int thread_id;
int test_count;
struct maat *maat_instance;
const char *table_name;
long long time_elapse_ms;
struct log_handle *logger;
};
void random_keyword_generate(char *keyword_buf, size_t sz)
{
#define MIN_KEYWORD_LEN 4
size_t i = 0;
size_t len = random() % (sz - 1 - MIN_KEYWORD_LEN) + MIN_KEYWORD_LEN;
for (i = 0; i < len; i++) {
keyword_buf[i] = 'a' + random() % ('z' - 'a');
}
keyword_buf[i] = '\0';
}
int count_line_num_cb(const char *table_name, const char *line, void *u_para)
{
(*((unsigned int *)u_para))++;
return 0;
}
int line_idx = 0;
long long absolute_expire_time=0;
int make_serial_rule(const char *table_name, const char *line, void *u_para)
{
struct serial_rule *s_rule=(struct serial_rule *)u_para;
int rule_id = 0;
char *buff = ALLOC(char, strlen(line) + 1);
memcpy(buff, line, strlen(line) + 1);
while (buff[strlen(buff) - 1] == '\n' || buff[strlen(buff) - 1] == '\t') {
buff[strlen(buff) - 1] = '\0';
}
int j = 0;
char *str1 = NULL;
char *token = NULL;
char *saveptr1 = NULL;
for (j = 0,str1 = buff; ; j++, str1 = NULL) {
token = strtok_r(str1, "\t ", &saveptr1);
if (token == NULL)
break;
if (j == 0) {
sscanf(token,"%d", &rule_id);
}
}
memcpy(buff, line, strlen(line)+1);
while(buff[strlen(buff)-1]=='\n'||buff[strlen(buff)-1]=='\t') {
buff[strlen(buff)-1]='\0';
}
maat_cmd_set_serial_rule(s_rule + line_idx, MAAT_OP_ADD, rule_id, table_name, buff, absolute_expire_time);
line_idx++;
FREE(buff);
return 0;
}
static int write_config_to_redis(char *redis_ip, int redis_port, int redis_db,
struct log_handle *logger)
{
char json_iris_path[512] = {0};
snprintf(json_iris_path, sizeof(json_iris_path), "./%s_iris_tmp", json_filename);
redisContext *c = maat_cmd_connect_redis(redis_ip, redis_port, redis_db, logger);
if (NULL == c) {
return -1;
}
redisReply *reply = maat_cmd_wrap_redis_command(c, "flushdb");
if (NULL == reply) {
return -1;
}
if (0 == access(json_iris_path, F_OK)) {
system_cmd_rmdir(json_iris_path);
}
if (access(json_iris_path, F_OK) < 0) {
char tmp_iris_path[128] = {0};
char *json_buff = NULL;
size_t json_buff_sz = 0;
int ret = load_file_to_memory(json_filename, (unsigned char **)&json_buff, &json_buff_sz);
if (ret < 0) {
return -1;
}
ret = json2iris(json_buff, json_filename, c, tmp_iris_path,
sizeof(tmp_iris_path), NULL, NULL, logger);
FREE(json_buff);
if (ret < 0) {
return -1;
}
}
size_t total_line_cnt = 0;
char tmp_iris_full_idx_path[PATH_MAX] = {0};
snprintf(tmp_iris_full_idx_path, sizeof(tmp_iris_full_idx_path), "%s/index", json_iris_path);
config_monitor_traverse(0, tmp_iris_full_idx_path, NULL, count_line_num_cb, NULL, &total_line_cnt, logger);
struct serial_rule *s_rule = ALLOC(struct serial_rule, total_line_cnt);
long long server_time = maat_cmd_redis_server_time_s(c);
if (server_time < 0) {
return -1;
}
absolute_expire_time = server_time + 300;
config_monitor_traverse(0, tmp_iris_full_idx_path, NULL, make_serial_rule, NULL, s_rule, logger);
line_idx = 0;
absolute_expire_time = 0;
int success_cnt = 0;
do {
success_cnt = maat_cmd_write_rule(c, s_rule, total_line_cnt, server_time, logger);
} while (success_cnt < 0);
EXPECT_EQ(success_cnt, (int)total_line_cnt);
for (size_t i = 0; i < total_line_cnt; i++) {
maat_cmd_clear_rule_cache(s_rule + i);
}
FREE(s_rule);
redisFree(c);
return 0;
}
static int compile_table_set_line(struct maat *maat_instance, const char *table_name,
enum maat_operation op, long long compile_id,
const char *user_region, int clause_num, int expire_after)
{
char table_line[1024 * 16] = {0};
sprintf(table_line, "%lld\t0\t0\t0\t0\t0\t%s\t%d\t%d\t0.0",
compile_id, user_region, op, clause_num);
struct maat_cmd_line line_rule;
line_rule.rule_id = compile_id;
line_rule.table_line = table_line;
line_rule.table_name = table_name;
line_rule.expire_after = expire_after;
return maat_cmd_set_line(maat_instance, &line_rule);
}
static int group2compile_table_set_line(struct maat *maat_instance, const char *table_name,
enum maat_operation op, long long group_id, long long compile_id,
int not_flag, const char *vtable_name, int clause_num,
int expire_after)
{
char table_line[128] = {0};
sprintf(table_line, "%lld\t%lld\t%d\t%d\t%s\t%d",
group_id, compile_id, op, not_flag, vtable_name, clause_num);
struct maat_cmd_line line_rule;
line_rule.rule_id = group_id;
line_rule.table_line = table_line;
line_rule.table_name = table_name;
line_rule.expire_after = expire_after;
return maat_cmd_set_line(maat_instance, &line_rule);
}
static int expr_table_set_line(struct maat *maat_instance, const char *table_name, enum maat_operation op,
long long item_id, long long group_id, const char *keywords, const char *district,
int expr_type, int match_method, int is_hexbin, int expire_after)
{
char table_line[1024] = {0};
int table_id = maat_get_table_id(maat_instance, table_name);
if (table_id < 0) {
return 0;
}
enum table_type table_type = table_manager_get_table_type(maat_instance->tbl_mgr, table_id);
assert(table_type == TABLE_TYPE_EXPR || table_type == TABLE_TYPE_EXPR_PLUS);
if (table_type == TABLE_TYPE_EXPR_PLUS) {
sprintf(table_line, "%lld\t%lld\t%s\t%s\t%d\t%d\t%d\t%d", item_id, group_id, district,
keywords, expr_type, match_method, is_hexbin, op);
} else {
sprintf(table_line, "%lld\t%lld\t%s\t%d\t%d\t%d\t%d", item_id, group_id, keywords, expr_type,
match_method, is_hexbin, op);
}
struct maat_cmd_line line_rule;
line_rule.rule_id = item_id;
line_rule.table_line = table_line;
line_rule.table_name = table_name;
line_rule.expire_after = expire_after;
return maat_cmd_set_line(maat_instance, &line_rule);
}
static int ip_table_set_line(struct maat *maat_instance, const char *table_name, enum maat_operation op,
long long item_id, long long group_id, enum IP_TYPE type, const char *ip1,
const char *ip2, uint16_t port_min, uint16_t port_max, int expire_after)
{
char table_line[1024] = {0};
int table_id = maat_get_table_id(maat_instance, table_name);
if (table_id < 0) {
return 0;
}
int ip_type = 4;
if (type == IPv6) {
ip_type = 6;
}
sprintf(table_line, "%lld\t%lld\t%d\t%s\t%s\t%s\t%s\t%u\t%u\t6\t%d",
item_id, group_id, ip_type, "range", ip1, ip2,
"range", port_min, port_max, op);
struct maat_cmd_line line_rule;
line_rule.rule_id = item_id;
line_rule.table_line = table_line;
line_rule.table_name = table_name;
line_rule.expire_after = expire_after;
return maat_cmd_set_line(maat_instance, &line_rule);
}
void test_add_expr_command(struct maat *maat_instance, const char *table_name,
const char *keywords)
{
long long compile_id = maat_cmd_incrby(maat_instance, "TEST_SEQ", 1);
int ret = compile_table_set_line(maat_instance, "COMPILE", MAAT_OP_ADD, compile_id, NULL, 1, 0);
EXPECT_EQ(ret, 1);
long long group_id = maat_cmd_incrby(maat_instance, "SEQUENCE_GROUP", 1);
ret = group2compile_table_set_line(maat_instance, "GROUP2COMPILE", MAAT_OP_ADD, group_id,
compile_id, 0, "null", 1, 0);
EXPECT_EQ(ret, 1);
long long item_id = maat_cmd_incrby(maat_instance, "SEQUENCE_REGION", 1);
ret = expr_table_set_line(maat_instance, table_name, MAAT_OP_ADD, item_id, group_id,
keywords, NULL, 1, 0, 0, 0);
EXPECT_EQ(ret, 1);
}
void test_add_ip_command(struct maat *maat_instance, const char *table_name,
const char *ip, uint16_t port)
{
long long compile_id = maat_cmd_incrby(maat_instance, "TEST_SEQ", 1);
int ret = compile_table_set_line(maat_instance, "COMPILE", MAAT_OP_ADD, compile_id, NULL, 1, 0);
EXPECT_EQ(ret, 1);
long long group_id = maat_cmd_incrby(maat_instance, "SEQUENCE_GROUP", 1);
ret = group2compile_table_set_line(maat_instance, "GROUP2COMPILE", MAAT_OP_ADD, group_id,
compile_id, 0, "null", 1, 0);
EXPECT_EQ(ret, 1);
long long item_id = maat_cmd_incrby(maat_instance, "SEQUENCE_REGION", 1);
ret = ip_table_set_line(maat_instance, table_name, MAAT_OP_ADD, item_id, group_id,
IPv4, ip, ip, port, port, 0);
EXPECT_EQ(ret, 1);
}
class MaatPerfStringScan : public testing::Test
{
protected:
static void SetUpTestCase() {
const char *accept_tags = "{\"tags\":[{\"tag\":\"location\",\"value\":\"北京/朝阳/华严北里/甲22号\"},{\"tag\":\"isp\",\"value\":\"移动\"},{\"tag\":\"location\",\"value\":\"Astana\"}]}";
char redis_ip[64] = "127.0.0.1";
int redis_port = 6379;
int redis_db = 0;
logger = log_handle_create("./maat_framework_perf_gtest.log", 0);
int ret = write_config_to_redis(redis_ip, redis_port, redis_db, logger);
if (ret < 0) {
log_error(logger, MODULE_FRAMEWORK_PERF_GTEST,
"[%s:%d] write config to redis failed.", __FUNCTION__, __LINE__);
}
struct maat_options *opts = maat_options_new();
maat_options_set_redis(opts, redis_ip, redis_port, redis_db);
maat_options_set_logger(opts, "./maat_framework_gtest.log", LOG_LEVEL_INFO);
maat_options_set_accept_tags(opts, accept_tags);
maat_options_set_caller_thread_number(opts, 5);
_shared_maat_instance = maat_new(opts, table_info_path);
maat_options_free(opts);
if (NULL == _shared_maat_instance) {
log_error(logger, MODULE_FRAMEWORK_PERF_GTEST,
"[%s:%d] create maat instance in MaatFlagScan failed.",
__FUNCTION__, __LINE__);
}
}
static void TearDownTestCase() {
maat_free(_shared_maat_instance);
log_handle_destroy(logger);
}
static struct log_handle *logger;
static struct maat *_shared_maat_instance;
};
struct maat *MaatPerfStringScan::_shared_maat_instance;
struct log_handle *MaatPerfStringScan::logger;
void *perf_string_scan_thread(void *arg)
{
struct thread_param *param = (struct thread_param *)arg;
struct maat *maat_instance = param->maat_instance;
const char *table_name = param->table_name;
struct timespec start, end;
const char *scan_data = "String TEST should hit";
long long results[ARRAY_SIZE] = {0};
size_t n_hit_result = 0;
struct maat_state *state = maat_state_new(maat_instance, param->thread_id);
int table_id = maat_get_table_id(maat_instance, table_name);
int hit_times = 0;
clock_gettime(CLOCK_MONOTONIC, &start);
for (int i = 0; i < param->test_count; i++) {
int ret = maat_scan_string(maat_instance, table_id, scan_data, strlen(scan_data),
results, ARRAY_SIZE, &n_hit_result, state);
if (ret == MAAT_SCAN_HIT) {
hit_times++;
}
maat_state_reset(state);
}
clock_gettime(CLOCK_MONOTONIC, &end);
param->time_elapse_ms = (end.tv_sec - start.tv_sec) * 1000 + (end.tv_nsec - start.tv_nsec) / 1000000;
int *is_all_hit = ALLOC(int, 1);
*is_all_hit = (hit_times == param->test_count ? 1 : 0);
log_info(param->logger, MODULE_FRAMEWORK_PERF_GTEST,
"thread_id:%d time_elapse:%lldms hit_times:%d",
param->thread_id, param->time_elapse_ms, hit_times);
return is_all_hit;
}
void *perf_string_update_thread(void *arg)
{
struct thread_param *param = (struct thread_param *)arg;
struct maat *maat_instance = param->maat_instance;
const char *table_name = param->table_name;
const int CMD_EXPR_NUM = 10;
char keyword_buf[128];
for (int i = 0; i < CMD_EXPR_NUM; i++) {
random_keyword_generate(keyword_buf, sizeof(keyword_buf));
test_add_expr_command(maat_instance, table_name, keyword_buf);
sleep(1);
}
int *is_all_hit = ALLOC(int, 1);
*is_all_hit = 1;
return is_all_hit;
}
void *perf_ip_scan_thread(void *arg)
{
struct thread_param *param = (struct thread_param *)arg;
struct maat *maat_instance = param->maat_instance;
const char *table_name = param->table_name;
struct timespec start, end;
char ip_str[32] = "10.0.7.100";
uint32_t ip_addr;
uint16_t port = htons(65530);
int ret = inet_pton(AF_INET, ip_str, &ip_addr);
EXPECT_EQ(ret, 1);
int table_id = maat_get_table_id(maat_instance, table_name);
int hit_times = 0;
int not_hit_times = 0;
clock_gettime(CLOCK_MONOTONIC, &start);
long long results[ARRAY_SIZE] = {0};
size_t n_hit_result = 0;
printf("param->thread_id:%d\n", param->thread_id);
struct maat_state *state = maat_state_new(maat_instance, param->thread_id);
for (int i = 0; i < param->test_count; i++) {
int ret = maat_scan_ipv4(maat_instance, table_id, ip_addr, port, 6,
results, ARRAY_SIZE, &n_hit_result, state);
if (ret == MAAT_SCAN_HIT) {
hit_times++;
} else {
not_hit_times++;
log_info(param->logger, MODULE_FRAMEWORK_PERF_GTEST,
"thread_id:%d no hit_times:%d ", param->thread_id, not_hit_times);
}
maat_state_reset(state);
}
clock_gettime(CLOCK_MONOTONIC, &end);
param->time_elapse_ms = (end.tv_sec - start.tv_sec) * 1000 + (end.tv_nsec - start.tv_nsec) / 1000000;
int *is_all_hit = ALLOC(int, 1);
*is_all_hit = (hit_times == param->test_count ? 1 : 0);
log_info(param->logger, MODULE_FRAMEWORK_PERF_GTEST,
"thread_id:%d time_elapse:%lldms hit_times:%d",
param->thread_id, param->time_elapse_ms, hit_times);
return is_all_hit;
}
void *perf_ip_update_thread(void *arg)
{
struct thread_param *param = (struct thread_param *)arg;
struct maat *maat_instance = param->maat_instance;
const char *table_name = param->table_name;
const int CMD_EXPR_NUM = 10;
char ip_str[10][32] = {
"10.0.7.101",
"10.0.7.102",
"10.0.7.103",
"10.0.7.104",
"10.0.7.105",
"10.0.7.106",
"10.0.7.107",
"10.0.7.108",
"10.0.7.109",
"10.0.7.110"};
uint16_t port = 65530;
for (int i = 0; i < CMD_EXPR_NUM; i++) {
test_add_ip_command(maat_instance, table_name, ip_str[i], port);
sleep(1);
}
int *is_all_hit = ALLOC(int, 1);
*is_all_hit = 1;
return is_all_hit;
}
TEST_F(MaatPerfStringScan, basic) {
const char *table_name = "KEYWORDS_TABLE";
struct maat *maat_instance = MaatPerfStringScan::_shared_maat_instance;
int table_id = maat_get_table_id(maat_instance, table_name);
ASSERT_GT(table_id, 0);
pthread_t threads[PERF_THREAD_NUM + 1];
struct thread_param thread_params[PERF_THREAD_NUM + 1];
int i = 0;
int *is_all_hit = NULL;
for (i = 0; i < PERF_THREAD_NUM + 1; i++) {
thread_params[i].maat_instance = maat_instance;
thread_params[i].thread_id = i;
thread_params[i].table_name = table_name;
thread_params[i].test_count = 1 * 1000 * 1000;
thread_params[i].time_elapse_ms = 0;
thread_params[i].logger = logger;
if (i < PERF_THREAD_NUM) {
pthread_create(&threads[i], NULL, perf_string_scan_thread, thread_params+i);
} else {
thread_params[i].test_count = 0;
pthread_create(&threads[i], NULL, perf_string_update_thread, thread_params+i);
}
}
long long time_elapse_ms = 0;
long long scan_count = 0;
long long scan_per_second = 0;
for (i = 0; i < PERF_THREAD_NUM + 1; i++) {
pthread_join(threads[i], (void **)&is_all_hit);
time_elapse_ms += thread_params[i].time_elapse_ms;
scan_count += thread_params[i].test_count;
EXPECT_EQ(*is_all_hit, 1);
*is_all_hit = 0;
free(is_all_hit);
}
scan_per_second = scan_count * 1000 / time_elapse_ms;
//EXPECT_GT(scan_per_second, 800 * 1000);
printf("High match rate on %d-threads speed %lld lookups/s/thread\n", PERF_THREAD_NUM, scan_per_second);
}
class MaatPerfIPScan : public testing::Test
{
protected:
static void SetUpTestCase() {
const char *accept_tags = "{\"tags\":[{\"tag\":\"location\",\"value\":\"北京/朝阳/华严北里/甲22号\"},{\"tag\":\"isp\",\"value\":\"移动\"},{\"tag\":\"location\",\"value\":\"Astana\"}]}";
char redis_ip[64] = "127.0.0.1";
int redis_port = 6379;
int redis_db = 0;
logger = log_handle_create("./maat_framework_perf_gtest.log", 0);
int ret = write_config_to_redis(redis_ip, redis_port, redis_db, logger);
if (ret < 0) {
log_error(logger, MODULE_FRAMEWORK_PERF_GTEST,
"[%s:%d] write config to redis failed.", __FUNCTION__, __LINE__);
}
struct maat_options *opts = maat_options_new();
maat_options_set_redis(opts, redis_ip, redis_port, redis_db);
maat_options_set_logger(opts, "./maat_framework_perf_gtest.log", LOG_LEVEL_INFO);
maat_options_set_accept_tags(opts, accept_tags);
maat_options_set_caller_thread_number(opts, 5);
_shared_maat_instance = maat_new(opts, table_info_path);
maat_options_free(opts);
if (NULL == _shared_maat_instance) {
log_error(logger, MODULE_FRAMEWORK_PERF_GTEST,
"[%s:%d] create maat instance in MaatFlagScan failed.",
__FUNCTION__, __LINE__);
}
}
static void TearDownTestCase() {
maat_free(_shared_maat_instance);
log_handle_destroy(logger);
}
static struct log_handle *logger;
static struct maat *_shared_maat_instance;
};
struct maat *MaatPerfIPScan::_shared_maat_instance;
struct log_handle *MaatPerfIPScan::logger;
TEST_F(MaatPerfIPScan, basic)
{
const char *table_name = "IP_PLUS_CONFIG";
struct maat *maat_instance = MaatPerfIPScan::_shared_maat_instance;
int table_id = maat_get_table_id(maat_instance, table_name);
ASSERT_GT(table_id, 0);
pthread_t threads[PERF_THREAD_NUM + 1];
struct thread_param thread_params[PERF_THREAD_NUM + 1];
int i = 0;
int *is_all_hit = NULL;
for (i = 0; i < PERF_THREAD_NUM + 1; i++) {
thread_params[i].maat_instance = maat_instance;
thread_params[i].thread_id = i;
thread_params[i].table_name = table_name;
thread_params[i].test_count = 1 * 1000 * 1000;
thread_params[i].time_elapse_ms = 0;
thread_params[i].logger = logger;
if (i < PERF_THREAD_NUM) {
pthread_create(&threads[i], NULL, perf_ip_scan_thread, thread_params+i);
} else {
thread_params[i].test_count = 0;
pthread_create(&threads[i], NULL, perf_ip_update_thread, thread_params+i);
}
}
long long time_elapse_ms = 0;
long long scan_count = 0;
long long scan_per_second = 0;
for (i = 0; i < PERF_THREAD_NUM + 1; i++) {
pthread_join(threads[i], (void **)&is_all_hit);
time_elapse_ms += thread_params[i].time_elapse_ms;
scan_count += thread_params[i].test_count;
EXPECT_EQ(*is_all_hit, 1);
*is_all_hit = 0;
free(is_all_hit);
}
scan_per_second = scan_count * 1000 / time_elapse_ms;
//EXPECT_GT(scan_per_second, 800 * 1000);
printf("High match rate on %d-threads speed %lld lookups/s/thread\n", PERF_THREAD_NUM, scan_per_second);
}
int main(int argc, char ** argv)
{
int ret=0;
::testing::InitGoogleTest(&argc, argv);
ret=RUN_ALL_TESTS();
return ret;
}

82
test/rcu_hash_gtest.cpp
View File

@@ -457,6 +457,88 @@ TEST(global_rcu_hash_del, single_thread) {
res = rcu_hash_find(g_htable, key, key_len);
EXPECT_TRUE(res == NULL);
}
#define THREAD_NUM 10
struct thread_param {
int thread_id;
int test_count;
unsigned long long time_elapse_ms;
};
void *rcu_scan_thread(void *arg)
{
struct thread_param *param = (struct thread_param *)arg;
int hit_cnt = 0;
void **data_array = NULL;
sleep(2);
for (int index = 0; index < param->test_count; index++) {
size_t count = rcu_hash_list(g_htable, &data_array);
for (size_t i = 0; i < count; i++) {
struct user_data *user_data = (struct user_data *)data_array[i];
if (user_data != NULL && user_data->id == 101) {
hit_cnt++;
}
}
}
int *is_all_hit = ALLOC(int, 1);
*is_all_hit = (hit_cnt == param->test_count) ? 1 : 0;
printf("thread[%d]: hit_cnt:%d\n", param->thread_id, hit_cnt);
return is_all_hit;
}
void *rcu_update_thread(void *arg)
{
const char *key = "http_url";
size_t key_len = strlen(key);
for (int i = 0; i < 10; i++) {
rcu_hash_del(g_htable, key, key_len);
rcu_hash_commit(g_htable);
sleep(1);
struct user_data *data = ALLOC(struct user_data, 1);
data->id = 101;
char name[64] = "www.baidu.com";
memcpy(data->name, name, strlen(name));
rcu_hash_add(g_htable, key, key_len, data);
rcu_hash_commit(g_htable);
}
int *is_all_hit = ALLOC(int, 1);
*is_all_hit = 1;
return is_all_hit;
}
TEST(rcu_hash_update, multi_thread) {
pthread_t threads[THREAD_NUM + 1];
struct thread_param t_param[THREAD_NUM + 1];
for (int i = 0; i < THREAD_NUM + 1; i++) {
t_param[i].thread_id = i;
t_param[i].test_count = 1000 * 1000;
t_param[i].time_elapse_ms = 0;
if (i < THREAD_NUM) {
pthread_create(&threads[i], NULL, rcu_scan_thread, t_param+i);
} else {
pthread_create(&threads[i], NULL, rcu_update_thread, t_param+i);
}
}
int *is_all_hit = NULL;
unsigned long long time_elapse_ms = 0, scan_count = 0;
for (int i = 0; i < THREAD_NUM + 1; i++) {
pthread_join(threads[i], (void **)&is_all_hit);
time_elapse_ms += t_param[i].time_elapse_ms;
scan_count += t_param[i].test_count;
EXPECT_EQ(*is_all_hit, 1);
*is_all_hit = 0;
free(is_all_hit);
}
}
int main(int argc, char ** argv)
{