diff --git a/src/json2iris.c b/src/json2iris.c index 08fa729..c0a1d2a 100644 --- a/src/json2iris.c +++ b/src/json2iris.c @@ -1094,17 +1094,32 @@ int json2iris(const char *json_buff, const char *json_filename, tmp_obj = cJSON_GetObjectItem(json, "compile_table"); if (tmp_obj) { compile_tbl_name = tmp_obj->valuestring; - } + } else { + log_error(logger, MODULE_JSON2IRIS, + "[%s:%d] json file:%s has no [compile_table] field", + __FUNCTION__, __LINE__, json_filename); + goto error_out; + } tmp_obj = cJSON_GetObjectItem(json, "group2compile_table"); if (tmp_obj) { group2compile_tbl_name = tmp_obj->valuestring; - } + } else { + log_error(logger, MODULE_JSON2IRIS, + "[%s:%d] json file:%s has no [group2compile_table] field", + __FUNCTION__, __LINE__, json_filename); + goto error_out; + } tmp_obj = cJSON_GetObjectItem(json, "group2group_table"); if (tmp_obj) { group2group_tbl_name = tmp_obj->valuestring; - } + } else { + log_error(logger, MODULE_JSON2IRIS, + "[%s:%d] json file:%s has no [group2group_table] field", + __FUNCTION__, __LINE__, json_filename); + goto error_out; + } ret = set_iris_descriptor(json_filename, json, encrypt_key, encrypt_algo, compile_tbl_name, group2compile_tbl_name, diff --git a/src/maat_api.c b/src/maat_api.c index 0bbc31c..1dcf9f0 100644 --- a/src/maat_api.c +++ b/src/maat_api.c @@ -1185,13 +1185,17 @@ size_t group_to_compile(struct maat *maat_instance, long long *results, size_t n { int compile_table_id = -1; - if (state->compile_table_id != 0) { + if (state->compile_table_id > 0) { compile_table_id = state->compile_table_id; } else { compile_table_id = maat_instance->default_compile_table_id; } void *compile_rt = table_manager_get_runtime(maat_instance->tbl_mgr, compile_table_id); + if (NULL == compile_rt) { + return 0; + } + size_t n_hit_compile = hit_group_to_compile(compile_rt, results, n_result, state); assert(state->is_last_scan < LAST_SCAN_FINISHED); @@ -1800,20 +1804,21 @@ int maat_state_get_hit_paths(struct maat_state *state, struct maat_hit_path *pat } int compile_table_id = -1; - if (state->compile_table_id != 0) { + if (state->compile_table_id > 0) { compile_table_id = state->compile_table_id; } else { compile_table_id = maat_instance->default_compile_table_id; } void *g2g_runtime = table_manager_get_runtime(maat_instance->tbl_mgr, maat_instance->g2g_table_id); - assert(g2g_runtime != NULL); size_t internal_hit_path_cnt = maat_compile_state_get_internal_hit_paths(state->compile_state, (struct group2group_runtime *)g2g_runtime, paths, n_path); void *compile_rt = table_manager_get_runtime(maat_instance->tbl_mgr, compile_table_id); - assert(NULL != compile_rt); + if (NULL == compile_rt) { + return internal_hit_path_cnt; + } return compile_runtime_get_hit_paths((struct compile_runtime *)compile_rt, state->compile_state, paths, n_path, diff --git a/src/maat_compile.c b/src/maat_compile.c index db2c1d0..3f7cb52 100644 --- a/src/maat_compile.c +++ b/src/maat_compile.c @@ -1244,10 +1244,6 @@ static inline int compare_clause_id(const void *a, const void *b) struct maat_compile_state *maat_compile_state_new(int thread_id) { - if (thread_id < 0) { - return NULL; - } - struct maat_compile_state *compile_state = ALLOC(struct maat_compile_state, 1); compile_state->thread_id = thread_id; @@ -1858,7 +1854,7 @@ int maat_compile_state_update(struct rcu_hash_table *item_htable, int vtable_id, /* update hit clause */ int compile_table_id = -1; - if (state->compile_table_id != 0) { + if (state->compile_table_id > 0) { compile_table_id = state->compile_table_id; } else { compile_table_id = state->maat_instance->default_compile_table_id; @@ -1866,7 +1862,9 @@ int maat_compile_state_update(struct rcu_hash_table *item_htable, int vtable_id, void *compile_rt = table_manager_get_runtime(state->maat_instance->tbl_mgr, compile_table_id); - assert(compile_rt != NULL); + if (NULL == compile_rt) { + return 0; + } long long top_group_ids[MAX_SCANNER_HIT_GROUP_NUM]; memset(top_group_ids, -1, sizeof(top_group_ids)); diff --git a/src/maat_table.c b/src/maat_table.c index 36e4f54..711e8d8 100644 --- a/src/maat_table.c +++ b/src/maat_table.c @@ -488,8 +488,8 @@ table_manager_create(const char *table_info_path, const char *accept_tags, } } - int default_compile_table_id = MAX_TABLE_NUM; - int g2g_table_id = MAX_TABLE_NUM; + int default_compile_table_id = -1; + int g2g_table_id = -1; struct maat_kv_store *reserved_word_map = maat_kv_store_new(); register_reserved_word(reserved_word_map); @@ -516,12 +516,14 @@ table_manager_create(const char *table_info_path, const char *accept_tags, maat_tbl->table_name, maat_tbl->table_id); if (maat_tbl->table_type == TABLE_TYPE_COMPILE) { - if (maat_tbl->table_id < default_compile_table_id) { + if (default_compile_table_id < 0) { + default_compile_table_id = maat_tbl->table_id; + } else if (maat_tbl->table_id < default_compile_table_id) { default_compile_table_id = maat_tbl->table_id; } } - if (maat_tbl->table_type == TABLE_TYPE_GROUP2GROUP) { + if (maat_tbl->table_type == TABLE_TYPE_GROUP2GROUP) { g2g_table_id = maat_tbl->table_id; } diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 5748ed2..32a935e 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -17,6 +17,9 @@ target_link_libraries(maat_input_mode_gtest maat_frame_static gtest_static) add_executable(maat_framework_gtest maat_framework_gtest.cpp) target_link_libraries(maat_framework_gtest maat_frame_static gtest_static) +add_executable(maat_framework_perf_gtest maat_framework_perf_gtest.cpp) +target_link_libraries(maat_framework_perf_gtest maat_frame_static gtest_static) + add_executable(adapter_hs_gtest adapter_hs_gtest.cpp) target_link_libraries(adapter_hs_gtest maat_frame_static gtest_static) diff --git a/test/maat_framework_gtest.cpp b/test/maat_framework_gtest.cpp index a05a71f..7cbdc85 100644 --- a/test/maat_framework_gtest.cpp +++ b/test/maat_framework_gtest.cpp @@ -4875,7 +4875,7 @@ TEST_F(MaatCmdTest, CompileDelete_TSG6548) { EXPECT_LE(hit_cnt, miss_cnt); maat_state_free(state); } -#if 0 +#if 1 TEST_F(MaatCmdTest, UpdateDeadLockDetection) { const char* g2c_table_name = "GROUP2COMPILE"; const char* compile_table_name = "COMPILE"; @@ -4901,7 +4901,7 @@ TEST_F(MaatCmdTest, UpdateDeadLockDetection) { "part-1", NULL, 0, 0, 0, 0); /*EXPR_TYPE_STRING MATCH_METHOD_SUB*/ EXPECT_EQ(ret, 1); - sleep(WAIT_FOR_EFFECTIVE_S); + sleep(WAIT_FOR_EFFECTIVE_S * 2); const char* scan_data1 = "scan string part-1."; const char* scan_data2 = "scan string part-2."; diff --git a/test/maat_framework_perf_gtest.cpp b/test/maat_framework_perf_gtest.cpp new file mode 100644 index 0000000..51b65c4 --- /dev/null +++ b/test/maat_framework_perf_gtest.cpp @@ -0,0 +1,602 @@ +#include + +#include "log/log.h" +#include "maat.h" +#include "maat_rule.h" +#include "maat_utils.h" +#include "ip_matcher.h" +#include "maat_command.h" +#include "json2iris.h" +#include "maat_config_monitor.h" + +#define MODULE_FRAMEWORK_PERF_GTEST module_name_str("maat.framework_perf_gtest") + +#define ARRAY_SIZE 10 +#define WAIT_FOR_EFFECTIVE_S 1 +#define PERF_THREAD_NUM 5 + +const char *table_info_path = "./table_info.conf"; +const char *json_filename = "maat_json.json"; + +struct thread_param { + int thread_id; + int test_count; + struct maat *maat_instance; + const char *table_name; + long long time_elapse_ms; + struct log_handle *logger; +}; + +void random_keyword_generate(char *keyword_buf, size_t sz) +{ +#define MIN_KEYWORD_LEN 4 + size_t i = 0; + size_t len = random() % (sz - 1 - MIN_KEYWORD_LEN) + MIN_KEYWORD_LEN; + for (i = 0; i < len; i++) { + keyword_buf[i] = 'a' + random() % ('z' - 'a'); + } + keyword_buf[i] = '\0'; +} + +int count_line_num_cb(const char *table_name, const char *line, void *u_para) +{ + (*((unsigned int *)u_para))++; + return 0; +} + +int line_idx = 0; +long long absolute_expire_time=0; +int make_serial_rule(const char *table_name, const char *line, void *u_para) +{ + struct serial_rule *s_rule=(struct serial_rule *)u_para; + int rule_id = 0; + char *buff = ALLOC(char, strlen(line) + 1); + + memcpy(buff, line, strlen(line) + 1); + + while (buff[strlen(buff) - 1] == '\n' || buff[strlen(buff) - 1] == '\t') { + buff[strlen(buff) - 1] = '\0'; + } + + int j = 0; + char *str1 = NULL; + char *token = NULL; + char *saveptr1 = NULL; + + for (j = 0,str1 = buff; ; j++, str1 = NULL) { + token = strtok_r(str1, "\t ", &saveptr1); + if (token == NULL) + break; + if (j == 0) { + sscanf(token,"%d", &rule_id); + } + } + + memcpy(buff, line, strlen(line)+1); + while(buff[strlen(buff)-1]=='\n'||buff[strlen(buff)-1]=='\t') { + buff[strlen(buff)-1]='\0'; + } + + maat_cmd_set_serial_rule(s_rule + line_idx, MAAT_OP_ADD, rule_id, table_name, buff, absolute_expire_time); + line_idx++; + + FREE(buff); + + return 0; +} + +static int write_config_to_redis(char *redis_ip, int redis_port, int redis_db, + struct log_handle *logger) +{ + char json_iris_path[512] = {0}; + + snprintf(json_iris_path, sizeof(json_iris_path), "./%s_iris_tmp", json_filename); + + redisContext *c = maat_cmd_connect_redis(redis_ip, redis_port, redis_db, logger); + if (NULL == c) { + return -1; + } + + redisReply *reply = maat_cmd_wrap_redis_command(c, "flushdb"); + if (NULL == reply) { + return -1; + } + + if (0 == access(json_iris_path, F_OK)) { + system_cmd_rmdir(json_iris_path); + } + + if (access(json_iris_path, F_OK) < 0) { + char tmp_iris_path[128] = {0}; + char *json_buff = NULL; + size_t json_buff_sz = 0; + + int ret = load_file_to_memory(json_filename, (unsigned char **)&json_buff, &json_buff_sz); + if (ret < 0) { + return -1; + } + + ret = json2iris(json_buff, json_filename, c, tmp_iris_path, + sizeof(tmp_iris_path), NULL, NULL, logger); + FREE(json_buff); + if (ret < 0) { + return -1; + } + } + + size_t total_line_cnt = 0; + char tmp_iris_full_idx_path[PATH_MAX] = {0}; + snprintf(tmp_iris_full_idx_path, sizeof(tmp_iris_full_idx_path), "%s/index", json_iris_path); + config_monitor_traverse(0, tmp_iris_full_idx_path, NULL, count_line_num_cb, NULL, &total_line_cnt, logger); + + struct serial_rule *s_rule = ALLOC(struct serial_rule, total_line_cnt); + long long server_time = maat_cmd_redis_server_time_s(c); + if (server_time < 0) { + return -1; + } + + absolute_expire_time = server_time + 300; + config_monitor_traverse(0, tmp_iris_full_idx_path, NULL, make_serial_rule, NULL, s_rule, logger); + line_idx = 0; + absolute_expire_time = 0; + + int success_cnt = 0; + do { + success_cnt = maat_cmd_write_rule(c, s_rule, total_line_cnt, server_time, logger); + } while (success_cnt < 0); + + EXPECT_EQ(success_cnt, (int)total_line_cnt); + + for (size_t i = 0; i < total_line_cnt; i++) { + maat_cmd_clear_rule_cache(s_rule + i); + } + FREE(s_rule); + redisFree(c); + + return 0; +} + +static int compile_table_set_line(struct maat *maat_instance, const char *table_name, + enum maat_operation op, long long compile_id, + const char *user_region, int clause_num, int expire_after) +{ + char table_line[1024 * 16] = {0}; + sprintf(table_line, "%lld\t0\t0\t0\t0\t0\t%s\t%d\t%d\t0.0", + compile_id, user_region, op, clause_num); + + struct maat_cmd_line line_rule; + line_rule.rule_id = compile_id; + line_rule.table_line = table_line; + line_rule.table_name = table_name; + line_rule.expire_after = expire_after; + + return maat_cmd_set_line(maat_instance, &line_rule); +} + +static int group2compile_table_set_line(struct maat *maat_instance, const char *table_name, + enum maat_operation op, long long group_id, long long compile_id, + int not_flag, const char *vtable_name, int clause_num, + int expire_after) +{ + char table_line[128] = {0}; + sprintf(table_line, "%lld\t%lld\t%d\t%d\t%s\t%d", + group_id, compile_id, op, not_flag, vtable_name, clause_num); + + struct maat_cmd_line line_rule; + line_rule.rule_id = group_id; + line_rule.table_line = table_line; + line_rule.table_name = table_name; + line_rule.expire_after = expire_after; + + return maat_cmd_set_line(maat_instance, &line_rule); +} + +static int expr_table_set_line(struct maat *maat_instance, const char *table_name, enum maat_operation op, + long long item_id, long long group_id, const char *keywords, const char *district, + int expr_type, int match_method, int is_hexbin, int expire_after) +{ + char table_line[1024] = {0}; + int table_id = maat_get_table_id(maat_instance, table_name); + if (table_id < 0) { + return 0; + } + + enum table_type table_type = table_manager_get_table_type(maat_instance->tbl_mgr, table_id); + assert(table_type == TABLE_TYPE_EXPR || table_type == TABLE_TYPE_EXPR_PLUS); + + if (table_type == TABLE_TYPE_EXPR_PLUS) { + sprintf(table_line, "%lld\t%lld\t%s\t%s\t%d\t%d\t%d\t%d", item_id, group_id, district, + keywords, expr_type, match_method, is_hexbin, op); + } else { + sprintf(table_line, "%lld\t%lld\t%s\t%d\t%d\t%d\t%d", item_id, group_id, keywords, expr_type, + match_method, is_hexbin, op); + } + + struct maat_cmd_line line_rule; + line_rule.rule_id = item_id; + line_rule.table_line = table_line; + line_rule.table_name = table_name; + line_rule.expire_after = expire_after; + + return maat_cmd_set_line(maat_instance, &line_rule); +} + +static int ip_table_set_line(struct maat *maat_instance, const char *table_name, enum maat_operation op, + long long item_id, long long group_id, enum IP_TYPE type, const char *ip1, + const char *ip2, uint16_t port_min, uint16_t port_max, int expire_after) +{ + char table_line[1024] = {0}; + int table_id = maat_get_table_id(maat_instance, table_name); + if (table_id < 0) { + return 0; + } + + int ip_type = 4; + if (type == IPv6) { + ip_type = 6; + } + + sprintf(table_line, "%lld\t%lld\t%d\t%s\t%s\t%s\t%s\t%u\t%u\t6\t%d", + item_id, group_id, ip_type, "range", ip1, ip2, + "range", port_min, port_max, op); + struct maat_cmd_line line_rule; + + line_rule.rule_id = item_id; + line_rule.table_line = table_line; + line_rule.table_name = table_name; + line_rule.expire_after = expire_after; + + return maat_cmd_set_line(maat_instance, &line_rule); +} + +void test_add_expr_command(struct maat *maat_instance, const char *table_name, + const char *keywords) +{ + long long compile_id = maat_cmd_incrby(maat_instance, "TEST_SEQ", 1); + int ret = compile_table_set_line(maat_instance, "COMPILE", MAAT_OP_ADD, compile_id, NULL, 1, 0); + EXPECT_EQ(ret, 1); + + long long group_id = maat_cmd_incrby(maat_instance, "SEQUENCE_GROUP", 1); + ret = group2compile_table_set_line(maat_instance, "GROUP2COMPILE", MAAT_OP_ADD, group_id, + compile_id, 0, "null", 1, 0); + EXPECT_EQ(ret, 1); + + long long item_id = maat_cmd_incrby(maat_instance, "SEQUENCE_REGION", 1); + ret = expr_table_set_line(maat_instance, table_name, MAAT_OP_ADD, item_id, group_id, + keywords, NULL, 1, 0, 0, 0); + EXPECT_EQ(ret, 1); +} + +void test_add_ip_command(struct maat *maat_instance, const char *table_name, + const char *ip, uint16_t port) +{ + long long compile_id = maat_cmd_incrby(maat_instance, "TEST_SEQ", 1); + int ret = compile_table_set_line(maat_instance, "COMPILE", MAAT_OP_ADD, compile_id, NULL, 1, 0); + EXPECT_EQ(ret, 1); + + long long group_id = maat_cmd_incrby(maat_instance, "SEQUENCE_GROUP", 1); + ret = group2compile_table_set_line(maat_instance, "GROUP2COMPILE", MAAT_OP_ADD, group_id, + compile_id, 0, "null", 1, 0); + EXPECT_EQ(ret, 1); + + long long item_id = maat_cmd_incrby(maat_instance, "SEQUENCE_REGION", 1); + ret = ip_table_set_line(maat_instance, table_name, MAAT_OP_ADD, item_id, group_id, + IPv4, ip, ip, port, port, 0); + EXPECT_EQ(ret, 1); +} + +class MaatPerfStringScan : public testing::Test +{ +protected: + static void SetUpTestCase() { + const char *accept_tags = "{\"tags\":[{\"tag\":\"location\",\"value\":\"北京/朝阳/华严北里/甲22号\"},{\"tag\":\"isp\",\"value\":\"移动\"},{\"tag\":\"location\",\"value\":\"Astana\"}]}"; + char redis_ip[64] = "127.0.0.1"; + int redis_port = 6379; + int redis_db = 0; + + logger = log_handle_create("./maat_framework_perf_gtest.log", 0); + int ret = write_config_to_redis(redis_ip, redis_port, redis_db, logger); + if (ret < 0) { + log_error(logger, MODULE_FRAMEWORK_PERF_GTEST, + "[%s:%d] write config to redis failed.", __FUNCTION__, __LINE__); + } + + struct maat_options *opts = maat_options_new(); + maat_options_set_redis(opts, redis_ip, redis_port, redis_db); + maat_options_set_logger(opts, "./maat_framework_gtest.log", LOG_LEVEL_INFO); + maat_options_set_accept_tags(opts, accept_tags); + maat_options_set_caller_thread_number(opts, 5); + + _shared_maat_instance = maat_new(opts, table_info_path); + maat_options_free(opts); + if (NULL == _shared_maat_instance) { + log_error(logger, MODULE_FRAMEWORK_PERF_GTEST, + "[%s:%d] create maat instance in MaatFlagScan failed.", + __FUNCTION__, __LINE__); + } + } + + static void TearDownTestCase() { + maat_free(_shared_maat_instance); + log_handle_destroy(logger); + } + + static struct log_handle *logger; + static struct maat *_shared_maat_instance; +}; + +struct maat *MaatPerfStringScan::_shared_maat_instance; +struct log_handle *MaatPerfStringScan::logger; + +void *perf_string_scan_thread(void *arg) +{ + struct thread_param *param = (struct thread_param *)arg; + struct maat *maat_instance = param->maat_instance; + const char *table_name = param->table_name; + struct timespec start, end; + const char *scan_data = "String TEST should hit"; + long long results[ARRAY_SIZE] = {0}; + size_t n_hit_result = 0; + struct maat_state *state = maat_state_new(maat_instance, param->thread_id); + + int table_id = maat_get_table_id(maat_instance, table_name); + int hit_times = 0; + + clock_gettime(CLOCK_MONOTONIC, &start); + for (int i = 0; i < param->test_count; i++) { + int ret = maat_scan_string(maat_instance, table_id, scan_data, strlen(scan_data), + results, ARRAY_SIZE, &n_hit_result, state); + if (ret == MAAT_SCAN_HIT) { + hit_times++; + } + maat_state_reset(state); + } + clock_gettime(CLOCK_MONOTONIC, &end); + + param->time_elapse_ms = (end.tv_sec - start.tv_sec) * 1000 + (end.tv_nsec - start.tv_nsec) / 1000000; + int *is_all_hit = ALLOC(int, 1); + *is_all_hit = (hit_times == param->test_count ? 1 : 0); + log_info(param->logger, MODULE_FRAMEWORK_PERF_GTEST, + "thread_id:%d time_elapse:%lldms hit_times:%d", + param->thread_id, param->time_elapse_ms, hit_times); + return is_all_hit; +} + +void *perf_string_update_thread(void *arg) +{ + struct thread_param *param = (struct thread_param *)arg; + struct maat *maat_instance = param->maat_instance; + const char *table_name = param->table_name; + const int CMD_EXPR_NUM = 10; + char keyword_buf[128]; + + for (int i = 0; i < CMD_EXPR_NUM; i++) { + random_keyword_generate(keyword_buf, sizeof(keyword_buf)); + test_add_expr_command(maat_instance, table_name, keyword_buf); + sleep(1); + } + + int *is_all_hit = ALLOC(int, 1); + *is_all_hit = 1; + + return is_all_hit; +} + +void *perf_ip_scan_thread(void *arg) +{ + struct thread_param *param = (struct thread_param *)arg; + struct maat *maat_instance = param->maat_instance; + const char *table_name = param->table_name; + struct timespec start, end; + char ip_str[32] = "10.0.7.100"; + uint32_t ip_addr; + uint16_t port = htons(65530); + + int ret = inet_pton(AF_INET, ip_str, &ip_addr); + EXPECT_EQ(ret, 1); + + int table_id = maat_get_table_id(maat_instance, table_name); + int hit_times = 0; + int not_hit_times = 0; + clock_gettime(CLOCK_MONOTONIC, &start); + + long long results[ARRAY_SIZE] = {0}; + size_t n_hit_result = 0; + printf("param->thread_id:%d\n", param->thread_id); + struct maat_state *state = maat_state_new(maat_instance, param->thread_id); + for (int i = 0; i < param->test_count; i++) { + int ret = maat_scan_ipv4(maat_instance, table_id, ip_addr, port, 6, + results, ARRAY_SIZE, &n_hit_result, state); + if (ret == MAAT_SCAN_HIT) { + hit_times++; + } else { + not_hit_times++; + log_info(param->logger, MODULE_FRAMEWORK_PERF_GTEST, + "thread_id:%d no hit_times:%d ", param->thread_id, not_hit_times); + } + maat_state_reset(state); + } + clock_gettime(CLOCK_MONOTONIC, &end); + + param->time_elapse_ms = (end.tv_sec - start.tv_sec) * 1000 + (end.tv_nsec - start.tv_nsec) / 1000000; + int *is_all_hit = ALLOC(int, 1); + *is_all_hit = (hit_times == param->test_count ? 1 : 0); + log_info(param->logger, MODULE_FRAMEWORK_PERF_GTEST, + "thread_id:%d time_elapse:%lldms hit_times:%d", + param->thread_id, param->time_elapse_ms, hit_times); + return is_all_hit; +} + +void *perf_ip_update_thread(void *arg) +{ + struct thread_param *param = (struct thread_param *)arg; + struct maat *maat_instance = param->maat_instance; + const char *table_name = param->table_name; + const int CMD_EXPR_NUM = 10; + char ip_str[10][32] = { + "10.0.7.101", + "10.0.7.102", + "10.0.7.103", + "10.0.7.104", + "10.0.7.105", + "10.0.7.106", + "10.0.7.107", + "10.0.7.108", + "10.0.7.109", + "10.0.7.110"}; + + uint16_t port = 65530; + for (int i = 0; i < CMD_EXPR_NUM; i++) { + test_add_ip_command(maat_instance, table_name, ip_str[i], port); + sleep(1); + } + + int *is_all_hit = ALLOC(int, 1); + *is_all_hit = 1; + + return is_all_hit; +} + +TEST_F(MaatPerfStringScan, basic) { + const char *table_name = "KEYWORDS_TABLE"; + struct maat *maat_instance = MaatPerfStringScan::_shared_maat_instance; + + int table_id = maat_get_table_id(maat_instance, table_name); + ASSERT_GT(table_id, 0); + + pthread_t threads[PERF_THREAD_NUM + 1]; + struct thread_param thread_params[PERF_THREAD_NUM + 1]; + int i = 0; + int *is_all_hit = NULL; + + for (i = 0; i < PERF_THREAD_NUM + 1; i++) { + thread_params[i].maat_instance = maat_instance; + thread_params[i].thread_id = i; + thread_params[i].table_name = table_name; + thread_params[i].test_count = 1 * 1000 * 1000; + thread_params[i].time_elapse_ms = 0; + thread_params[i].logger = logger; + + if (i < PERF_THREAD_NUM) { + pthread_create(&threads[i], NULL, perf_string_scan_thread, thread_params+i); + } else { + thread_params[i].test_count = 0; + pthread_create(&threads[i], NULL, perf_string_update_thread, thread_params+i); + } + } + + long long time_elapse_ms = 0; + long long scan_count = 0; + long long scan_per_second = 0; + for (i = 0; i < PERF_THREAD_NUM + 1; i++) { + pthread_join(threads[i], (void **)&is_all_hit); + time_elapse_ms += thread_params[i].time_elapse_ms; + scan_count += thread_params[i].test_count; + + EXPECT_EQ(*is_all_hit, 1); + *is_all_hit = 0; + free(is_all_hit); + } + scan_per_second = scan_count * 1000 / time_elapse_ms; + //EXPECT_GT(scan_per_second, 800 * 1000); + printf("High match rate on %d-threads speed %lld lookups/s/thread\n", PERF_THREAD_NUM, scan_per_second); +} + +class MaatPerfIPScan : public testing::Test +{ +protected: + static void SetUpTestCase() { + const char *accept_tags = "{\"tags\":[{\"tag\":\"location\",\"value\":\"北京/朝阳/华严北里/甲22号\"},{\"tag\":\"isp\",\"value\":\"移动\"},{\"tag\":\"location\",\"value\":\"Astana\"}]}"; + char redis_ip[64] = "127.0.0.1"; + int redis_port = 6379; + int redis_db = 0; + + logger = log_handle_create("./maat_framework_perf_gtest.log", 0); + int ret = write_config_to_redis(redis_ip, redis_port, redis_db, logger); + if (ret < 0) { + log_error(logger, MODULE_FRAMEWORK_PERF_GTEST, + "[%s:%d] write config to redis failed.", __FUNCTION__, __LINE__); + } + + struct maat_options *opts = maat_options_new(); + maat_options_set_redis(opts, redis_ip, redis_port, redis_db); + maat_options_set_logger(opts, "./maat_framework_perf_gtest.log", LOG_LEVEL_INFO); + maat_options_set_accept_tags(opts, accept_tags); + maat_options_set_caller_thread_number(opts, 5); + + _shared_maat_instance = maat_new(opts, table_info_path); + maat_options_free(opts); + if (NULL == _shared_maat_instance) { + log_error(logger, MODULE_FRAMEWORK_PERF_GTEST, + "[%s:%d] create maat instance in MaatFlagScan failed.", + __FUNCTION__, __LINE__); + } + } + + static void TearDownTestCase() { + maat_free(_shared_maat_instance); + log_handle_destroy(logger); + } + + static struct log_handle *logger; + static struct maat *_shared_maat_instance; +}; + +struct maat *MaatPerfIPScan::_shared_maat_instance; +struct log_handle *MaatPerfIPScan::logger; + +TEST_F(MaatPerfIPScan, basic) +{ + const char *table_name = "IP_PLUS_CONFIG"; + struct maat *maat_instance = MaatPerfIPScan::_shared_maat_instance; + + int table_id = maat_get_table_id(maat_instance, table_name); + ASSERT_GT(table_id, 0); + + pthread_t threads[PERF_THREAD_NUM + 1]; + struct thread_param thread_params[PERF_THREAD_NUM + 1]; + int i = 0; + int *is_all_hit = NULL; + + for (i = 0; i < PERF_THREAD_NUM + 1; i++) { + thread_params[i].maat_instance = maat_instance; + thread_params[i].thread_id = i; + thread_params[i].table_name = table_name; + thread_params[i].test_count = 1 * 1000 * 1000; + thread_params[i].time_elapse_ms = 0; + thread_params[i].logger = logger; + + if (i < PERF_THREAD_NUM) { + pthread_create(&threads[i], NULL, perf_ip_scan_thread, thread_params+i); + } else { + thread_params[i].test_count = 0; + pthread_create(&threads[i], NULL, perf_ip_update_thread, thread_params+i); + } + } + + long long time_elapse_ms = 0; + long long scan_count = 0; + long long scan_per_second = 0; + for (i = 0; i < PERF_THREAD_NUM + 1; i++) { + pthread_join(threads[i], (void **)&is_all_hit); + time_elapse_ms += thread_params[i].time_elapse_ms; + scan_count += thread_params[i].test_count; + + EXPECT_EQ(*is_all_hit, 1); + *is_all_hit = 0; + free(is_all_hit); + } + scan_per_second = scan_count * 1000 / time_elapse_ms; + //EXPECT_GT(scan_per_second, 800 * 1000); + printf("High match rate on %d-threads speed %lld lookups/s/thread\n", PERF_THREAD_NUM, scan_per_second); +} + +int main(int argc, char ** argv) +{ + int ret=0; + ::testing::InitGoogleTest(&argc, argv); + + ret=RUN_ALL_TESTS(); + + return ret; +} \ No newline at end of file diff --git a/test/rcu_hash_gtest.cpp b/test/rcu_hash_gtest.cpp index 563e0d7..0d4172c 100644 --- a/test/rcu_hash_gtest.cpp +++ b/test/rcu_hash_gtest.cpp @@ -457,6 +457,88 @@ TEST(global_rcu_hash_del, single_thread) { res = rcu_hash_find(g_htable, key, key_len); EXPECT_TRUE(res == NULL); } +#define THREAD_NUM 10 +struct thread_param { + int thread_id; + int test_count; + unsigned long long time_elapse_ms; +}; + +void *rcu_scan_thread(void *arg) +{ + struct thread_param *param = (struct thread_param *)arg; + + int hit_cnt = 0; + void **data_array = NULL; + + sleep(2); + for (int index = 0; index < param->test_count; index++) { + size_t count = rcu_hash_list(g_htable, &data_array); + + for (size_t i = 0; i < count; i++) { + struct user_data *user_data = (struct user_data *)data_array[i]; + if (user_data != NULL && user_data->id == 101) { + hit_cnt++; + } + } + } + int *is_all_hit = ALLOC(int, 1); + *is_all_hit = (hit_cnt == param->test_count) ? 1 : 0; + printf("thread[%d]: hit_cnt:%d\n", param->thread_id, hit_cnt); + return is_all_hit; +} + +void *rcu_update_thread(void *arg) +{ + const char *key = "http_url"; + size_t key_len = strlen(key); + + for (int i = 0; i < 10; i++) { + rcu_hash_del(g_htable, key, key_len); + rcu_hash_commit(g_htable); + + sleep(1); + + struct user_data *data = ALLOC(struct user_data, 1); + data->id = 101; + char name[64] = "www.baidu.com"; + memcpy(data->name, name, strlen(name)); + rcu_hash_add(g_htable, key, key_len, data); + rcu_hash_commit(g_htable); + } + int *is_all_hit = ALLOC(int, 1); + *is_all_hit = 1; + + return is_all_hit; +} + +TEST(rcu_hash_update, multi_thread) { + pthread_t threads[THREAD_NUM + 1]; + struct thread_param t_param[THREAD_NUM + 1]; + for (int i = 0; i < THREAD_NUM + 1; i++) { + t_param[i].thread_id = i; + t_param[i].test_count = 1000 * 1000; + t_param[i].time_elapse_ms = 0; + + if (i < THREAD_NUM) { + pthread_create(&threads[i], NULL, rcu_scan_thread, t_param+i); + } else { + pthread_create(&threads[i], NULL, rcu_update_thread, t_param+i); + } + } + + int *is_all_hit = NULL; + unsigned long long time_elapse_ms = 0, scan_count = 0; + for (int i = 0; i < THREAD_NUM + 1; i++) { + pthread_join(threads[i], (void **)&is_all_hit); + time_elapse_ms += t_param[i].time_elapse_ms; + scan_count += t_param[i].test_count; + + EXPECT_EQ(*is_all_hit, 1); + *is_all_hit = 0; + free(is_all_hit); + } +} int main(int argc, char ** argv) {