gfwleak/tango-maat
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[PATCH]add log handle for maat_wrap_redis_command

Browse Source
This commit is contained in:
liuwentan
2024-02-22 17:51:23 +08:00
parent 26d642bdcf
commit 7de0db5ebc
10 changed files with 558 additions and 327 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
src/inc_internal/maat_redis_monitor.h
View File

@@ -19,6 +19,7 @@ extern "C"
#include <stdint.h> #include <stdint.h>
#include <sys/queue.h> #include <sys/queue.h>
#include "log/log.h"
#include "hiredis/hiredis.h" #include "hiredis/hiredis.h"
#include "maat_command.h" #include "maat_command.h"
#include "uthash/uthash.h" #include "uthash/uthash.h"
@@ -51,9 +52,10 @@ void maat_set_serial_rule(struct serial_rule *rule, enum maat_operation op,
const char *line, long long timeout); const char *line, long long timeout);
redisContext *maat_connect_redis(const char *redis_ip, int redis_port, redisContext *maat_connect_redis(const char *redis_ip, int redis_port,
int redis_db, struct log_handle *logger); int redis_db, struct log_handle *logger);
redisReply *maat_wrap_redis_command(redisContext *c, const char *format, ...); redisReply *maat_wrap_redis_command(redisContext *c, struct log_handle *logger,
const char *format, ...);
int maat_wrap_redis_get_reply(redisContext *c, redisReply **reply); int maat_wrap_redis_get_reply(redisContext *c, redisReply **reply);
@@ -68,10 +70,10 @@ int maat_cmd_write_rule(redisContext *c, struct serial_rule *s_rule,
struct log_handle *logger); struct log_handle *logger);
int maat_get_redis_value(redisContext *c, struct serial_rule *rule_list, int maat_get_redis_value(redisContext *c, struct serial_rule *rule_list,
int rule_num, int print_process, struct log_handle *logger); int rule_num, int print_process, struct log_handle *logger);
int maat_get_foreign_keys_by_prefix(redisContext *ctx, struct serial_rule *rule_list, int maat_get_foreign_keys_by_prefix(redisContext *ctx, struct serial_rule *rule_list,
int rule_num, const char* dir, struct log_handle *logger); int rule_num, const char *dir, struct log_handle *logger);
void maat_get_foreign_conts(redisContext *c, struct serial_rule *rule_list, void maat_get_foreign_conts(redisContext *c, struct serial_rule *rule_list,
int rule_num, int print_fn, struct log_handle *logger); int rule_num, int print_fn, struct log_handle *logger);

6
src/json2iris.c
View File

@@ -333,8 +333,10 @@ static int get_region_seq(struct iris_description *iris_cfg)
if (NULL == iris_cfg->redis_write_ctx) { if (NULL == iris_cfg->redis_write_ctx) {
sequence = iris_cfg->region_cnt + 1; sequence = iris_cfg->region_cnt + 1;
} else { } else {
redisReply *data_reply = maat_wrap_redis_command(iris_cfg->redis_write_ctx, redisReply *data_reply =
"INCRBY %s 1", mr_region_id_var); maat_wrap_redis_command(iris_cfg->redis_write_ctx, NULL,
"INCRBY %s 1", mr_region_id_var);
sequence = (int)data_reply->integer; sequence = (int)data_reply->integer;
freeReplyObject(data_reply); freeReplyObject(data_reply);
} }

112
src/maat_command.c
View File

@@ -35,7 +35,9 @@ extern const char *mr_label_sset;
extern const int MAAT_REDIS_SYNC_TIME; extern const int MAAT_REDIS_SYNC_TIME;
redisReply *maat_wrap_redis_command(redisContext *c, const char *format, ...) redisReply *
maat_wrap_redis_command(redisContext *c, struct log_handle *logger,
const char *format, ...)
{ {
va_list ap; va_list ap;
void *reply = NULL; void *reply = NULL;
@@ -47,6 +49,12 @@ redisReply *maat_wrap_redis_command(redisContext *c, const char *format, ...)
reply = redisvCommand(c,format,ap); reply = redisvCommand(c,format,ap);
va_end(ap); va_end(ap);
if (NULL == reply) { if (NULL == reply) {
if (logger != NULL) {
log_fatal(logger, MODULE_MAAT_COMMAND,
"[%s:%d]execute redis command error:%s",
__FUNCTION__, __LINE__, c->errstr);
}
ret = redisReconnect(c); ret = redisReconnect(c);
retry++; retry++;
} }
@@ -56,21 +64,24 @@ redisReply *maat_wrap_redis_command(redisContext *c, const char *format, ...)
} }
redisContext *maat_connect_redis(const char *redis_ip, int redis_port, redisContext *maat_connect_redis(const char *redis_ip, int redis_port,
int redis_db, struct log_handle *logger) int redis_db, struct log_handle *logger)
{ {
struct timeval connect_timeout; struct timeval connect_timeout;
connect_timeout.tv_sec = 0; connect_timeout.tv_sec = 0;
connect_timeout.tv_usec = 100 * 1000; // 100 ms connect_timeout.tv_usec = 100 * 1000; // 100 ms
redisContext *c = redisConnectWithTimeout(redis_ip, redis_port, connect_timeout); redisContext *c = redisConnectWithTimeout(redis_ip, redis_port,
connect_timeout);
if (NULL == c || c->err) { if (NULL == c || c->err) {
if (NULL == logger) { if (NULL == logger) {
printf("Unable to connect redis server %s:%d db%d, error: %s", printf("Unable to connect redis server %s:%d db%d, error: %s",
redis_ip, redis_port, redis_db, c == NULL ? "Unknown" : c->errstr); redis_ip, redis_port, redis_db,
c == NULL ? "Unknown" : c->errstr);
} else { } else {
log_fatal(logger, MODULE_MAAT_COMMAND, log_fatal(logger, MODULE_MAAT_COMMAND,
"[%s:%d] Unable to connect redis server %s:%d db%d, error: %s", "[%s:%d] Unable to connect redis server"
__FUNCTION__, __LINE__, redis_ip, redis_port, redis_db, " %s:%d db%d, error: %s", __FUNCTION__,
__LINE__, redis_ip, redis_port, redis_db,
c == NULL ? "Unknown" : c->errstr); c == NULL ? "Unknown" : c->errstr);
} }
@@ -81,7 +92,8 @@ redisContext *maat_connect_redis(const char *redis_ip, int redis_port,
} }
redisEnableKeepAlive(c); redisEnableKeepAlive(c);
redisReply *reply = maat_wrap_redis_command(c, "select %d", redis_db); redisReply *reply =
maat_wrap_redis_command(c, logger, "select %d", redis_db);
freeReplyObject(reply); freeReplyObject(reply);
reply = NULL; reply = NULL;
@@ -108,15 +120,17 @@ long long maat_read_redis_integer(const redisReply *reply)
return 0; return 0;
} }
static int redis_flushDB(redisContext *ctx, int db_index, struct log_handle *logger) static int
redis_flushDB(redisContext *ctx, int db_index, struct log_handle *logger)
{ {
long long maat_redis_version = 0; long long maat_redis_version = 0;
redisReply *data_reply = maat_wrap_redis_command(ctx, "WATCH MAAT_VERSION"); redisReply *data_reply =
maat_wrap_redis_command(ctx, logger, "WATCH MAAT_VERSION");
freeReplyObject(data_reply); freeReplyObject(data_reply);
data_reply = NULL; data_reply = NULL;
data_reply = maat_wrap_redis_command(ctx, "GET MAAT_VERSION"); data_reply = maat_wrap_redis_command(ctx, logger, "GET MAAT_VERSION");
if (data_reply->type == REDIS_REPLY_NIL) { if (data_reply->type == REDIS_REPLY_NIL) {
maat_redis_version = 0; maat_redis_version = 0;
} else { } else {
@@ -126,12 +140,12 @@ static int redis_flushDB(redisContext *ctx, int db_index, struct log_handle *log
data_reply = NULL; data_reply = NULL;
} }
data_reply = maat_wrap_redis_command(ctx, "DBSIZE"); data_reply = maat_wrap_redis_command(ctx, logger, "DBSIZE");
long long dbsize = maat_read_redis_integer(data_reply); long long dbsize = maat_read_redis_integer(data_reply);
freeReplyObject(data_reply); freeReplyObject(data_reply);
data_reply = NULL; data_reply = NULL;
data_reply = maat_wrap_redis_command(ctx, "MULTI"); data_reply = maat_wrap_redis_command(ctx, logger, "MULTI");
freeReplyObject(data_reply); freeReplyObject(data_reply);
data_reply = NULL; data_reply = NULL;
@@ -165,7 +179,8 @@ static int redis_flushDB(redisContext *ctx, int db_index, struct log_handle *log
if (redis_transaction_success == 1) { if (redis_transaction_success == 1) {
log_info(logger, MODULE_MAAT_COMMAND, log_info(logger, MODULE_MAAT_COMMAND,
"FlushDB %d, MAAT_VERSION:%llu, DBSize:%llu.", "FlushDB %d, MAAT_VERSION:%llu, DBSize:%llu.",
db_index, (maat_redis_version == 0)?0:(maat_redis_version-1),dbsize); db_index, (maat_redis_version == 0)?0:(maat_redis_version-1),
dbsize);
} }
return redis_transaction_success; return redis_transaction_success;
@@ -176,8 +191,8 @@ static int connect_redis_for_write(struct source_redis_ctx *redis_ctx,
{ {
assert(redis_ctx->write_ctx == NULL); assert(redis_ctx->write_ctx == NULL);
redis_ctx->write_ctx = maat_connect_redis(redis_ctx->redis_ip, redis_ctx->write_ctx = maat_connect_redis(redis_ctx->redis_ip,
redis_ctx->redis_port, redis_ctx->redis_port,
redis_ctx->redis_db, logger); redis_ctx->redis_db, logger);
if (NULL == redis_ctx->write_ctx) { if (NULL == redis_ctx->write_ctx) {
return -1; return -1;
} else { } else {
@@ -234,7 +249,7 @@ long long maat_redis_server_time_s(redisContext *c)
{ {
long long server_time = 0; long long server_time = 0;
redisReply *data_reply = maat_wrap_redis_command(c, "TIME"); redisReply *data_reply = maat_wrap_redis_command(c, NULL, "TIME");
if (data_reply->type == REDIS_REPLY_ARRAY) { if (data_reply->type == REDIS_REPLY_ARRAY) {
server_time = atoll(data_reply->element[0]->str); server_time = atoll(data_reply->element[0]->str);
freeReplyObject(data_reply); freeReplyObject(data_reply);
@@ -249,7 +264,8 @@ int maat_wrap_redis_get_reply(redisContext *c, redisReply **reply)
return redisGetReply(c, (void **)reply); return redisGetReply(c, (void **)reply);
} }
int maat_cmd_set_line(struct maat *maat_inst, const struct maat_cmd_line *line_rule) int maat_cmd_set_line(struct maat *maat_inst,
const struct maat_cmd_line *line_rule)
{ {
int ret = 0; int ret = 0;
long long absolute_expire_time = 0; long long absolute_expire_time = 0;
@@ -266,29 +282,36 @@ int maat_cmd_set_line(struct maat *maat_inst, const struct maat_cmd_line *line_r
struct serial_rule *s_rule = ALLOC(struct serial_rule, 1); struct serial_rule *s_rule = ALLOC(struct serial_rule, 1);
int table_id = table_manager_get_table_id(maat_inst->tbl_mgr, line_rule->table_name); int table_id = table_manager_get_table_id(maat_inst->tbl_mgr,
line_rule->table_name);
if (table_id < 0) { if (table_id < 0) {
log_fatal(maat_inst->logger, MODULE_MAAT_COMMAND, log_fatal(maat_inst->logger, MODULE_MAAT_COMMAND,
"[%s:%d] Command set line id %lld failed: unknown table %s", "[%s:%d] Command set line id %lld failed: "
__FUNCTION__, __LINE__, line_rule->rule_id, line_rule->table_name); "unknown table %s", __FUNCTION__, __LINE__,
line_rule->rule_id, line_rule->table_name);
FREE(s_rule); FREE(s_rule);
return -1; return -1;
} }
int valid_column = table_manager_get_valid_column(maat_inst->tbl_mgr, table_id); int valid_column = table_manager_get_valid_column(maat_inst->tbl_mgr,
table_id);
if (valid_column < 0) { if (valid_column < 0) {
log_fatal(maat_inst->logger, MODULE_MAAT_COMMAND, log_fatal(maat_inst->logger, MODULE_MAAT_COMMAND,
"[%s:%d] Command set line id %lld failed: table %s is not a plugin or ip_plugin table", "[%s:%d] Command set line id %lld failed: "
__FUNCTION__, __LINE__, line_rule->rule_id, line_rule->table_name); "table %s is not a plugin or ip_plugin table",
__FUNCTION__, __LINE__, line_rule->rule_id,
line_rule->table_name);
FREE(s_rule); FREE(s_rule);
return -1; return -1;
} }
int valid_offset = maat_get_valid_flag_offset(line_rule->table_line, valid_column); int valid_offset = maat_get_valid_flag_offset(line_rule->table_line,
valid_column);
if (valid_offset < 0) { if (valid_offset < 0) {
log_fatal(maat_inst->logger, MODULE_MAAT_COMMAND, log_fatal(maat_inst->logger, MODULE_MAAT_COMMAND,
"[%s:%d] Command set line id %lld failed: table %s valid_offset error", "[%s:%d] Command set line id %lld failed: "
__FUNCTION__, __LINE__, line_rule->rule_id, line_rule->table_name); "table %s valid_offset error", __FUNCTION__, __LINE__,
line_rule->rule_id, line_rule->table_name);
FREE(s_rule); FREE(s_rule);
return -1; return -1;
} }
@@ -298,10 +321,12 @@ int maat_cmd_set_line(struct maat *maat_inst, const struct maat_cmd_line *line_r
absolute_expire_time = server_time + line_rule->expire_after; absolute_expire_time = server_time + line_rule->expire_after;
} }
maat_set_serial_rule(s_rule, (enum maat_operation)is_valid, line_rule->rule_id, maat_set_serial_rule(s_rule, (enum maat_operation)is_valid,
line_rule->table_name, line_rule->table_line, absolute_expire_time); line_rule->rule_id, line_rule->table_name,
line_rule->table_line, absolute_expire_time);
int success_cnt = maat_cmd_write_rule(write_ctx, s_rule, 1, server_time, maat_inst->logger); int success_cnt = maat_cmd_write_rule(write_ctx, s_rule, 1, server_time,
maat_inst->logger);
if (success_cnt != 1) { if (success_cnt != 1) {
ret = -1; ret = -1;
goto error_out; goto error_out;
@@ -317,8 +342,9 @@ error_out:
return ret; return ret;
} }
int maat_cmd_set_file(struct maat *maat_inst, const char *key, const char *value, #define ARG_VEC_NUM 3
size_t size, enum maat_operation op) int maat_cmd_set_file(struct maat *maat_inst, const char *key,
const char *value, size_t size, enum maat_operation op)
{ {
redisContext *ctx = maat_inst->opts.redis_ctx.write_ctx; redisContext *ctx = maat_inst->opts.redis_ctx.write_ctx;
if (NULL == ctx) { if (NULL == ctx) {
@@ -328,8 +354,8 @@ int maat_cmd_set_file(struct maat *maat_inst, const char *key, const char *value
return -1; return -1;
} }
const char *arg_vec[3]; const char *arg_vec[ARG_VEC_NUM];
size_t len_vec[3]; size_t len_vec[ARG_VEC_NUM];
arg_vec[0] = "SET"; arg_vec[0] = "SET";
len_vec[0] = strlen("SET"); len_vec[0] = strlen("SET");
@@ -343,24 +369,28 @@ int maat_cmd_set_file(struct maat *maat_inst, const char *key, const char *value
redisReply *reply = NULL; redisReply *reply = NULL;
if (0 != strncmp(key, foreign_key_prefix, strlen(foreign_key_prefix))) { if (0 != strncmp(key, foreign_key_prefix, strlen(foreign_key_prefix))) {
log_fatal(maat_inst->logger, MODULE_MAAT_COMMAND, log_fatal(maat_inst->logger, MODULE_MAAT_COMMAND,
"Invalid File key, prefix %s is mandatory.", foreign_key_prefix); "Invalid File key, prefix %s is mandatory.",
foreign_key_prefix);
return -1; return -1;
} }
switch (op) { switch (op) {
case MAAT_OP_ADD: case MAAT_OP_ADD:
reply = (redisReply *)redisCommandArgv(ctx, sizeof(arg_vec) / sizeof(arg_vec[0]), reply = (redisReply *)redisCommandArgv(ctx, ARG_VEC_NUM,
arg_vec, len_vec); arg_vec, len_vec);
break; break;
case MAAT_OP_DEL: case MAAT_OP_DEL:
reply = maat_wrap_redis_command(ctx, "EXPIRE %s %d", key, MAAT_REDIS_SYNC_TIME); reply = maat_wrap_redis_command(ctx, maat_inst->logger,
"EXPIRE %s %d", key,
MAAT_REDIS_SYNC_TIME);
break; break;
default: default:
return -1; return -1;
break; break;
} }
if (NULL == reply || reply->type == REDIS_REPLY_NIL || reply->type == REDIS_REPLY_ERROR) { if (NULL == reply || reply->type == REDIS_REPLY_NIL ||
reply->type == REDIS_REPLY_ERROR) {
log_fatal(maat_inst->logger, MODULE_MAAT_COMMAND, log_fatal(maat_inst->logger, MODULE_MAAT_COMMAND,
"Set file failed, maybe Redis is busy."); "Set file failed, maybe Redis is busy.");
freeReplyObject(reply); freeReplyObject(reply);
@@ -373,7 +403,8 @@ int maat_cmd_set_file(struct maat *maat_inst, const char *key, const char *value
return 1; return 1;
} }
long long maat_cmd_incrby(struct maat *maat_inst, const char *key, int increment) long long maat_cmd_incrby(struct maat *maat_inst, const char *key,
int increment)
{ {
long long result = 0; long long result = 0;
@@ -382,7 +413,10 @@ long long maat_cmd_incrby(struct maat *maat_inst, const char *key, int increment
return -1; return -1;
} }
redisReply *data_reply = maat_wrap_redis_command(write_ctx, "INCRBY %s %d", key, increment); redisReply *data_reply =
maat_wrap_redis_command(write_ctx, NULL, "INCRBY %s %d",
key, increment);
if (data_reply->type == REDIS_REPLY_INTEGER) { if (data_reply->type == REDIS_REPLY_INTEGER) {
result = data_reply->integer; result = data_reply->integer;
} else { } else {

47
src/maat_redis_monitor.c
View File

@@ -402,9 +402,10 @@ get_inc_key_list(long long instance_version, long long target_version,
return 0; return 0;
} }
redisReply *tmp_reply= maat_wrap_redis_command(c, "ZSCORE %s %s", redisReply *tmp_reply =
mr_status_sset, maat_wrap_redis_command(c, logger, "ZSCORE %s %s",
reply->element[0]->str); mr_status_sset,
reply->element[0]->str);
if (tmp_reply->type != REDIS_REPLY_STRING) { if (tmp_reply->type != REDIS_REPLY_STRING) {
log_fatal(logger, MODULE_REDIS_MONITOR, log_fatal(logger, MODULE_REDIS_MONITOR,
"[%s:%d] ZSCORE %s %s failed Version: %lld->%lld", "[%s:%d] ZSCORE %s %s failed Version: %lld->%lld",
@@ -419,12 +420,15 @@ get_inc_key_list(long long instance_version, long long target_version,
} }
long long nearest_rule_version = maat_read_redis_integer(tmp_reply); long long nearest_rule_version = maat_read_redis_integer(tmp_reply);
freeReplyObject(tmp_reply);
tmp_reply = NULL;
if (nearest_rule_version < 0) { if (nearest_rule_version < 0) {
log_fatal(logger, MODULE_REDIS_MONITOR,
"[%s:%d]get nearest_rule_version failed, reply->type:%d",
__FUNCTION__, __LINE__, tmp_reply->type);
freeReplyObject(tmp_reply);
return -1; return -1;
} }
freeReplyObject(tmp_reply);
if (nearest_rule_version != instance_version + 1) { if (nearest_rule_version != instance_version + 1) {
log_info(logger, MODULE_REDIS_MONITOR, log_info(logger, MODULE_REDIS_MONITOR,
@@ -601,7 +605,7 @@ FULL_UPDATE:
reply = NULL; reply = NULL;
} }
reply = maat_wrap_redis_command(c, "EXEC"); reply = maat_wrap_redis_command(c, logger, "EXEC");
if (NULL == reply) { if (NULL == reply) {
log_fatal(logger, MODULE_REDIS_MONITOR, log_fatal(logger, MODULE_REDIS_MONITOR,
"[%s:%d] Redis Communication error: %s", "[%s:%d] Redis Communication error: %s",
@@ -844,8 +848,9 @@ void maat_rewrite_table_line_with_foreign(struct serial_rule *s_rule)
s_rule->table_line = rewrite_line; s_rule->table_line = rewrite_line;
} }
static void expected_reply_add(struct expected_reply* expected, int s_rule_seq, static void
int type, long long integer) expected_reply_add(struct expected_reply* expected, int s_rule_seq,
int type, long long integer)
{ {
int i = expected->possible_reply_num; int i = expected->possible_reply_num;
assert(i < REDIS_REPLY_SIZE); assert(i < REDIS_REPLY_SIZE);
@@ -855,13 +860,15 @@ static void expected_reply_add(struct expected_reply* expected, int s_rule_seq,
expected->possible_reply_num++; expected->possible_reply_num++;
} }
static int redlock_try_lock(redisContext *c, const char *lock_name, static int
long long expire) redlock_try_lock(redisContext *c, const char *lock_name,
long long expire)
{ {
int ret = 0; int ret = 0;
redisReply *reply = redisReply *reply =
maat_wrap_redis_command(c, "SET %s locked NX PX %lld", lock_name, expire); maat_wrap_redis_command(c, NULL, "SET %s locked NX PX %lld",
lock_name, expire);
if (reply->type == REDIS_REPLY_NIL) { if (reply->type == REDIS_REPLY_NIL) {
ret = 0; ret = 0;
} else { } else {
@@ -891,7 +898,7 @@ exec_serial_rule_begin(redisContext* c, size_t rule_num,
} }
if (rule_num > renew_rule_num) { if (rule_num > renew_rule_num) {
data_reply = maat_wrap_redis_command(c, "INCRBY MAAT_PRE_VER 1"); data_reply = maat_wrap_redis_command(c, NULL, "INCRBY MAAT_PRE_VER 1");
*transaction_version = maat_read_redis_integer(data_reply); *transaction_version = maat_read_redis_integer(data_reply);
freeReplyObject(data_reply); freeReplyObject(data_reply);
data_reply = NULL; data_reply = NULL;
@@ -901,7 +908,7 @@ exec_serial_rule_begin(redisContext* c, size_t rule_num,
} }
if (*renew_allowed == 1 || rule_num > renew_rule_num) { if (*renew_allowed == 1 || rule_num > renew_rule_num) {
data_reply = maat_wrap_redis_command(c, "MULTI"); data_reply = maat_wrap_redis_command(c, NULL, "MULTI");
freeReplyObject(data_reply); freeReplyObject(data_reply);
data_reply = NULL; data_reply = NULL;
ret = 0; ret = 0;
@@ -912,7 +919,7 @@ exec_serial_rule_begin(redisContext* c, size_t rule_num,
static void redlock_unlock(redisContext *c, const char *lock_name) static void redlock_unlock(redisContext *c, const char *lock_name)
{ {
redisReply *reply = maat_wrap_redis_command(c, "DEL %s", lock_name); redisReply *reply = maat_wrap_redis_command(c, NULL, "DEL %s", lock_name);
freeReplyObject(reply); freeReplyObject(reply);
reply = NULL; reply = NULL;
} }
@@ -941,7 +948,7 @@ exec_serial_rule_end(redisContext *c, const char *transaction_list,
if (strlen(transaction_list) > 0) { if (strlen(transaction_list) > 0) {
data_reply = data_reply =
maat_wrap_redis_command(c, "eval %s 4 MAAT_VERSION %s %s %s %lld", maat_wrap_redis_command(c, NULL, "eval %s 4 MAAT_VERSION %s %s %s %lld",
lua_exec_done, mr_status_sset, mr_version_sset, lua_exec_done, mr_status_sset, mr_version_sset,
transaction_list, server_time); transaction_list, server_time);
freeReplyObject(data_reply); freeReplyObject(data_reply);
@@ -950,7 +957,7 @@ exec_serial_rule_end(redisContext *c, const char *transaction_list,
(*cnt)++; (*cnt)++;
} }
data_reply = maat_wrap_redis_command(c, "EXEC"); data_reply = maat_wrap_redis_command(c, NULL, "EXEC");
return data_reply; return data_reply;
} }
@@ -1225,7 +1232,7 @@ cleanup_update_status(redisContext *c, struct log_handle *logger)
return; return;
} }
redisReply *reply = maat_wrap_redis_command(c, "MULTI"); redisReply *reply = maat_wrap_redis_command(c, logger, "MULTI");
freeReplyObject(reply); freeReplyObject(reply);
reply = NULL; reply = NULL;
@@ -1246,7 +1253,7 @@ cleanup_update_status(redisContext *c, struct log_handle *logger)
} }
redisReply *sub_reply = NULL; redisReply *sub_reply = NULL;
reply = maat_wrap_redis_command(c, "EXEC"); reply = maat_wrap_redis_command(c, logger, "EXEC");
if (reply->type != REDIS_REPLY_ARRAY) { if (reply->type != REDIS_REPLY_ARRAY) {
goto error_out; goto error_out;
} }
@@ -1269,7 +1276,7 @@ cleanup_update_status(redisContext *c, struct log_handle *logger)
reply = NULL; reply = NULL;
//To deal with maat_version reset to 0, do NOT use -inf as lower bound intentionally. //To deal with maat_version reset to 0, do NOT use -inf as lower bound intentionally.
reply = maat_wrap_redis_command(c, "ZREMRANGEBYSCORE %s %lld %lld", reply = maat_wrap_redis_command(c, logger, "ZREMRANGEBYSCORE %s %lld %lld",
mr_status_sset, version_lower_bound, mr_status_sset, version_lower_bound,
version_upper_bound); version_upper_bound);
entry_num = maat_read_redis_integer(reply); entry_num = maat_read_redis_integer(reply);
@@ -1298,7 +1305,7 @@ check_maat_expiration(redisContext *c, struct log_handle *logger)
} }
redisReply *data_reply = redisReply *data_reply =
maat_wrap_redis_command(c, "ZRANGEBYSCORE %s -inf %lld", maat_wrap_redis_command(c, logger, "ZRANGEBYSCORE %s -inf %lld",
mr_expire_sset, server_time); mr_expire_sset, server_time);
if (data_reply->type != REDIS_REPLY_ARRAY || if (data_reply->type != REDIS_REPLY_ARRAY ||
0 == data_reply->elements) { 0 == data_reply->elements) {

100
src/maat_rule.c
View File

@@ -55,7 +55,8 @@ maat_runtime_create(long long version, struct maat *maat_inst)
maat_rt->sequence_map = maat_kv_store_new(); maat_rt->sequence_map = maat_kv_store_new();
maat_rt->logger = maat_inst->logger; maat_rt->logger = maat_inst->logger;
maat_rt->ref_garbage_bin = maat_inst->garbage_bin; maat_rt->ref_garbage_bin = maat_inst->garbage_bin;
maat_rt->ref_cnt = alignment_int64_array_alloc(maat_inst->opts.nr_worker_thread); maat_rt->ref_cnt =
alignment_int64_array_alloc(maat_inst->opts.nr_worker_thread);
return maat_rt; return maat_rt;
} }
@@ -79,17 +80,20 @@ static void maat_runtime_destroy(struct maat_runtime *maat_rt)
FREE(maat_rt); FREE(maat_rt);
} }
static void maat_runtime_commit(struct maat_runtime *maat_rt, int update_type, static void
long long maat_rt_version, struct log_handle *logger) maat_runtime_commit(struct maat_runtime *maat_rt, int update_type,
long long maat_rt_version, struct log_handle *logger)
{ {
for (size_t i = 0; i < maat_rt->max_table_num; i++) { for (size_t i = 0; i < maat_rt->max_table_num; i++) {
table_manager_commit_runtime(maat_rt->ref_tbl_mgr, i, update_type, maat_rt_version); table_manager_commit_runtime(maat_rt->ref_tbl_mgr, i,
update_type, maat_rt_version);
} }
maat_rt->last_update_time = time(NULL); maat_rt->last_update_time = time(NULL);
} }
static void maat_start_cb(long long new_version, int update_type, void *u_param) static void
maat_start_cb(long long new_version, int update_type, void *u_param)
{ {
size_t i = 0; size_t i = 0;
enum table_type table_type = TABLE_TYPE_INVALID; enum table_type table_type = TABLE_TYPE_INVALID;
@@ -99,13 +103,16 @@ static void maat_start_cb(long long new_version, int update_type, void *u_param)
maat_inst->new_version = new_version; maat_inst->new_version = new_version;
if (update_type == MAAT_UPDATE_TYPE_FULL) { if (update_type == MAAT_UPDATE_TYPE_FULL) {
maat_inst->creating_maat_rt = maat_runtime_create(new_version, maat_inst); maat_inst->creating_maat_rt =
maat_runtime_create(new_version, maat_inst);
for (i = 0; i < max_table_cnt; i++) { for (i = 0; i < max_table_cnt; i++) {
table_type = table_manager_get_table_type(maat_inst->tbl_mgr, i); table_type = table_manager_get_table_type(maat_inst->tbl_mgr, i);
if (table_type == TABLE_TYPE_COMPILE) { if (table_type == TABLE_TYPE_COMPILE) {
// compile runtime need a reference to maat runtime // compile runtime need a reference to maat runtime
void *compile_rt = table_manager_get_updating_runtime(maat_inst->tbl_mgr, i); void *compile_rt =
table_manager_get_updating_runtime(maat_inst->tbl_mgr, i);
compile_runtime_init(compile_rt, maat_inst->creating_maat_rt); compile_runtime_init(compile_rt, maat_inst->creating_maat_rt);
} }
} }
@@ -117,12 +124,14 @@ static void maat_start_cb(long long new_version, int update_type, void *u_param)
table_type = table_manager_get_table_type(maat_inst->tbl_mgr, i); table_type = table_manager_get_table_type(maat_inst->tbl_mgr, i);
if (table_type == TABLE_TYPE_PLUGIN) { if (table_type == TABLE_TYPE_PLUGIN) {
void *schema = table_manager_get_schema(maat_inst->tbl_mgr, i); void *schema = table_manager_get_schema(maat_inst->tbl_mgr, i);
plugin_table_all_callback_start((struct plugin_schema *)schema, update_type); plugin_table_all_callback_start((struct plugin_schema *)schema,
update_type);
} }
} }
} }
static int maat_update_cb(const char *table_name, const char *line, void *u_param) static int
maat_update_cb(const char *table_name, const char *line, void *u_param)
{ {
if (NULL == table_name || NULL == line || NULL == u_param) { if (NULL == table_name || NULL == line || NULL == u_param) {
return 0; return 0;
@@ -144,12 +153,15 @@ static int maat_update_cb(const char *table_name, const char *line, void *u_para
// find conjunction id for table_id // find conjunction id for table_id
long long conj_parent_table_ids[MAX_CONJ_PARENTS_NUM]; long long conj_parent_table_ids[MAX_CONJ_PARENTS_NUM];
int conj_parent_table_cnt = table_manager_get_conj_parent_table_ids(maat_inst->tbl_mgr, table_name, int conj_parent_table_cnt =
conj_parent_table_ids, MAX_CONJ_PARENTS_NUM); table_manager_get_conj_parent_table_ids(maat_inst->tbl_mgr, table_name,
conj_parent_table_ids,
MAX_CONJ_PARENTS_NUM);
if (conj_parent_table_cnt > 0) { if (conj_parent_table_cnt > 0) {
for (int i = 0; i < conj_parent_table_cnt; i++) { for (int i = 0; i < conj_parent_table_cnt; i++) {
int ret = table_manager_update_runtime(maat_inst->tbl_mgr, table_name, int ret = table_manager_update_runtime(maat_inst->tbl_mgr, table_name,
(int)conj_parent_table_ids[i], line, update_type); (int)conj_parent_table_ids[i],
line, update_type);
if (ret < 0) { if (ret < 0) {
log_fatal(maat_inst->logger, MODULE_MAAT_RULE, log_fatal(maat_inst->logger, MODULE_MAAT_RULE,
"[%s:%d] table<%s> update runtime error for rule:%s", "[%s:%d] table<%s> update runtime error for rule:%s",
@@ -176,20 +188,23 @@ static long long maat_runtime_rule_num(struct maat_runtime *maat_rt)
long long total = 0; long long total = 0;
for (size_t i = 0; i < maat_rt->max_table_num; i++) { for (size_t i = 0; i < maat_rt->max_table_num; i++) {
long long rule_cnt = table_manager_runtime_rule_count(maat_rt->ref_tbl_mgr, i); long long rule_cnt =
table_manager_runtime_rule_count(maat_rt->ref_tbl_mgr, i);
total += rule_cnt; total += rule_cnt;
if (rule_cnt != 0) { if (rule_cnt != 0) {
log_info(maat_rt->logger, MODULE_MAAT_RULE, log_info(maat_rt->logger, MODULE_MAAT_RULE,
"table:<%s> rule_count:%lld", "table:<%s> rule_count:%lld",
table_manager_get_table_name(maat_rt->ref_tbl_mgr, i), rule_cnt); table_manager_get_table_name(maat_rt->ref_tbl_mgr, i),
rule_cnt);
} }
} }
return total; return total;
} }
static void maat_plugin_table_all_callback_finish(struct table_manager *tbl_mgr) static void
maat_plugin_table_all_callback_finish(struct table_manager *tbl_mgr)
{ {
size_t max_table_cnt = table_manager_table_size(tbl_mgr); size_t max_table_cnt = table_manager_table_size(tbl_mgr);
enum table_type table_type = TABLE_TYPE_INVALID; enum table_type table_type = TABLE_TYPE_INVALID;
@@ -205,7 +220,8 @@ static void maat_plugin_table_all_callback_finish(struct table_manager *tbl_mgr)
} }
} }
static void maat_plugin_table_garbage_collect_routine(struct table_manager *tbl_mgr) static void
maat_plugin_table_garbage_collect_routine(struct table_manager *tbl_mgr)
{ {
size_t max_table_cnt = table_manager_table_size(tbl_mgr); size_t max_table_cnt = table_manager_table_size(tbl_mgr);
enum table_type table_type = TABLE_TYPE_INVALID; enum table_type table_type = TABLE_TYPE_INVALID;
@@ -254,9 +270,14 @@ static void maat_finish_cb(void *u_param)
maat_plugin_table_all_callback_finish(maat_inst->tbl_mgr); maat_plugin_table_all_callback_finish(maat_inst->tbl_mgr);
if (maat_inst->creating_maat_rt != NULL) { if (maat_inst->creating_maat_rt != NULL) {
maat_runtime_commit(maat_inst->creating_maat_rt, MAAT_UPDATE_TYPE_FULL, maat_runtime_commit(maat_inst->creating_maat_rt,
maat_inst->creating_maat_rt->version, maat_inst->logger); MAAT_UPDATE_TYPE_FULL,
maat_inst->creating_maat_rt->rule_num = maat_runtime_rule_num(maat_inst->creating_maat_rt); maat_inst->creating_maat_rt->version,
maat_inst->logger);
maat_inst->creating_maat_rt->rule_num =
maat_runtime_rule_num(maat_inst->creating_maat_rt);
log_info(maat_inst->logger, MODULE_MAAT_RULE, log_info(maat_inst->logger, MODULE_MAAT_RULE,
"Full config version %llu load %d entries complete", "Full config version %llu load %d entries complete",
maat_inst->creating_maat_rt->version, maat_inst->creating_maat_rt->version,
@@ -265,7 +286,10 @@ static void maat_finish_cb(void *u_param)
maat_inst->maat_rt->version = maat_inst->maat_version; maat_inst->maat_rt->version = maat_inst->maat_version;
maat_runtime_commit(maat_inst->maat_rt, MAAT_UPDATE_TYPE_INC, maat_runtime_commit(maat_inst->maat_rt, MAAT_UPDATE_TYPE_INC,
maat_inst->maat_rt->version, maat_inst->logger); maat_inst->maat_rt->version, maat_inst->logger);
maat_inst->maat_rt->rule_num = maat_runtime_rule_num(maat_inst->maat_rt);
maat_inst->maat_rt->rule_num =
maat_runtime_rule_num(maat_inst->maat_rt);
log_info(maat_inst->logger, MODULE_MAAT_RULE, log_info(maat_inst->logger, MODULE_MAAT_RULE,
"Inc config version %llu load %d entries complete", "Inc config version %llu load %d entries complete",
maat_inst->maat_rt->version, maat_inst->maat_rt->version,
@@ -290,21 +314,23 @@ void maat_read_full_config(struct maat *maat_inst)
redis_ctx = &(maat_inst->opts.redis_ctx); redis_ctx = &(maat_inst->opts.redis_ctx);
log_info(maat_inst->logger, MODULE_MAAT_RULE, log_info(maat_inst->logger, MODULE_MAAT_RULE,
"Maat initiate from Redis %s:%hu db%d", "Maat initiate from Redis %s:%hu db%d",
redis_ctx->redis_ip, redis_ctx->redis_port, redis_ctx->redis_db); redis_ctx->redis_ip, redis_ctx->redis_port,
redis_ctx->redis_db);
redis_ctx->read_ctx = maat_connect_redis(redis_ctx->redis_ip, redis_ctx->read_ctx = maat_connect_redis(redis_ctx->redis_ip,
redis_ctx->redis_port, redis_ctx->redis_port,
redis_ctx->redis_db, redis_ctx->redis_db,
maat_inst->logger); maat_inst->logger);
if (redis_ctx->read_ctx != NULL) { if (redis_ctx->read_ctx != NULL) {
redis_monitor_traverse(maat_inst->maat_version, redis_ctx, redis_monitor_traverse(maat_inst->maat_version, redis_ctx,
maat_start_cb, maat_update_cb, maat_finish_cb, maat_start_cb, maat_update_cb,
maat_inst); maat_finish_cb, maat_inst);
} }
if (NULL == maat_inst->creating_maat_rt) { if (NULL == maat_inst->creating_maat_rt) {
log_fatal(maat_inst->logger, MODULE_MAAT_RULE, log_fatal(maat_inst->logger, MODULE_MAAT_RULE,
"[%s:%d] At initiation: NO effective rule in redis %s:%hu db%d", "[%s:%d] At initiation: NO effective rule in redis"
__FUNCTION__, __LINE__, redis_ctx->redis_ip, redis_ctx->redis_port, " %s:%hu db%d", __FUNCTION__, __LINE__,
redis_ctx->redis_ip, redis_ctx->redis_port,
redis_ctx->redis_db); redis_ctx->redis_db);
} }
break; break;
@@ -326,7 +352,8 @@ void maat_read_full_config(struct maat *maat_inst)
if (ret < 0) { if (ret < 0) {
log_fatal(maat_inst->logger, MODULE_MAAT_RULE, log_fatal(maat_inst->logger, MODULE_MAAT_RULE,
"[%s:%d] Maat re-initiate with JSON file %s failed: %s", "[%s:%d] Maat re-initiate with JSON file %s failed: %s",
__FUNCTION__, __LINE__, maat_inst->opts.json_ctx.json_file, err_str); __FUNCTION__, __LINE__, maat_inst->opts.json_ctx.json_file,
err_str);
} }
config_monitor_traverse(maat_inst->maat_version, config_monitor_traverse(maat_inst->maat_version,
@@ -347,13 +374,15 @@ void maat_read_full_config(struct maat *maat_inst)
maat_inst->maat_rt = maat_inst->creating_maat_rt; maat_inst->maat_rt = maat_inst->creating_maat_rt;
maat_inst->creating_maat_rt = NULL; maat_inst->creating_maat_rt = NULL;
maat_inst->is_running = 1; maat_inst->is_running = 1;
if (maat_inst->maat_rt != NULL) { if (maat_inst->maat_rt != NULL) {
maat_inst->maat_version = maat_inst->maat_rt->version; maat_inst->maat_version = maat_inst->maat_rt->version;
maat_inst->last_full_version = maat_inst->maat_rt->version; maat_inst->last_full_version = maat_inst->maat_rt->version;
} }
} }
long long maat_runtime_get_sequence(struct maat_runtime *maat_rt, const char *key) long long maat_runtime_get_sequence(struct maat_runtime *maat_rt,
const char *key)
{ {
if (NULL == maat_rt || NULL == key) { if (NULL == maat_rt || NULL == key) {
return -1; return -1;
@@ -406,7 +435,8 @@ void *rule_monitor_loop(void *arg)
/* if deferred load on */ /* if deferred load on */
if (maat_inst->opts.deferred_load_on != 0) { if (maat_inst->opts.deferred_load_on != 0) {
log_info(maat_inst->logger, MODULE_MAAT_RULE, log_info(maat_inst->logger, MODULE_MAAT_RULE,
"Deferred Loading ON, updating in %s:%d", __FUNCTION__, __LINE__); "Deferred Loading ON, updating in %s:%d",
__FUNCTION__, __LINE__);
maat_read_full_config(maat_inst); maat_read_full_config(maat_inst);
} }
pthread_mutex_unlock(&(maat_inst->background_update_mutex)); pthread_mutex_unlock(&(maat_inst->background_update_mutex));
@@ -484,8 +514,9 @@ void *rule_monitor_loop(void *arg)
old_maat_rt->version, maat_inst->maat_rt->version); old_maat_rt->version, maat_inst->maat_rt->version);
} }
maat_inst->stat->zombie_rs_stream += alignment_int64_array_sum(old_maat_rt->ref_cnt, maat_inst->stat->zombie_rs_stream +=
maat_inst->opts.nr_worker_thread); alignment_int64_array_sum(old_maat_rt->ref_cnt,
maat_inst->opts.nr_worker_thread);
maat_garbage_bagging(maat_inst->garbage_bin, old_maat_rt, NULL, maat_garbage_bagging(maat_inst->garbage_bin, old_maat_rt, NULL,
garbage_maat_runtime_destroy); garbage_maat_runtime_destroy);
} }
@@ -514,7 +545,8 @@ void *rule_monitor_loop(void *arg)
maat_plugin_table_garbage_collect_routine(maat_inst->tbl_mgr); maat_plugin_table_garbage_collect_routine(maat_inst->tbl_mgr);
if ((1 == maat_inst->opts.stat_on) && (time(NULL) % 2 == 0)) { if ((1 == maat_inst->opts.stat_on) && (time(NULL) % 2 == 0)) {
maat_stat_output(maat_inst->stat, maat_inst->maat_version, maat_inst->opts.perf_on); maat_stat_output(maat_inst->stat, maat_inst->maat_version,
maat_inst->opts.perf_on);
} }
} }

77
test/ipport_plugin/ipport_plugin_gtest.cpp
View File

@@ -53,7 +53,8 @@ int make_serial_rule(const char *table_name, const char *line, void *u_para)
} }
const char *redis_rule_key = "TEST_RULE_KEY"; const char *redis_rule_key = "TEST_RULE_KEY";
redisReply *reply = maat_wrap_redis_command(ctx, "INCRBY %s %d", redis_rule_key, 1); redisReply *reply =
maat_wrap_redis_command(ctx, NULL, "INCRBY %s %d", redis_rule_key, 1);
if (reply->type == REDIS_REPLY_NIL) { if (reply->type == REDIS_REPLY_NIL) {
printf("incrby redis_rule_key:%s failed.", redis_rule_key); printf("incrby redis_rule_key:%s failed.", redis_rule_key);
return -1; return -1;
@@ -63,8 +64,8 @@ int make_serial_rule(const char *table_name, const char *line, void *u_para)
reply = NULL; reply = NULL;
} }
maat_set_serial_rule(s_rule + line_idx, MAAT_OP_ADD, s_rule->rule_id, table_name, maat_set_serial_rule(s_rule + line_idx, MAAT_OP_ADD, s_rule->rule_id,
buff, absolute_expire_time); table_name, buff, absolute_expire_time);
(s_rule + line_idx)->ref_ctx = ctx; (s_rule + line_idx)->ref_ctx = ctx;
line_idx++; line_idx++;
@@ -76,12 +77,13 @@ int make_serial_rule(const char *table_name, const char *line, void *u_para)
int write_config_to_redis(char *redis_ip, int redis_port, int redis_db, int write_config_to_redis(char *redis_ip, int redis_port, int redis_db,
struct log_handle *logger) struct log_handle *logger)
{ {
redisContext *c = maat_connect_redis(redis_ip, redis_port, redis_db, logger); redisContext *c =
maat_connect_redis(redis_ip, redis_port, redis_db, logger);
if (NULL == c) { if (NULL == c) {
return -1; return -1;
} }
redisReply *reply = maat_wrap_redis_command(c, "flushdb"); redisReply *reply = maat_wrap_redis_command(c, logger, "flushdb");
if (NULL == reply) { if (NULL == reply) {
return -1; return -1;
} else { } else {
@@ -89,7 +91,7 @@ int write_config_to_redis(char *redis_ip, int redis_port, int redis_db,
reply = NULL; reply = NULL;
} }
reply = maat_wrap_redis_command(c, "SET MAAT_VERSION 1"); reply = maat_wrap_redis_command(c, logger, "SET MAAT_VERSION 1");
if (NULL == reply) { if (NULL == reply) {
return -1; return -1;
} else { } else {
@@ -97,7 +99,7 @@ int write_config_to_redis(char *redis_ip, int redis_port, int redis_db,
reply = NULL; reply = NULL;
} }
reply = maat_wrap_redis_command(c, "SET MAAT_PRE_VER 1"); reply = maat_wrap_redis_command(c, logger, "SET MAAT_PRE_VER 1");
if (NULL == reply) { if (NULL == reply) {
return -1; return -1;
} else { } else {
@@ -105,7 +107,7 @@ int write_config_to_redis(char *redis_ip, int redis_port, int redis_db,
reply = NULL; reply = NULL;
} }
reply = maat_wrap_redis_command(c, "SET SEQUENCE_REGION 1"); reply = maat_wrap_redis_command(c, logger, "SET SEQUENCE_REGION 1");
if (NULL == reply) { if (NULL == reply) {
return -1; return -1;
} else { } else {
@@ -113,7 +115,7 @@ int write_config_to_redis(char *redis_ip, int redis_port, int redis_db,
reply = NULL; reply = NULL;
} }
reply = maat_wrap_redis_command(c, "SET SEQUENCE_GROUP 1"); reply = maat_wrap_redis_command(c, logger, "SET SEQUENCE_GROUP 1");
if (NULL == reply) { if (NULL == reply) {
return -1; return -1;
} else { } else {
@@ -143,7 +145,8 @@ int write_config_to_redis(char *redis_ip, int redis_port, int redis_db,
int success_cnt = 0; int success_cnt = 0;
do { do {
success_cnt = maat_cmd_write_rule(c, s_rule, total_line_cnt, server_time, logger); success_cnt = maat_cmd_write_rule(c, s_rule, total_line_cnt,
server_time, logger);
} while (success_cnt < 0); } while (success_cnt < 0);
EXPECT_EQ(success_cnt, (int)total_line_cnt); EXPECT_EQ(success_cnt, (int)total_line_cnt);
@@ -162,8 +165,9 @@ struct ipport_plugin_ud {
char *buffer; char *buffer;
size_t buf_len; size_t buf_len;
}; };
void ipport_plugin_ex_new_cb(const char *table_name, int table_id, const char *key, void ipport_plugin_ex_new_cb(const char *table_name, int table_id,
const char *table_line, void **ad, long argl, void *argp) const char *key, const char *table_line,
void **ad, long argl, void *argp)
{ {
int *counter = (int *)argp; int *counter = (int *)argp;
size_t column_offset=0, column_len=0; size_t column_offset=0, column_len=0;
@@ -198,7 +202,8 @@ void ipport_plugin_ex_free_cb(int table_id, void **ad, long argl, void *argp)
*ad = NULL; *ad = NULL;
} }
void ipport_plugin_ex_dup_cb(int table_id, void **to, void **from, long argl, void *argp) void ipport_plugin_ex_dup_cb(int table_id, void **to, void **from,
long argl, void *argp)
{ {
struct ipport_plugin_ud *ud = (struct ipport_plugin_ud *)(*from); struct ipport_plugin_ud *ud = (struct ipport_plugin_ud *)(*from);
@@ -234,8 +239,9 @@ void *ipport_plugin_scan_thread(void *arg)
clock_gettime(CLOCK_MONOTONIC, &start); clock_gettime(CLOCK_MONOTONIC, &start);
for (int i = 0; i < param->test_count; i++) { for (int i = 0; i < param->test_count; i++) {
ret = maat_ipport_plugin_table_get_ex_data(maat_inst, table_id, &ipv4, port, ret = maat_ipport_plugin_table_get_ex_data(maat_inst, table_id, &ipv4,
(void **)results, ARRAY_SIZE); port, (void **)results,
ARRAY_SIZE);
EXPECT_EQ(ret, 1); EXPECT_EQ(ret, 1);
if (ret == 1) { if (ret == 1) {
hit_times++; hit_times++;
@@ -243,7 +249,8 @@ void *ipport_plugin_scan_thread(void *arg)
} }
clock_gettime(CLOCK_MONOTONIC, &end); clock_gettime(CLOCK_MONOTONIC, &end);
param->time_elapse_ms = (end.tv_sec - start.tv_sec) * 1000 + (end.tv_nsec - start.tv_nsec) / 1000000; param->time_elapse_ms = (end.tv_sec - start.tv_sec) * 1000 +
(end.tv_nsec - start.tv_nsec) / 1000000;
int *is_all_hit = ALLOC(int, 1); int *is_all_hit = ALLOC(int, 1);
*is_all_hit = (hit_times == param->test_count ? 1 : 0); *is_all_hit = (hit_times == param->test_count ? 1 : 0);
log_info(param->logger, MODULE_IPPORT_PLUGIN_GTEST, log_info(param->logger, MODULE_IPPORT_PLUGIN_GTEST,
@@ -252,15 +259,18 @@ void *ipport_plugin_scan_thread(void *arg)
return is_all_hit; return is_all_hit;
} }
static int test_add_ipport_plugin_command(struct maat *maat_inst, const char *table_name, static int
long long item_id, const char *ip_str, int port1, int port2) test_add_ipport_plugin_command(struct maat *maat_inst, const char *table_name,
long long item_id, const char *ip_str, int port1,
int port2)
{ {
int table_id = maat_get_table_id(maat_inst, table_name); int table_id = maat_get_table_id(maat_inst, table_name);
if (table_id < 0) { if (table_id < 0) {
return -1; return -1;
} }
enum table_type table_type = table_manager_get_table_type(maat_inst->tbl_mgr, table_id); enum table_type table_type =
table_manager_get_table_type(maat_inst->tbl_mgr, table_id);
if (table_type != TABLE_TYPE_IPPORT_PLUGIN) { if (table_type != TABLE_TYPE_IPPORT_PLUGIN) {
return -1; return -1;
} }
@@ -279,15 +289,18 @@ static int test_add_ipport_plugin_command(struct maat *maat_inst, const char *ta
return 0; return 0;
} }
static int test_del_ipport_plugin_command(struct maat *maat_inst, const char *table_name, static int
long long item_id, const char *ip_str, int port1, int port2) test_del_ipport_plugin_command(struct maat *maat_inst, const char *table_name,
long long item_id, const char *ip_str, int port1,
int port2)
{ {
int table_id = maat_get_table_id(maat_inst, table_name); int table_id = maat_get_table_id(maat_inst, table_name);
if (table_id < 0) { if (table_id < 0) {
return -1; return -1;
} }
enum table_type table_type = table_manager_get_table_type(maat_inst->tbl_mgr, table_id); enum table_type table_type =
table_manager_get_table_type(maat_inst->tbl_mgr, table_id);
if (table_type != TABLE_TYPE_IPPORT_PLUGIN) { if (table_type != TABLE_TYPE_IPPORT_PLUGIN) {
return -1; return -1;
} }
@@ -316,7 +329,8 @@ void *ipport_plugin_update_thread(void *arg)
int ret = 0; int ret = 0;
for (int i = 0; i < CMD_EXPR_NUM; i++) { for (int i = 0; i < CMD_EXPR_NUM; i++) {
ret = test_add_ipport_plugin_command(maat_inst, table_name, item_id, g_ip_str, i+201, i+201); ret = test_add_ipport_plugin_command(maat_inst, table_name, item_id,
g_ip_str, i+201, i+201);
if (ret < 0) { if (ret < 0) {
log_fatal(param->logger, MODULE_IPPORT_PLUGIN_GTEST, log_fatal(param->logger, MODULE_IPPORT_PLUGIN_GTEST,
"[%s:%d]add ipport rule(item_id:%lld) for table:%s failed.", "[%s:%d]add ipport rule(item_id:%lld) for table:%s failed.",
@@ -329,7 +343,8 @@ void *ipport_plugin_update_thread(void *arg)
item_id = 9000000; item_id = 9000000;
for (int i = 0; i < CMD_EXPR_NUM; i++) { for (int i = 0; i < CMD_EXPR_NUM; i++) {
ret = test_del_ipport_plugin_command(maat_inst, table_name, item_id, g_ip_str, i+201, i+201); ret = test_del_ipport_plugin_command(maat_inst, table_name, item_id,
g_ip_str, i+201, i+201);
if (ret < 0) { if (ret < 0) {
log_fatal(param->logger, MODULE_IPPORT_PLUGIN_GTEST, log_fatal(param->logger, MODULE_IPPORT_PLUGIN_GTEST,
"[%s:%d]del ipport rule(item_id:%lld) for table:%s failed.", "[%s:%d]del ipport rule(item_id:%lld) for table:%s failed.",
@@ -393,7 +408,8 @@ TEST_F(IPPortPluginTable, WITHOUT_SAME_IP) {
} }
pthread_create(&threads[0], NULL, ipport_plugin_scan_thread, thread_params); pthread_create(&threads[0], NULL, ipport_plugin_scan_thread, thread_params);
pthread_create(&threads[1], NULL, ipport_plugin_update_thread, thread_params + 1); pthread_create(&threads[1], NULL, ipport_plugin_update_thread,
thread_params + 1);
int *is_all_hit = NULL; int *is_all_hit = NULL;
long long time_elapse_ms = 0; long long time_elapse_ms = 0;
@@ -410,8 +426,8 @@ TEST_F(IPPortPluginTable, WITHOUT_SAME_IP) {
maat_free(maat_inst); maat_free(maat_inst);
scan_per_second = PERF_scan_times * 1000 / time_elapse_ms; scan_per_second = PERF_scan_times * 1000 / time_elapse_ms;
log_info(maat_inst->logger, MODULE_IPPORT_PLUGIN_GTEST, log_info(maat_inst->logger, MODULE_IPPORT_PLUGIN_GTEST,
"IpportPluginScan without same ip match rate speed %lld lookups/s/thread", "IpportPluginScan without same ip match rate speed"
scan_per_second); " %lld lookups/s/thread", scan_per_second);
} }
TEST_F(IPPortPluginTable, WITH_256SAME_IP) { TEST_F(IPPortPluginTable, WITH_256SAME_IP) {
@@ -461,7 +477,8 @@ TEST_F(IPPortPluginTable, WITH_256SAME_IP) {
} }
pthread_create(&threads[0], NULL, ipport_plugin_scan_thread, thread_params); pthread_create(&threads[0], NULL, ipport_plugin_scan_thread, thread_params);
pthread_create(&threads[1], NULL, ipport_plugin_update_thread, thread_params + 1); pthread_create(&threads[1], NULL, ipport_plugin_update_thread,
thread_params + 1);
int *is_all_hit = NULL; int *is_all_hit = NULL;
long long time_elapse_ms = 0; long long time_elapse_ms = 0;
@@ -478,8 +495,8 @@ TEST_F(IPPortPluginTable, WITH_256SAME_IP) {
scan_per_second = PERF_scan_times * 1000 / time_elapse_ms; scan_per_second = PERF_scan_times * 1000 / time_elapse_ms;
log_info(maat_inst->logger, MODULE_IPPORT_PLUGIN_GTEST, log_info(maat_inst->logger, MODULE_IPPORT_PLUGIN_GTEST,
"IpportPluginScan with 256 same ip match rate speed %lld lookups/s/thread", "IpportPluginScan with 256 same ip match rate speed"
scan_per_second); " %lld lookups/s/thread", scan_per_second);
} }
int main(int argc, char ** argv) int main(int argc, char ** argv)

21
test/maat_framework_gtest.cpp
View File

@@ -44,7 +44,8 @@ int make_serial_rule(const char *table_name, const char *line, void *u_para)
} }
const char *redis_rule_key = "TEST_RULE_KEY"; const char *redis_rule_key = "TEST_RULE_KEY";
redisReply *reply = maat_wrap_redis_command(ctx, "INCRBY %s %d", redis_rule_key, 1); redisReply *reply = maat_wrap_redis_command(ctx, NULL, "INCRBY %s %d",
redis_rule_key, 1);
if (reply->type == REDIS_REPLY_NIL) { if (reply->type == REDIS_REPLY_NIL) {
printf("incrby redis_rule_key:%s failed.", redis_rule_key); printf("incrby redis_rule_key:%s failed.", redis_rule_key);
return -1; return -1;
@@ -77,7 +78,7 @@ int write_config_to_redis(char *redis_ip, int redis_port, int redis_db,
return -1; return -1;
} }
redisReply *reply = maat_wrap_redis_command(c, "flushdb"); redisReply *reply = maat_wrap_redis_command(c, logger, "flushdb");
if (NULL == reply) { if (NULL == reply) {
return -1; return -1;
} else { } else {
@@ -11168,9 +11169,10 @@ struct log_handle *MaatRollbackTest::logger;
static int clear_config_in_redis(redisContext *c, struct log_handle *logger) static int clear_config_in_redis(redisContext *c, struct log_handle *logger)
{ {
long long redis_version = 0; long long redis_version = 0;
redisReply *reply = maat_wrap_redis_command(c, "GET MAAT_VERSION"); redisReply *reply = maat_wrap_redis_command(c, logger, "GET MAAT_VERSION");
if (reply != NULL) { if (reply != NULL) {
if (reply->type == REDIS_REPLY_NIL || reply->type == REDIS_REPLY_ERROR) { if (reply->type == REDIS_REPLY_NIL ||
reply->type == REDIS_REPLY_ERROR) {
log_fatal(logger, MODULE_FRAMEWORK_GTEST, log_fatal(logger, MODULE_FRAMEWORK_GTEST,
"[%s:%d] GET MAAT_VERSION failed, maybe Redis is busy", "[%s:%d] GET MAAT_VERSION failed, maybe Redis is busy",
__FUNCTION__, __LINE__); __FUNCTION__, __LINE__);
@@ -11200,7 +11202,7 @@ static int clear_config_in_redis(redisContext *c, struct log_handle *logger)
freeReplyObject(reply); freeReplyObject(reply);
reply = NULL; reply = NULL;
reply = maat_wrap_redis_command(c, "MULTI"); reply = maat_wrap_redis_command(c, logger, "MULTI");
freeReplyObject(reply); freeReplyObject(reply);
reply = NULL; reply = NULL;
@@ -11237,12 +11239,15 @@ static int clear_config_in_redis(redisContext *c, struct log_handle *logger)
return 0; return 0;
} }
static int rollback_redis_version(redisContext *c, struct log_handle *logger) static int
rollback_redis_version(redisContext *c, struct log_handle *logger)
{ {
redisReply *reply = maat_wrap_redis_command(c, "SET MAAT_VERSION 0"); redisReply *reply =
maat_wrap_redis_command(c, logger, "SET MAAT_VERSION 0");
if (NULL == reply) { if (NULL == reply) {
log_fatal(logger, MODULE_FRAMEWORK_GTEST, log_fatal(logger, MODULE_FRAMEWORK_GTEST,
"[%s:%d] set MAAT_VERSION failed, Redis Communication error: %s", "[%s:%d] set MAAT_VERSION failed, "
"Redis Communication error: %s",
__FUNCTION__, __LINE__, c->errstr); __FUNCTION__, __LINE__, c->errstr);
return -1; return -1;
} }

390
test/maat_framework_perf_gtest.cpp
View File

@@ -87,15 +87,18 @@ int make_serial_rule(const char *table_name, const char *line, void *u_para)
return 0; return 0;
} }
static int write_config_to_redis(const char *json_iris_path, char *redis_ip, static int
int redis_port, int redis_db, struct log_handle *logger) write_config_to_redis(const char *json_iris_path,
char *redis_ip, int redis_port,
int redis_db, struct log_handle *logger)
{ {
redisContext *c = maat_connect_redis(redis_ip, redis_port, redis_db, logger); redisContext *c =
maat_connect_redis(redis_ip, redis_port, redis_db, logger);
if (NULL == c) { if (NULL == c) {
return -1; return -1;
} }
redisReply *reply = maat_wrap_redis_command(c, "flushdb"); redisReply *reply = maat_wrap_redis_command(c, logger, "flushdb");
if (NULL == reply) { if (NULL == reply) {
return -1; return -1;
} else { } else {
@@ -108,7 +111,8 @@ static int write_config_to_redis(const char *json_iris_path, char *redis_ip,
char *json_buff = NULL; char *json_buff = NULL;
size_t json_buff_sz = 0; size_t json_buff_sz = 0;
int ret = load_file_to_memory(json_filename, (unsigned char **)&json_buff, int ret = load_file_to_memory(json_filename,
(unsigned char **)&json_buff,
&json_buff_sz); &json_buff_sz);
if (ret < 0) { if (ret < 0) {
return -1; return -1;
@@ -127,8 +131,9 @@ static int write_config_to_redis(const char *json_iris_path, char *redis_ip,
snprintf(tmp_iris_full_idx_path, sizeof(tmp_iris_full_idx_path), snprintf(tmp_iris_full_idx_path, sizeof(tmp_iris_full_idx_path),
"%s/index", json_iris_path); "%s/index", json_iris_path);
config_monitor_traverse(0, tmp_iris_full_idx_path, NULL, count_line_num_cb, config_monitor_traverse(0, tmp_iris_full_idx_path, NULL,
NULL, &total_line_cnt, NULL, logger); count_line_num_cb, NULL, &total_line_cnt,
NULL, logger);
struct serial_rule *s_rule = ALLOC(struct serial_rule, total_line_cnt); struct serial_rule *s_rule = ALLOC(struct serial_rule, total_line_cnt);
long long server_time = maat_redis_server_time_s(c); long long server_time = maat_redis_server_time_s(c);
@@ -137,14 +142,16 @@ static int write_config_to_redis(const char *json_iris_path, char *redis_ip,
} }
//absolute_expire_time = server_time + 300; //absolute_expire_time = server_time + 300;
config_monitor_traverse(0, tmp_iris_full_idx_path, NULL, make_serial_rule, config_monitor_traverse(0, tmp_iris_full_idx_path, NULL,
NULL, s_rule, NULL, logger); make_serial_rule, NULL, s_rule,
NULL, logger);
line_idx = 0; line_idx = 0;
absolute_expire_time = 0; absolute_expire_time = 0;
int success_cnt = 0; int success_cnt = 0;
do { do {
success_cnt = maat_cmd_write_rule(c, s_rule, total_line_cnt, server_time, logger); success_cnt = maat_cmd_write_rule(c, s_rule, total_line_cnt,
server_time, logger);
} while (success_cnt < 0); } while (success_cnt < 0);
EXPECT_EQ(success_cnt, (int)total_line_cnt); EXPECT_EQ(success_cnt, (int)total_line_cnt);
@@ -158,10 +165,11 @@ static int write_config_to_redis(const char *json_iris_path, char *redis_ip,
return 0; return 0;
} }
static int compile_table_set_line(struct maat *maat_inst, const char *table_name, static int
enum maat_operation op, long long compile_id, compile_table_set_line(struct maat *maat_inst, const char *table_name,
const char *user_region, int clause_num, enum maat_operation op, long long compile_id,
int expire_after) const char *user_region, int clause_num,
int expire_after)
{ {
char table_line[1024 * 16] = {0}; char table_line[1024 * 16] = {0};
sprintf(table_line, "%lld\t0\t0\t0\t0\t0\t%s\t%d\t%d\t0.0", sprintf(table_line, "%lld\t0\t0\t0\t0\t0\t%s\t%d\t%d\t0.0",
@@ -176,11 +184,12 @@ static int compile_table_set_line(struct maat *maat_inst, const char *table_name
return maat_cmd_set_line(maat_inst, &line_rule); return maat_cmd_set_line(maat_inst, &line_rule);
} }
static int group2compile_table_set_line(struct maat *maat_inst, const char *table_name, static int
enum maat_operation op, long long group_id, group2compile_table_set_line(struct maat *maat_inst, const char *table_name,
long long compile_id, int not_flag, enum maat_operation op, long long group_id,
const char *vtable_name, int clause_index, long long compile_id, int not_flag,
int expire_after) const char *vtable_name, int clause_index,
int expire_after)
{ {
char table_line[128] = {0}; char table_line[128] = {0};
sprintf(table_line, "%lld\t%lld\t%d\t%s\t%d\t%d", sprintf(table_line, "%lld\t%lld\t%d\t%s\t%d\t%d",
@@ -195,11 +204,12 @@ static int group2compile_table_set_line(struct maat *maat_inst, const char *tabl
return maat_cmd_set_line(maat_inst, &line_rule); return maat_cmd_set_line(maat_inst, &line_rule);
} }
static int expr_table_set_line(struct maat *maat_inst, const char *table_name, static int
enum maat_operation op, long long item_id, expr_table_set_line(struct maat *maat_inst, const char *table_name,
long long group_id, const char *keywords, enum maat_operation op, long long item_id,
const char *district, int expr_type, long long group_id, const char *keywords,
int match_method, int is_hexbin, int expire_after) const char *district, int expr_type,
int match_method, int is_hexbin, int expire_after)
{ {
char table_line[1024] = {0}; char table_line[1024] = {0};
int table_id = maat_get_table_id(maat_inst, table_name); int table_id = maat_get_table_id(maat_inst, table_name);
@@ -207,7 +217,8 @@ static int expr_table_set_line(struct maat *maat_inst, const char *table_name,
return 0; return 0;
} }
enum table_type table_type = table_manager_get_table_type(maat_inst->tbl_mgr, table_id); enum table_type table_type =
table_manager_get_table_type(maat_inst->tbl_mgr, table_id);
assert(table_type == TABLE_TYPE_EXPR || table_type == TABLE_TYPE_EXPR_PLUS); assert(table_type == TABLE_TYPE_EXPR || table_type == TABLE_TYPE_EXPR_PLUS);
if (table_type == TABLE_TYPE_EXPR_PLUS) { if (table_type == TABLE_TYPE_EXPR_PLUS) {
@@ -229,10 +240,11 @@ static int expr_table_set_line(struct maat *maat_inst, const char *table_name,
return maat_cmd_set_line(maat_inst, &line_rule); return maat_cmd_set_line(maat_inst, &line_rule);
} }
static int ip_table_set_line(struct maat *maat_inst, const char *table_name, static int
enum maat_operation op, long long item_id, ip_table_set_line(struct maat *maat_inst, const char *table_name,
long long group_id, enum IP_TYPE type, const char *ip1, enum maat_operation op, long long item_id,
const char *ip2, int expire_after) long long group_id, enum IP_TYPE type,
const char *ip1, const char *ip2, int expire_after)
{ {
char table_line[1024] = {0}; char table_line[1024] = {0};
int table_id = maat_get_table_id(maat_inst, table_name); int table_id = maat_get_table_id(maat_inst, table_name);
@@ -259,8 +271,9 @@ static int ip_table_set_line(struct maat *maat_inst, const char *table_name,
static int static int
integer_table_set_line(struct maat *maat_inst, const char *table_name, integer_table_set_line(struct maat *maat_inst, const char *table_name,
enum maat_operation op, long long item_id, long long group_id, enum maat_operation op, long long item_id,
int low_boundary, int up_boundary, int expire_after) long long group_id, int low_boundary,
int up_boundary, int expire_after)
{ {
char table_line[1024] = {0}; char table_line[1024] = {0};
int table_id = maat_get_table_id(maat_inst, table_name); int table_id = maat_get_table_id(maat_inst, table_name);
@@ -280,10 +293,11 @@ integer_table_set_line(struct maat *maat_inst, const char *table_name,
return maat_cmd_set_line(maat_inst, &line_rule); return maat_cmd_set_line(maat_inst, &line_rule);
} }
static int flag_table_set_line(struct maat *maat_inst, const char *table_name, static int
enum maat_operation op, long long item_id, flag_table_set_line(struct maat *maat_inst, const char *table_name,
long long group_id, long long flag, enum maat_operation op, long long item_id,
long long flag_mask, int expire_after) long long group_id, long long flag,
long long flag_mask, int expire_after)
{ {
char table_line[1024] = {0}; char table_line[1024] = {0};
int table_id = maat_get_table_id(maat_inst, table_name); int table_id = maat_get_table_id(maat_inst, table_name);
@@ -303,12 +317,14 @@ static int flag_table_set_line(struct maat *maat_inst, const char *table_name,
return maat_cmd_set_line(maat_inst, &line_rule); return maat_cmd_set_line(maat_inst, &line_rule);
} }
static void test_add_expr_command(struct maat *maat_inst, const char *table_name, static void
const char *keywords) test_add_expr_command(struct maat *maat_inst, const char *table_name,
const char *keywords)
{ {
long long compile_id = maat_cmd_incrby(maat_inst, "TEST_SEQ", 1); long long compile_id = maat_cmd_incrby(maat_inst, "TEST_SEQ", 1);
int ret = compile_table_set_line(maat_inst, "COMPILE_DEFAULT", MAAT_OP_ADD, int ret = compile_table_set_line(maat_inst, "COMPILE_DEFAULT",
compile_id, "null", 1, 0); MAAT_OP_ADD, compile_id, "null",
1, 0);
EXPECT_EQ(ret, 1); EXPECT_EQ(ret, 1);
long long group_id = maat_cmd_incrby(maat_inst, "SEQUENCE_GROUP", 1); long long group_id = maat_cmd_incrby(maat_inst, "SEQUENCE_GROUP", 1);
@@ -323,12 +339,14 @@ static void test_add_expr_command(struct maat *maat_inst, const char *table_name
EXPECT_EQ(ret, 1); EXPECT_EQ(ret, 1);
} }
static void test_add_ip_command(struct maat *maat_inst, const char *table_name, static void
const char *ip) test_add_ip_command(struct maat *maat_inst, const char *table_name,
const char *ip)
{ {
long long compile_id = maat_cmd_incrby(maat_inst, "TEST_SEQ", 1); long long compile_id = maat_cmd_incrby(maat_inst, "TEST_SEQ", 1);
int ret = compile_table_set_line(maat_inst, "COMPILE_DEFAULT", MAAT_OP_ADD, int ret = compile_table_set_line(maat_inst, "COMPILE_DEFAULT",
compile_id, "null", 1, 0); MAAT_OP_ADD, compile_id, "null",
1, 0);
EXPECT_EQ(ret, 1); EXPECT_EQ(ret, 1);
long long group_id = maat_cmd_incrby(maat_inst, "SEQUENCE_GROUP", 1); long long group_id = maat_cmd_incrby(maat_inst, "SEQUENCE_GROUP", 1);
@@ -343,12 +361,14 @@ static void test_add_ip_command(struct maat *maat_inst, const char *table_name,
EXPECT_EQ(ret, 1); EXPECT_EQ(ret, 1);
} }
static void test_add_integer_command(struct maat *maat_inst, const char *table_name, static void
int low_boundary, int up_boundary) test_add_integer_command(struct maat *maat_inst, const char *table_name,
int low_boundary, int up_boundary)
{ {
long long compile_id = maat_cmd_incrby(maat_inst, "TEST_SEQ", 1); long long compile_id = maat_cmd_incrby(maat_inst, "TEST_SEQ", 1);
int ret = compile_table_set_line(maat_inst, "COMPILE_DEFAULT", MAAT_OP_ADD, int ret = compile_table_set_line(maat_inst, "COMPILE_DEFAULT",
compile_id, "null", 1, 0); MAAT_OP_ADD, compile_id, "null",
1, 0);
EXPECT_EQ(ret, 1); EXPECT_EQ(ret, 1);
long long group_id = maat_cmd_incrby(maat_inst, "SEQUENCE_GROUP", 1); long long group_id = maat_cmd_incrby(maat_inst, "SEQUENCE_GROUP", 1);
@@ -363,12 +383,14 @@ static void test_add_integer_command(struct maat *maat_inst, const char *table_n
EXPECT_EQ(ret, 1); EXPECT_EQ(ret, 1);
} }
static void test_add_flag_command(struct maat *maat_inst, const char *table_name, static void
long long flag, long long flag_mask) test_add_flag_command(struct maat *maat_inst, const char *table_name,
long long flag, long long flag_mask)
{ {
long long compile_id = maat_cmd_incrby(maat_inst, "TEST_SEQ", 1); long long compile_id = maat_cmd_incrby(maat_inst, "TEST_SEQ", 1);
int ret = compile_table_set_line(maat_inst, "COMPILE_DEFAULT", MAAT_OP_ADD, int ret = compile_table_set_line(maat_inst, "COMPILE_DEFAULT",
compile_id, "null", 1, 0); MAAT_OP_ADD, compile_id, "null",
1, 0);
EXPECT_EQ(ret, 1); EXPECT_EQ(ret, 1);
long long group_id = maat_cmd_incrby(maat_inst, "SEQUENCE_GROUP", 1); long long group_id = maat_cmd_incrby(maat_inst, "SEQUENCE_GROUP", 1);
@@ -387,26 +409,31 @@ class MaatPerfStringScan : public testing::Test
{ {
protected: protected:
static void SetUpTestCase() { static void SetUpTestCase() {
const char *accept_tags = "{\"tags\":[{\"tag\":\"location\",\"value\":\"北京/朝阳/华严北里/甲22号\"}," const char *accept_tags =
"{\"tag\":\"isp\",\"value\":\"移动\"},{\"tag\":\"location\",\"value\":\"Astana\"}]}"; "{\"tags\":[{\"tag\":\"location\",\"value\":\"北京/朝阳/华严北里/甲22号\"},"
"{\"tag\":\"isp\",\"value\":\"移动\"},{\"tag\":\"location\",\"value\":\"Astana\"}]}";
char redis_ip[64] = "127.0.0.1"; char redis_ip[64] = "127.0.0.1";
int redis_port = 6379; int redis_port = 6379;
int redis_db = 0; int redis_db = 0;
char json_iris_path[512] = {0}; char json_iris_path[512] = {0};
snprintf(json_iris_path, sizeof(json_iris_path), "./%s_iris_tmp", json_filename); snprintf(json_iris_path, sizeof(json_iris_path),
"./%s_iris_tmp", json_filename);
logger = log_handle_create("./maat_framework_perf_gtest.log", 0); logger = log_handle_create("./maat_framework_perf_gtest.log", 0);
int ret = write_config_to_redis(json_iris_path, redis_ip, redis_port, redis_db, logger); int ret = write_config_to_redis(json_iris_path, redis_ip, redis_port,
redis_db, logger);
if (ret < 0) { if (ret < 0) {
log_fatal(logger, MODULE_FRAMEWORK_PERF_GTEST, log_fatal(logger, MODULE_FRAMEWORK_PERF_GTEST,
"[%s:%d] write config to redis failed.", __FUNCTION__, __LINE__); "[%s:%d] write config to redis failed.",
__FUNCTION__, __LINE__);
} }
struct maat_options *opts = maat_options_new(); struct maat_options *opts = maat_options_new();
maat_options_set_stat_file(opts, "./stat.log"); maat_options_set_stat_file(opts, "./stat.log");
maat_options_set_perf_on(opts); maat_options_set_perf_on(opts);
maat_options_set_redis(opts, redis_ip, redis_port, redis_db); maat_options_set_redis(opts, redis_ip, redis_port, redis_db);
maat_options_set_logger(opts, "./maat_framework_perf_gtest.log", LOG_LEVEL_INFO); maat_options_set_logger(opts, "./maat_framework_perf_gtest.log",
LOG_LEVEL_INFO);
maat_options_set_accept_tags(opts, accept_tags); maat_options_set_accept_tags(opts, accept_tags);
maat_options_set_caller_thread_number(opts, 5); maat_options_set_caller_thread_number(opts, 5);
@@ -448,8 +475,9 @@ void *perf_string_scan_thread(void *arg)
struct timespec start, end; struct timespec start, end;
clock_gettime(CLOCK_MONOTONIC, &start); clock_gettime(CLOCK_MONOTONIC, &start);
for (int i = 0; i < param->test_times; i++) { for (int i = 0; i < param->test_times; i++) {
int ret = maat_scan_string(maat_inst, table_id, scan_data, strlen(scan_data), int ret = maat_scan_string(maat_inst, table_id, scan_data,
results, ARRAY_SIZE, &n_hit_result, state); strlen(scan_data), results, ARRAY_SIZE,
&n_hit_result, state);
if (ret == MAAT_SCAN_HIT) { if (ret == MAAT_SCAN_HIT) {
hit_times++; hit_times++;
} }
@@ -508,10 +536,12 @@ TEST_F(MaatPerfStringScan, LiteralMultiThread) {
thread_params[i].logger = logger; thread_params[i].logger = logger;
if (i < PERF_THREAD_NUM) { if (i < PERF_THREAD_NUM) {
pthread_create(&threads[i], NULL, perf_string_scan_thread, thread_params+i); pthread_create(&threads[i], NULL, perf_string_scan_thread,
thread_params+i);
} else { } else {
thread_params[i].test_times = 0; thread_params[i].test_times = 0;
pthread_create(&threads[i], NULL, perf_string_update_thread, thread_params+i); pthread_create(&threads[i], NULL, perf_string_update_thread,
thread_params+i);
} }
} }
@@ -537,26 +567,31 @@ class MaatPerfRegexScan : public testing::Test
{ {
protected: protected:
static void SetUpTestCase() { static void SetUpTestCase() {
const char *accept_tags = "{\"tags\":[{\"tag\":\"location\",\"value\":\"北京/朝阳/华严北里/甲22号\"}," const char *accept_tags =
"{\"tag\":\"isp\",\"value\":\"移动\"},{\"tag\":\"location\",\"value\":\"Astana\"}]}"; "{\"tags\":[{\"tag\":\"location\",\"value\":\"北京/朝阳/华严北里/甲22号\"},"
"{\"tag\":\"isp\",\"value\":\"移动\"},{\"tag\":\"location\",\"value\":\"Astana\"}]}";
char redis_ip[64] = "127.0.0.1"; char redis_ip[64] = "127.0.0.1";
int redis_port = 6379; int redis_port = 6379;
int redis_db = 0; int redis_db = 0;
char json_iris_path[512] = {0}; char json_iris_path[512] = {0};
snprintf(json_iris_path, sizeof(json_iris_path), "./%s_iris_tmp", json_filename); snprintf(json_iris_path, sizeof(json_iris_path),
"./%s_iris_tmp", json_filename);
logger = log_handle_create("./maat_framework_perf_gtest.log", 0); logger = log_handle_create("./maat_framework_perf_gtest.log", 0);
int ret = write_config_to_redis(json_iris_path, redis_ip, redis_port, redis_db, logger); int ret = write_config_to_redis(json_iris_path, redis_ip, redis_port,
redis_db, logger);
if (ret < 0) { if (ret < 0) {
log_fatal(logger, MODULE_FRAMEWORK_PERF_GTEST, log_fatal(logger, MODULE_FRAMEWORK_PERF_GTEST,
"[%s:%d] write config to redis failed.", __FUNCTION__, __LINE__); "[%s:%d] write config to redis failed.",
__FUNCTION__, __LINE__);
} }
struct maat_options *opts = maat_options_new(); struct maat_options *opts = maat_options_new();
maat_options_set_stat_file(opts, "./stat.log"); maat_options_set_stat_file(opts, "./stat.log");
maat_options_set_perf_on(opts); maat_options_set_perf_on(opts);
maat_options_set_redis(opts, redis_ip, redis_port, redis_db); maat_options_set_redis(opts, redis_ip, redis_port, redis_db);
maat_options_set_logger(opts, "./maat_framework_perf_gtest.log", LOG_LEVEL_INFO); maat_options_set_logger(opts, "./maat_framework_perf_gtest.log",
LOG_LEVEL_INFO);
maat_options_set_accept_tags(opts, accept_tags); maat_options_set_accept_tags(opts, accept_tags);
maat_options_set_caller_thread_number(opts, 5); maat_options_set_caller_thread_number(opts, 5);
@@ -587,7 +622,8 @@ void *perf_regex_scan_thread(void *arg)
struct maat *maat_inst = param->maat_inst; struct maat *maat_inst = param->maat_inst;
const char *table_name = param->table_name; const char *table_name = param->table_name;
struct timespec start, end; struct timespec start, end;
const char *scan_data = "http://www.cyberessays.com/search_results.php?action=search&query=username,abckkk,1234567"; const char *scan_data =
"http://www.cyberessays.com/search_results.php?action=search&query=username,abckkk,1234567";
long long results[ARRAY_SIZE] = {0}; long long results[ARRAY_SIZE] = {0};
int hit_times = 0; int hit_times = 0;
size_t n_hit_result = 0; size_t n_hit_result = 0;
@@ -598,8 +634,9 @@ void *perf_regex_scan_thread(void *arg)
clock_gettime(CLOCK_MONOTONIC, &start); clock_gettime(CLOCK_MONOTONIC, &start);
for (int i = 0; i < param->test_times; i++) { for (int i = 0; i < param->test_times; i++) {
int ret = maat_scan_string(maat_inst, table_id, scan_data, strlen(scan_data), int ret = maat_scan_string(maat_inst, table_id, scan_data,
results, ARRAY_SIZE, &n_hit_result, state); strlen(scan_data), results, ARRAY_SIZE,
&n_hit_result, state);
if (ret == MAAT_SCAN_HIT) { if (ret == MAAT_SCAN_HIT) {
hit_times++; hit_times++;
} }
@@ -658,10 +695,12 @@ TEST_F(MaatPerfRegexScan, RegexMultiThread) {
thread_params[i].logger = logger; thread_params[i].logger = logger;
if (i < PERF_THREAD_NUM) { if (i < PERF_THREAD_NUM) {
pthread_create(&threads[i], NULL, perf_regex_scan_thread, thread_params+i); pthread_create(&threads[i], NULL, perf_regex_scan_thread,
thread_params+i);
} else { } else {
thread_params[i].test_times = 0; thread_params[i].test_times = 0;
pthread_create(&threads[i], NULL, perf_regex_update_thread, thread_params+i); pthread_create(&threads[i], NULL, perf_regex_update_thread,
thread_params+i);
} }
} }
@@ -741,26 +780,31 @@ class MaatPerfStreamScan : public testing::Test
{ {
protected: protected:
static void SetUpTestCase() { static void SetUpTestCase() {
const char *accept_tags = "{\"tags\":[{\"tag\":\"location\",\"value\":\"北京/朝阳/华严北里/甲22号\"}," const char *accept_tags =
"{\"tag\":\"isp\",\"value\":\"移动\"},{\"tag\":\"location\",\"value\":\"Astana\"}]}"; "{\"tags\":[{\"tag\":\"location\",\"value\":\"北京/朝阳/华严北里/甲22号\"},"
"{\"tag\":\"isp\",\"value\":\"移动\"},{\"tag\":\"location\",\"value\":\"Astana\"}]}";
char redis_ip[64] = "127.0.0.1"; char redis_ip[64] = "127.0.0.1";
int redis_port = 6379; int redis_port = 6379;
int redis_db = 0; int redis_db = 0;
char json_iris_path[512] = {0}; char json_iris_path[512] = {0};
snprintf(json_iris_path, sizeof(json_iris_path), "./%s_iris_tmp", json_filename); snprintf(json_iris_path, sizeof(json_iris_path),
"./%s_iris_tmp", json_filename);
logger = log_handle_create("./maat_framework_perf_gtest.log", 0); logger = log_handle_create("./maat_framework_perf_gtest.log", 0);
int ret = write_config_to_redis(json_iris_path, redis_ip, redis_port, redis_db, logger); int ret = write_config_to_redis(json_iris_path, redis_ip, redis_port,
redis_db, logger);
if (ret < 0) { if (ret < 0) {
log_fatal(logger, MODULE_FRAMEWORK_PERF_GTEST, log_fatal(logger, MODULE_FRAMEWORK_PERF_GTEST,
"[%s:%d] write config to redis failed.", __FUNCTION__, __LINE__); "[%s:%d] write config to redis failed.",
__FUNCTION__, __LINE__);
} }
struct maat_options *opts = maat_options_new(); struct maat_options *opts = maat_options_new();
maat_options_set_stat_file(opts, "./stat.log"); maat_options_set_stat_file(opts, "./stat.log");
maat_options_set_perf_on(opts); maat_options_set_perf_on(opts);
maat_options_set_redis(opts, redis_ip, redis_port, redis_db); maat_options_set_redis(opts, redis_ip, redis_port, redis_db);
maat_options_set_logger(opts, "./maat_framework_perf_gtest.log", LOG_LEVEL_INFO); maat_options_set_logger(opts, "./maat_framework_perf_gtest.log",
LOG_LEVEL_INFO);
maat_options_set_accept_tags(opts, accept_tags); maat_options_set_accept_tags(opts, accept_tags);
maat_options_set_caller_thread_number(opts, 5); maat_options_set_caller_thread_number(opts, 5);
@@ -791,7 +835,8 @@ void *perf_stream_scan_thread(void *arg)
struct maat *maat_inst = param->maat_inst; struct maat *maat_inst = param->maat_inst;
const char *table_name = param->table_name; const char *table_name = param->table_name;
struct timespec start, end; struct timespec start, end;
const char *scan_data = "http://www.cyberessays.com/search_results.php?action=search&query=username,abckkk,1234567"; const char *scan_data =
"http://www.cyberessays.com/search_results.php?action=search&query=username,abckkk,1234567";
long long results[ARRAY_SIZE] = {0}; long long results[ARRAY_SIZE] = {0};
int ret = 0, hit_times = 0; int ret = 0, hit_times = 0;
size_t n_hit_result = 0; size_t n_hit_result = 0;
@@ -803,8 +848,8 @@ void *perf_stream_scan_thread(void *arg)
clock_gettime(CLOCK_MONOTONIC, &start); clock_gettime(CLOCK_MONOTONIC, &start);
for (int i = 0; i < param->test_times; i++) { for (int i = 0; i < param->test_times; i++) {
ret = maat_stream_scan(sp, scan_data, strlen(scan_data), results, ARRAY_SIZE, ret = maat_stream_scan(sp, scan_data, strlen(scan_data), results,
&n_hit_result, state); ARRAY_SIZE, &n_hit_result, state);
if (ret == MAAT_SCAN_HIT) { if (ret == MAAT_SCAN_HIT) {
hit_times++; hit_times++;
} }
@@ -846,7 +891,8 @@ TEST_F(MaatPerfStreamScan, MultiThread) {
thread_params[i].logger = logger; thread_params[i].logger = logger;
if (i < PERF_THREAD_NUM) { if (i < PERF_THREAD_NUM) {
pthread_create(&threads[i], NULL, perf_stream_scan_thread, thread_params+i); pthread_create(&threads[i], NULL, perf_stream_scan_thread,
thread_params+i);
} }
} }
@@ -872,26 +918,31 @@ class MaatPerfIPScan : public testing::Test
{ {
protected: protected:
static void SetUpTestCase() { static void SetUpTestCase() {
const char *accept_tags = "{\"tags\":[{\"tag\":\"location\",\"value\":\"北京/朝阳/华严北里/甲22号\"}," const char *accept_tags =
"{\"tag\":\"isp\",\"value\":\"移动\"},{\"tag\":\"location\",\"value\":\"Astana\"}]}"; "{\"tags\":[{\"tag\":\"location\",\"value\":\"北京/朝阳/华严北里/甲22号\"},"
"{\"tag\":\"isp\",\"value\":\"移动\"},{\"tag\":\"location\",\"value\":\"Astana\"}]}";
char redis_ip[64] = "127.0.0.1"; char redis_ip[64] = "127.0.0.1";
int redis_port = 6379; int redis_port = 6379;
int redis_db = 0; int redis_db = 0;
char json_iris_path[512] = {0}; char json_iris_path[512] = {0};
snprintf(json_iris_path, sizeof(json_iris_path), "./%s_iris_tmp", json_filename); snprintf(json_iris_path, sizeof(json_iris_path),
"./%s_iris_tmp", json_filename);
logger = log_handle_create("./maat_framework_perf_gtest.log", 0); logger = log_handle_create("./maat_framework_perf_gtest.log", 0);
int ret = write_config_to_redis(json_iris_path, redis_ip, redis_port, redis_db, logger); int ret = write_config_to_redis(json_iris_path, redis_ip, redis_port,
redis_db, logger);
if (ret < 0) { if (ret < 0) {
log_fatal(logger, MODULE_FRAMEWORK_PERF_GTEST, log_fatal(logger, MODULE_FRAMEWORK_PERF_GTEST,
"[%s:%d] write config to redis failed.", __FUNCTION__, __LINE__); "[%s:%d] write config to redis failed.",
__FUNCTION__, __LINE__);
} }
struct maat_options *opts = maat_options_new(); struct maat_options *opts = maat_options_new();
maat_options_set_stat_file(opts, "./stat.log"); maat_options_set_stat_file(opts, "./stat.log");
maat_options_set_perf_on(opts); maat_options_set_perf_on(opts);
maat_options_set_redis(opts, redis_ip, redis_port, redis_db); maat_options_set_redis(opts, redis_ip, redis_port, redis_db);
maat_options_set_logger(opts, "./maat_framework_perf_gtest.log", LOG_LEVEL_INFO); maat_options_set_logger(opts, "./maat_framework_perf_gtest.log",
LOG_LEVEL_INFO);
maat_options_set_accept_tags(opts, accept_tags); maat_options_set_accept_tags(opts, accept_tags);
maat_options_set_caller_thread_number(opts, 5); maat_options_set_caller_thread_number(opts, 5);
@@ -1007,10 +1058,12 @@ TEST_F(MaatPerfIPScan, MultiThread)
thread_params[i].logger = logger; thread_params[i].logger = logger;
if (i < PERF_THREAD_NUM) { if (i < PERF_THREAD_NUM) {
pthread_create(&threads[i], NULL, perf_ip_scan_thread, thread_params+i); pthread_create(&threads[i], NULL, perf_ip_scan_thread,
thread_params+i);
} else { } else {
thread_params[i].test_times = 0; thread_params[i].test_times = 0;
pthread_create(&threads[i], NULL, perf_ip_update_thread, thread_params+i); pthread_create(&threads[i], NULL, perf_ip_update_thread,
thread_params+i);
} }
} }
@@ -1037,26 +1090,31 @@ class MaatPerfIntegerScan : public testing::Test
{ {
protected: protected:
static void SetUpTestCase() { static void SetUpTestCase() {
const char *accept_tags = "{\"tags\":[{\"tag\":\"location\",\"value\":\"北京/朝阳/华严北里/甲22号\"}," const char *accept_tags =
"{\"tag\":\"isp\",\"value\":\"移动\"},{\"tag\":\"location\",\"value\":\"Astana\"}]}"; "{\"tags\":[{\"tag\":\"location\",\"value\":\"北京/朝阳/华严北里/甲22号\"},"
"{\"tag\":\"isp\",\"value\":\"移动\"},{\"tag\":\"location\",\"value\":\"Astana\"}]}";
char redis_ip[64] = "127.0.0.1"; char redis_ip[64] = "127.0.0.1";
int redis_port = 6379; int redis_port = 6379;
int redis_db = 0; int redis_db = 0;
char json_iris_path[512] = {0}; char json_iris_path[512] = {0};
snprintf(json_iris_path, sizeof(json_iris_path), "./%s_iris_tmp", json_filename); snprintf(json_iris_path, sizeof(json_iris_path),
"./%s_iris_tmp", json_filename);
logger = log_handle_create("./maat_framework_perf_gtest.log", 0); logger = log_handle_create("./maat_framework_perf_gtest.log", 0);
int ret = write_config_to_redis(json_iris_path, redis_ip, redis_port, redis_db, logger); int ret = write_config_to_redis(json_iris_path, redis_ip, redis_port,
redis_db, logger);
if (ret < 0) { if (ret < 0) {
log_fatal(logger, MODULE_FRAMEWORK_PERF_GTEST, log_fatal(logger, MODULE_FRAMEWORK_PERF_GTEST,
"[%s:%d] write config to redis failed.", __FUNCTION__, __LINE__); "[%s:%d] write config to redis failed.",
__FUNCTION__, __LINE__);
} }
struct maat_options *opts = maat_options_new(); struct maat_options *opts = maat_options_new();
maat_options_set_stat_file(opts, "./stat.log"); maat_options_set_stat_file(opts, "./stat.log");
maat_options_set_perf_on(opts); maat_options_set_perf_on(opts);
maat_options_set_redis(opts, redis_ip, redis_port, redis_db); maat_options_set_redis(opts, redis_ip, redis_port, redis_db);
maat_options_set_logger(opts, "./maat_framework_perf_gtest.log", LOG_LEVEL_INFO); maat_options_set_logger(opts, "./maat_framework_perf_gtest.log",
LOG_LEVEL_INFO);
maat_options_set_accept_tags(opts, accept_tags); maat_options_set_accept_tags(opts, accept_tags);
maat_options_set_caller_thread_number(opts, 5); maat_options_set_caller_thread_number(opts, 5);
@@ -1102,10 +1160,12 @@ TEST_F(MaatPerfIntegerScan, MultiThread) {
thread_params[i].logger = logger; thread_params[i].logger = logger;
if (i < PERF_THREAD_NUM) { if (i < PERF_THREAD_NUM) {
pthread_create(&threads[i], NULL, perf_integer_scan_thread, thread_params+i); pthread_create(&threads[i], NULL, perf_integer_scan_thread,
thread_params+i);
} else { } else {
thread_params[i].test_times = 0; thread_params[i].test_times = 0;
pthread_create(&threads[i], NULL, perf_integer_update_thread, thread_params+i); pthread_create(&threads[i], NULL, perf_integer_update_thread,
thread_params+i);
} }
} }
@@ -1132,26 +1192,31 @@ class MaatPerfFlagScan : public testing::Test
{ {
protected: protected:
static void SetUpTestCase() { static void SetUpTestCase() {
const char *accept_tags = "{\"tags\":[{\"tag\":\"location\",\"value\":\"北京/朝阳/华严北里/甲22号\"}," const char *accept_tags =
"{\"tag\":\"isp\",\"value\":\"移动\"},{\"tag\":\"location\",\"value\":\"Astana\"}]}"; "{\"tags\":[{\"tag\":\"location\",\"value\":\"北京/朝阳/华严北里/甲22号\"},"
"{\"tag\":\"isp\",\"value\":\"移动\"},{\"tag\":\"location\",\"value\":\"Astana\"}]}";
char redis_ip[64] = "127.0.0.1"; char redis_ip[64] = "127.0.0.1";
int redis_port = 6379; int redis_port = 6379;
int redis_db = 0; int redis_db = 0;
char json_iris_path[512] = {0}; char json_iris_path[512] = {0};
snprintf(json_iris_path, sizeof(json_iris_path), "./%s_iris_tmp", json_filename); snprintf(json_iris_path, sizeof(json_iris_path),
"./%s_iris_tmp", json_filename);
logger = log_handle_create("./maat_framework_perf_gtest.log", 0); logger = log_handle_create("./maat_framework_perf_gtest.log", 0);
int ret = write_config_to_redis(json_iris_path, redis_ip, redis_port, redis_db, logger); int ret = write_config_to_redis(json_iris_path, redis_ip, redis_port,
redis_db, logger);
if (ret < 0) { if (ret < 0) {
log_fatal(logger, MODULE_FRAMEWORK_PERF_GTEST, log_fatal(logger, MODULE_FRAMEWORK_PERF_GTEST,
"[%s:%d] write config to redis failed.", __FUNCTION__, __LINE__); "[%s:%d] write config to redis failed.",
__FUNCTION__, __LINE__);
} }
struct maat_options *opts = maat_options_new(); struct maat_options *opts = maat_options_new();
maat_options_set_stat_file(opts, "./stat.log"); maat_options_set_stat_file(opts, "./stat.log");
maat_options_set_perf_on(opts); maat_options_set_perf_on(opts);
maat_options_set_redis(opts, redis_ip, redis_port, redis_db); maat_options_set_redis(opts, redis_ip, redis_port, redis_db);
maat_options_set_logger(opts, "./maat_framework_perf_gtest.log", LOG_LEVEL_INFO); maat_options_set_logger(opts, "./maat_framework_perf_gtest.log",
LOG_LEVEL_INFO);
maat_options_set_accept_tags(opts, accept_tags); maat_options_set_accept_tags(opts, accept_tags);
maat_options_set_caller_thread_number(opts, 5); maat_options_set_caller_thread_number(opts, 5);
@@ -1250,10 +1315,12 @@ TEST_F(MaatPerfFlagScan, MultiThread) {
thread_params[i].logger = logger; thread_params[i].logger = logger;
if (i < PERF_THREAD_NUM) { if (i < PERF_THREAD_NUM) {
pthread_create(&threads[i], NULL, perf_flag_scan_thread, thread_params+i); pthread_create(&threads[i], NULL, perf_flag_scan_thread,
thread_params+i);
} else { } else {
thread_params[i].test_times = 0; thread_params[i].test_times = 0;
pthread_create(&threads[i], NULL, perf_flag_update_thread, thread_params+i); pthread_create(&threads[i], NULL, perf_flag_update_thread,
thread_params+i);
} }
} }
@@ -1280,26 +1347,31 @@ class MaatPerfFQDNPluginScan : public testing::Test
{ {
protected: protected:
static void SetUpTestCase() { static void SetUpTestCase() {
const char *accept_tags = "{\"tags\":[{\"tag\":\"location\",\"value\":\"北京/朝阳/华严北里/甲22号\"}," const char *accept_tags =
"{\"tag\":\"isp\",\"value\":\"移动\"},{\"tag\":\"location\",\"value\":\"Astana\"}]}"; "{\"tags\":[{\"tag\":\"location\",\"value\":\"北京/朝阳/华严北里/甲22号\"},"
"{\"tag\":\"isp\",\"value\":\"移动\"},{\"tag\":\"location\",\"value\":\"Astana\"}]}";
char redis_ip[64] = "127.0.0.1"; char redis_ip[64] = "127.0.0.1";
int redis_port = 6379; int redis_port = 6379;
int redis_db = 0; int redis_db = 0;
char json_iris_path[512] = {0}; char json_iris_path[512] = {0};
snprintf(json_iris_path, sizeof(json_iris_path), "./%s_iris_tmp", json_filename); snprintf(json_iris_path, sizeof(json_iris_path),
"./%s_iris_tmp", json_filename);
logger = log_handle_create("./maat_framework_perf_gtest.log", 0); logger = log_handle_create("./maat_framework_perf_gtest.log", 0);
int ret = write_config_to_redis(json_iris_path, redis_ip, redis_port, redis_db, logger); int ret = write_config_to_redis(json_iris_path, redis_ip, redis_port,
redis_db, logger);
if (ret < 0) { if (ret < 0) {
log_fatal(logger, MODULE_FRAMEWORK_PERF_GTEST, log_fatal(logger, MODULE_FRAMEWORK_PERF_GTEST,
"[%s:%d] write config to redis failed.", __FUNCTION__, __LINE__); "[%s:%d] write config to redis failed.",
__FUNCTION__, __LINE__);
} }
struct maat_options *opts = maat_options_new(); struct maat_options *opts = maat_options_new();
maat_options_set_stat_file(opts, "./stat.log"); maat_options_set_stat_file(opts, "./stat.log");
maat_options_set_perf_on(opts); maat_options_set_perf_on(opts);
maat_options_set_redis(opts, redis_ip, redis_port, redis_db); maat_options_set_redis(opts, redis_ip, redis_port, redis_db);
maat_options_set_logger(opts, "./maat_framework_perf_gtest.log", LOG_LEVEL_INFO); maat_options_set_logger(opts, "./maat_framework_perf_gtest.log",
LOG_LEVEL_INFO);
maat_options_set_accept_tags(opts, accept_tags); maat_options_set_accept_tags(opts, accept_tags);
maat_options_set_caller_thread_number(opts, 5); maat_options_set_caller_thread_number(opts, 5);
@@ -1329,8 +1401,9 @@ struct perf_fqdn_plugin_ud {
int catid; int catid;
}; };
void perf_fqdn_plugin_EX_new_cb(const char *table_name, int table_id, const char *key, void perf_fqdn_plugin_EX_new_cb(const char *table_name, int table_id,
const char *table_line, void **ad, long argl, void *argp) const char *key, const char *table_line,
void **ad, long argl, void *argp)
{ {
int *counter=(int *)argp; int *counter=(int *)argp;
size_t column_offset=0, column_len=0; size_t column_offset=0, column_len=0;
@@ -1357,7 +1430,8 @@ void perf_fqdn_plugin_EX_free_cb(int table_id, void **ad, long argl, void *argp)
*ad = NULL; *ad = NULL;
} }
void perf_fqdn_plugin_EX_dup_cb(int table_id, void **to, void **from, long argl, void *argp) void perf_fqdn_plugin_EX_dup_cb(int table_id, void **to, void **from,
long argl, void *argp)
{ {
struct perf_fqdn_plugin_ud *u = (struct perf_fqdn_plugin_ud *)(*from); struct perf_fqdn_plugin_ud *u = (struct perf_fqdn_plugin_ud *)(*from);
@@ -1436,7 +1510,8 @@ void *perf_fqdn_plugin_update_thread(void *arg)
line_rule.rule_id = (int)maat_cmd_incrby(maat_inst, "TEST_PLUG_SEQ", 1); line_rule.rule_id = (int)maat_cmd_incrby(maat_inst, "TEST_PLUG_SEQ", 1);
line_rule.table_name = param->table_name; line_rule.table_name = param->table_name;
random_fqdn_generate(fqdn_buff, sizeof(fqdn_buff)); random_fqdn_generate(fqdn_buff, sizeof(fqdn_buff));
snprintf(line_buff, 1024, "%lld\t1\t%s\tcatid=4\t1", line_rule.rule_id, fqdn_buff); snprintf(line_buff, 1024, "%lld\t1\t%s\tcatid=4\t1",
line_rule.rule_id, fqdn_buff);
line_rule.table_line = line_buff; line_rule.table_line = line_buff;
line_rule.expire_after = 0; line_rule.expire_after = 0;
maat_cmd_set_line(maat_inst, &line_rule); maat_cmd_set_line(maat_inst, &line_rule);
@@ -1478,10 +1553,12 @@ TEST_F(MaatPerfFQDNPluginScan, MultiThread) {
thread_params[i].logger = logger; thread_params[i].logger = logger;
if (i < PERF_THREAD_NUM) { if (i < PERF_THREAD_NUM) {
pthread_create(&threads[i], NULL, perf_fqdn_plugin_scan_thread, thread_params + i); pthread_create(&threads[i], NULL, perf_fqdn_plugin_scan_thread,
thread_params + i);
} else { } else {
thread_params[i].test_times = 0; thread_params[i].test_times = 0;
pthread_create(&threads[i], NULL, perf_fqdn_plugin_update_thread, thread_params + i); pthread_create(&threads[i], NULL, perf_fqdn_plugin_update_thread,
thread_params + i);
} }
} }
@@ -1508,26 +1585,31 @@ class MaatPerfBoolPluginScan : public testing::Test
{ {
protected: protected:
static void SetUpTestCase() { static void SetUpTestCase() {
const char *accept_tags = "{\"tags\":[{\"tag\":\"location\",\"value\":\"北京/朝阳/华严北里/甲22号\"}," const char *accept_tags =
"{\"tag\":\"isp\",\"value\":\"移动\"},{\"tag\":\"location\",\"value\":\"Astana\"}]}"; "{\"tags\":[{\"tag\":\"location\",\"value\":\"北京/朝阳/华严北里/甲22号\"},"
"{\"tag\":\"isp\",\"value\":\"移动\"},{\"tag\":\"location\",\"value\":\"Astana\"}]}";
char redis_ip[64] = "127.0.0.1"; char redis_ip[64] = "127.0.0.1";
int redis_port = 6379; int redis_port = 6379;
int redis_db = 0; int redis_db = 0;
char json_iris_path[512] = {0}; char json_iris_path[512] = {0};
snprintf(json_iris_path, sizeof(json_iris_path), "./%s_iris_tmp", json_filename); snprintf(json_iris_path, sizeof(json_iris_path),
"./%s_iris_tmp", json_filename);
logger = log_handle_create("./maat_framework_perf_gtest.log", 0); logger = log_handle_create("./maat_framework_perf_gtest.log", 0);
int ret = write_config_to_redis(json_iris_path, redis_ip, redis_port, redis_db, logger); int ret = write_config_to_redis(json_iris_path, redis_ip, redis_port,
redis_db, logger);
if (ret < 0) { if (ret < 0) {
log_fatal(logger, MODULE_FRAMEWORK_PERF_GTEST, log_fatal(logger, MODULE_FRAMEWORK_PERF_GTEST,
"[%s:%d] write config to redis failed.", __FUNCTION__, __LINE__); "[%s:%d] write config to redis failed.",
__FUNCTION__, __LINE__);
} }
struct maat_options *opts = maat_options_new(); struct maat_options *opts = maat_options_new();
maat_options_set_stat_file(opts, "./stat.log"); maat_options_set_stat_file(opts, "./stat.log");
maat_options_set_perf_on(opts); maat_options_set_perf_on(opts);
maat_options_set_redis(opts, redis_ip, redis_port, redis_db); maat_options_set_redis(opts, redis_ip, redis_port, redis_db);
maat_options_set_logger(opts, "./maat_framework_perf_gtest.log", LOG_LEVEL_INFO); maat_options_set_logger(opts, "./maat_framework_perf_gtest.log",
LOG_LEVEL_INFO);
maat_options_set_accept_tags(opts, accept_tags); maat_options_set_accept_tags(opts, accept_tags);
maat_options_set_caller_thread_number(opts, 5); maat_options_set_caller_thread_number(opts, 5);
@@ -1557,8 +1639,9 @@ struct bool_plugin_ud {
char *name; char *name;
}; };
void perf_bool_plugin_ex_new_cb(const char *table_name, int table_id, const char *key, void perf_bool_plugin_ex_new_cb(const char *table_name, int table_id,
const char *table_line, void **ad, long argl, void *argp) const char *key, const char *table_line,
void **ad, long argl, void *argp)
{ {
int *counter=(int *)argp; int *counter=(int *)argp;
size_t column_offset=0, column_len=0; size_t column_offset=0, column_len=0;
@@ -1587,7 +1670,8 @@ void perf_bool_plugin_ex_free_cb(int table_id, void **ad, long argl, void *argp)
*ad = NULL; *ad = NULL;
} }
void perf_bool_plugin_ex_dup_cb(int table_id, void **to, void **from, long argl, void *argp) void perf_bool_plugin_ex_dup_cb(int table_id, void **to, void **from,
long argl, void *argp)
{ {
struct bool_plugin_ud *u = (struct bool_plugin_ud *)(*from); struct bool_plugin_ud *u = (struct bool_plugin_ud *)(*from);
@@ -1686,10 +1770,12 @@ TEST_F(MaatPerfBoolPluginScan, MultiThread) {
thread_params[i].logger = logger; thread_params[i].logger = logger;
if (i < PERF_THREAD_NUM) { if (i < PERF_THREAD_NUM) {
pthread_create(&threads[i], NULL, perf_bool_plugin_scan_thread, thread_params + i); pthread_create(&threads[i], NULL, perf_bool_plugin_scan_thread,
thread_params + i);
} else { } else {
thread_params[i].test_times = 0; thread_params[i].test_times = 0;
pthread_create(&threads[i], NULL, perf_bool_plugin_update_thread, thread_params + i); pthread_create(&threads[i], NULL, perf_bool_plugin_update_thread,
thread_params + i);
} }
} }
@@ -1724,7 +1810,8 @@ protected:
maat_options_set_stat_file(opts, "./stat.log"); maat_options_set_stat_file(opts, "./stat.log");
maat_options_set_perf_on(opts); maat_options_set_perf_on(opts);
maat_options_set_iris(opts, rule_folder, rule_folder); maat_options_set_iris(opts, rule_folder, rule_folder);
maat_options_set_logger(opts, "./maat_framework_perf_gtest.log", LOG_LEVEL_INFO); maat_options_set_logger(opts, "./maat_framework_perf_gtest.log",
LOG_LEVEL_INFO);
maat_options_set_caller_thread_number(opts, 5); maat_options_set_caller_thread_number(opts, 5);
_shared_maat_inst = maat_new(opts, table_info); _shared_maat_inst = maat_new(opts, table_info);
@@ -1752,8 +1839,9 @@ struct perf_ip_plugin_ud {
long long rule_id; long long rule_id;
}; };
void perf_ip_plugin_EX_new_cb(const char *table_name, int table_id, const char *key, void perf_ip_plugin_EX_new_cb(const char *table_name, int table_id,
const char *table_line, void **ad, long argl, void *argp) const char *key, const char *table_line,
void **ad, long argl, void *argp)
{ {
int *counter=(int *)argp, ret=0; int *counter=(int *)argp, ret=0;
size_t column_offset=0, column_len=0; size_t column_offset=0, column_len=0;
@@ -1778,7 +1866,8 @@ void perf_ip_plugin_EX_free_cb(int table_id, void **ad, long argl, void *argp)
*ad = NULL; *ad = NULL;
} }
void perf_ip_plugin_EX_dup_cb(int table_id, void **to, void **from, long argl, void *argp) void perf_ip_plugin_EX_dup_cb(int table_id, void **to, void **from,
long argl, void *argp)
{ {
struct perf_ip_plugin_ud *u = (struct perf_ip_plugin_ud*)(*from); struct perf_ip_plugin_ud *u = (struct perf_ip_plugin_ud*)(*from);
@@ -1803,7 +1892,8 @@ static void *ip_plugin_get_thread(void *arg)
clock_gettime(CLOCK_MONOTONIC, &start); clock_gettime(CLOCK_MONOTONIC, &start);
struct perf_ip_plugin_ud *results[ARRAY_SIZE]; struct perf_ip_plugin_ud *results[ARRAY_SIZE];
for (i = 0; i < test_times; i++) { for (i = 0; i < test_times; i++) {
ret = maat_ip_plugin_table_get_ex_data(maat_inst, table_id, &ipv4, (void**)results, 4); ret = maat_ip_plugin_table_get_ex_data(maat_inst, table_id, &ipv4,
(void**)results, 4);
if (ret > 0) { if (ret > 0) {
hit_times++; hit_times++;
} }
@@ -1864,17 +1954,20 @@ protected:
snprintf(json_iris_path, sizeof(json_iris_path), "./tsgrule"); snprintf(json_iris_path, sizeof(json_iris_path), "./tsgrule");
logger = log_handle_create("./maat_framework_perf_gtest.log", 0); logger = log_handle_create("./maat_framework_perf_gtest.log", 0);
int ret = write_config_to_redis(json_iris_path, redis_ip, redis_port, redis_db, logger); int ret = write_config_to_redis(json_iris_path, redis_ip, redis_port,
redis_db, logger);
if (ret < 0) { if (ret < 0) {
log_fatal(logger, MODULE_FRAMEWORK_PERF_GTEST, log_fatal(logger, MODULE_FRAMEWORK_PERF_GTEST,
"[%s:%d] write config to redis failed.", __FUNCTION__, __LINE__); "[%s:%d] write config to redis failed.",
__FUNCTION__, __LINE__);
} }
struct maat_options *opts = maat_options_new(); struct maat_options *opts = maat_options_new();
maat_options_set_stat_file(opts, "./stat.log"); maat_options_set_stat_file(opts, "./stat.log");
maat_options_set_perf_on(opts); maat_options_set_perf_on(opts);
maat_options_set_redis(opts, redis_ip, redis_port, redis_db); maat_options_set_redis(opts, redis_ip, redis_port, redis_db);
maat_options_set_logger(opts, "./maat_framework_perf_gtest.log", LOG_LEVEL_INFO); maat_options_set_logger(opts, "./maat_framework_perf_gtest.log",
LOG_LEVEL_INFO);
_shared_maat_inst = maat_new(opts, tsg_table_info); _shared_maat_inst = maat_new(opts, tsg_table_info);
maat_options_free(opts); maat_options_free(opts);
@@ -1927,26 +2020,31 @@ class MaatPerfIPPortPluginScan : public testing::Test
{ {
protected: protected:
static void SetUpTestCase() { static void SetUpTestCase() {
const char *accept_tags = "{\"tags\":[{\"tag\":\"location\",\"value\":\"北京/朝阳/华严北里/甲22号\"}," const char *accept_tags =
"{\"tag\":\"isp\",\"value\":\"移动\"},{\"tag\":\"location\",\"value\":\"Astana\"}]}"; "{\"tags\":[{\"tag\":\"location\",\"value\":\"北京/朝阳/华严北里/甲22号\"},"
"{\"tag\":\"isp\",\"value\":\"移动\"},{\"tag\":\"location\",\"value\":\"Astana\"}]}";
char redis_ip[64] = "127.0.0.1"; char redis_ip[64] = "127.0.0.1";
int redis_port = 6379; int redis_port = 6379;
int redis_db = 0; int redis_db = 0;
char json_iris_path[512] = {0}; char json_iris_path[512] = {0};
snprintf(json_iris_path, sizeof(json_iris_path), "./%s_iris_tmp", json_filename); snprintf(json_iris_path, sizeof(json_iris_path),
"./%s_iris_tmp", json_filename);
logger = log_handle_create("./maat_framework_perf_gtest.log", 0); logger = log_handle_create("./maat_framework_perf_gtest.log", 0);
int ret = write_config_to_redis(json_iris_path, redis_ip, redis_port, redis_db, logger); int ret = write_config_to_redis(json_iris_path, redis_ip, redis_port,
redis_db, logger);
if (ret < 0) { if (ret < 0) {
log_fatal(logger, MODULE_FRAMEWORK_PERF_GTEST, log_fatal(logger, MODULE_FRAMEWORK_PERF_GTEST,
"[%s:%d] write config to redis failed.", __FUNCTION__, __LINE__); "[%s:%d] write config to redis failed.",
__FUNCTION__, __LINE__);
} }
struct maat_options *opts = maat_options_new(); struct maat_options *opts = maat_options_new();
maat_options_set_stat_file(opts, "./stat.log"); maat_options_set_stat_file(opts, "./stat.log");
maat_options_set_perf_on(opts); maat_options_set_perf_on(opts);
maat_options_set_redis(opts, redis_ip, redis_port, redis_db); maat_options_set_redis(opts, redis_ip, redis_port, redis_db);
maat_options_set_logger(opts, "./maat_framework_perf_gtest.log", LOG_LEVEL_INFO); maat_options_set_logger(opts, "./maat_framework_perf_gtest.log",
LOG_LEVEL_INFO);
maat_options_set_accept_tags(opts, accept_tags); maat_options_set_accept_tags(opts, accept_tags);
maat_options_set_caller_thread_number(opts, 5); maat_options_set_caller_thread_number(opts, 5);
@@ -2057,10 +2155,12 @@ TEST_F(MaatPerfIPPortPluginScan, MultiThread) {
thread_params[i].logger = logger; thread_params[i].logger = logger;
if (i < PERF_THREAD_NUM) { if (i < PERF_THREAD_NUM) {
pthread_create(&threads[i], NULL, perf_ipport_plugin_scan_thread, thread_params+i); pthread_create(&threads[i], NULL, perf_ipport_plugin_scan_thread,
thread_params+i);
} else { } else {
thread_params[i].test_times = 0; thread_params[i].test_times = 0;
pthread_create(&threads[i], NULL, perf_ipport_plugin_update_thread, thread_params+i); pthread_create(&threads[i], NULL, perf_ipport_plugin_update_thread,
thread_params+i);
} }
} }

80
test/maat_input_mode_gtest.cpp
View File

@@ -17,17 +17,20 @@ struct log_handle *g_logger = NULL;
TEST(json_mode, maat_scan_string) { TEST(json_mode, maat_scan_string) {
char tmp_iris_path[PATH_MAX] = {0}; char tmp_iris_path[PATH_MAX] = {0};
char json_iris_path[PATH_MAX] = {0}; char json_iris_path[PATH_MAX] = {0};
snprintf(json_iris_path, sizeof(json_iris_path), "./%s_iris_tmp", json_filename); snprintf(json_iris_path, sizeof(json_iris_path),
"./%s_iris_tmp", json_filename);
if (access(json_iris_path, F_OK) < 0) { if (access(json_iris_path, F_OK) < 0) {
char *json_buff = NULL; char *json_buff = NULL;
size_t json_buff_sz = 0; size_t json_buff_sz = 0;
int ret = load_file_to_memory(json_filename, (unsigned char**)&json_buff, &json_buff_sz); int ret = load_file_to_memory(json_filename,
(unsigned char**)&json_buff,
&json_buff_sz);
EXPECT_NE(ret, -1); EXPECT_NE(ret, -1);
ret = json2iris(json_buff, json_filename, NULL, tmp_iris_path, sizeof(tmp_iris_path), ret = json2iris(json_buff, json_filename, NULL, tmp_iris_path,
NULL, NULL, g_logger); sizeof(tmp_iris_path), NULL, NULL, g_logger);
FREE(json_buff); FREE(json_buff);
EXPECT_NE(ret, -1); EXPECT_NE(ret, -1);
} }
@@ -43,7 +46,8 @@ TEST(json_mode, maat_scan_string) {
const char *table_name = "KEYWORDS_TABLE"; const char *table_name = "KEYWORDS_TABLE";
int table_id = maat_get_table_id(maat_inst, table_name); int table_id = maat_get_table_id(maat_inst, table_name);
char scan_data[128] = "string1, string2, string3, string4, string5, string6, string7, string8"; char scan_data[128] =
"string1, string2, string3, string4, string5, string6, string7, string8";
long long results[5] = {0}; long long results[5] = {0};
size_t n_hit_result = 0; size_t n_hit_result = 0;
int thread_id = 0; int thread_id = 0;
@@ -63,25 +67,30 @@ TEST(json_mode, maat_scan_string) {
TEST(iris_mode, maat_scan_string) { TEST(iris_mode, maat_scan_string) {
char tmp_iris_path[512] = {0}; char tmp_iris_path[512] = {0};
char json_iris_path[512] = {0}; char json_iris_path[512] = {0};
snprintf(json_iris_path, sizeof(json_iris_path), "./%s_iris_tmp", json_filename); snprintf(json_iris_path, sizeof(json_iris_path),
"./%s_iris_tmp", json_filename);
if (access(json_iris_path, F_OK) < 0) { if (access(json_iris_path, F_OK) < 0) {
char *json_buff = NULL; char *json_buff = NULL;
size_t json_buff_sz = 0; size_t json_buff_sz = 0;
int ret = load_file_to_memory(json_filename, (unsigned char**)&json_buff, &json_buff_sz); int ret = load_file_to_memory(json_filename,
(unsigned char**)&json_buff,
&json_buff_sz);
EXPECT_NE(ret, -1); EXPECT_NE(ret, -1);
ret = json2iris(json_buff, json_filename, NULL, tmp_iris_path, sizeof(tmp_iris_path), ret = json2iris(json_buff, json_filename, NULL, tmp_iris_path,
NULL, NULL, g_logger); sizeof(tmp_iris_path), NULL, NULL, g_logger);
FREE(json_buff); FREE(json_buff);
EXPECT_NE(ret, -1); EXPECT_NE(ret, -1);
} }
char tmp_iris_full_idx_path[PATH_MAX] = {0}; char tmp_iris_full_idx_path[PATH_MAX] = {0};
char tmp_iris_inc_idx_path[PATH_MAX] = {0}; char tmp_iris_inc_idx_path[PATH_MAX] = {0};
snprintf(tmp_iris_full_idx_path, sizeof(tmp_iris_full_idx_path), "%s/index", json_iris_path); snprintf(tmp_iris_full_idx_path, sizeof(tmp_iris_full_idx_path),
snprintf(tmp_iris_inc_idx_path, sizeof(tmp_iris_inc_idx_path), "%s/index", json_iris_path); "%s/index", json_iris_path);
snprintf(tmp_iris_inc_idx_path, sizeof(tmp_iris_inc_idx_path),
"%s/index", json_iris_path);
struct maat_options *opts = maat_options_new(); struct maat_options *opts = maat_options_new();
maat_options_set_iris(opts, tmp_iris_full_idx_path, tmp_iris_inc_idx_path); maat_options_set_iris(opts, tmp_iris_full_idx_path, tmp_iris_inc_idx_path);
@@ -92,7 +101,8 @@ TEST(iris_mode, maat_scan_string) {
const char *table_name = "KEYWORDS_TABLE"; const char *table_name = "KEYWORDS_TABLE";
int table_id = maat_get_table_id(maat_inst, table_name); int table_id = maat_get_table_id(maat_inst, table_name);
char scan_data[128] = "string1, string2, string3, string4, string5, string6, string7, string8"; char scan_data[128] =
"string1, string2, string3, string4, string5, string6, string7, string8";
long long results[5] = {0}; long long results[5] = {0};
size_t n_hit_result = 0; size_t n_hit_result = 0;
int thread_id = 0; int thread_id = 0;
@@ -129,7 +139,8 @@ int make_serial_rule(const char *table_name, const char *line, void *u_para)
} }
const char *redis_rule_key = "TEST_RULE_KEY"; const char *redis_rule_key = "TEST_RULE_KEY";
redisReply *reply = maat_wrap_redis_command(ctx, "INCRBY %s %d", redis_rule_key, 1); redisReply *reply =
maat_wrap_redis_command(ctx, NULL, "INCRBY %s %d", redis_rule_key, 1);
if (reply->type == REDIS_REPLY_NIL) { if (reply->type == REDIS_REPLY_NIL) {
printf("incrby redis_rule_key:%s failed.", redis_rule_key); printf("incrby redis_rule_key:%s failed.", redis_rule_key);
return -1; return -1;
@@ -139,8 +150,8 @@ int make_serial_rule(const char *table_name, const char *line, void *u_para)
reply = NULL; reply = NULL;
} }
maat_set_serial_rule(s_rule + line_idx, MAAT_OP_ADD, s_rule->rule_id, table_name, maat_set_serial_rule(s_rule + line_idx, MAAT_OP_ADD, s_rule->rule_id,
buff, absolute_expire_time); table_name, buff, absolute_expire_time);
(s_rule + line_idx)->ref_ctx = ctx; (s_rule + line_idx)->ref_ctx = ctx;
line_idx++; line_idx++;
@@ -155,12 +166,14 @@ TEST(redis_mode, maat_scan_string) {
int redis_port = 6379; int redis_port = 6379;
int redis_db = 0; int redis_db = 0;
snprintf(json_iris_path, sizeof(json_iris_path), "./%s_iris_tmp", json_filename); snprintf(json_iris_path, sizeof(json_iris_path),
"./%s_iris_tmp", json_filename);
redisContext *c = maat_connect_redis(redis_ip, redis_port, redis_db, g_logger); redisContext *c =
maat_connect_redis(redis_ip, redis_port, redis_db, g_logger);
EXPECT_TRUE(c != NULL); EXPECT_TRUE(c != NULL);
redisReply *reply = maat_wrap_redis_command(c, "flushdb"); redisReply *reply = maat_wrap_redis_command(c, NULL, "flushdb");
EXPECT_TRUE(reply != NULL); EXPECT_TRUE(reply != NULL);
freeReplyObject(reply); freeReplyObject(reply);
reply = NULL; reply = NULL;
@@ -170,20 +183,23 @@ TEST(redis_mode, maat_scan_string) {
char *json_buff = NULL; char *json_buff = NULL;
size_t json_buff_sz = 0; size_t json_buff_sz = 0;
int ret = load_file_to_memory(json_filename, (unsigned char **)&json_buff, &json_buff_sz); int ret = load_file_to_memory(json_filename,
(unsigned char **)&json_buff,
&json_buff_sz);
EXPECT_NE(ret, -1); EXPECT_NE(ret, -1);
ret = json2iris(json_buff, json_filename, c, tmp_iris_path, sizeof(tmp_iris_path), ret = json2iris(json_buff, json_filename, c, tmp_iris_path,
NULL, NULL, g_logger); sizeof(tmp_iris_path), NULL, NULL, g_logger);
FREE(json_buff); FREE(json_buff);
EXPECT_NE(ret, -1); EXPECT_NE(ret, -1);
} }
size_t total_line_cnt = 0; size_t total_line_cnt = 0;
char tmp_iris_full_idx_path[PATH_MAX] = {0}; char tmp_iris_full_idx_path[PATH_MAX] = {0};
snprintf(tmp_iris_full_idx_path, sizeof(tmp_iris_full_idx_path), "%s/index", json_iris_path); snprintf(tmp_iris_full_idx_path, sizeof(tmp_iris_full_idx_path),
config_monitor_traverse(0, tmp_iris_full_idx_path, NULL, count_line_num_cb, NULL, "%s/index", json_iris_path);
&total_line_cnt, NULL, g_logger); config_monitor_traverse(0, tmp_iris_full_idx_path, NULL, count_line_num_cb,
NULL, &total_line_cnt, NULL, g_logger);
struct serial_rule *s_rule = ALLOC(struct serial_rule, total_line_cnt); struct serial_rule *s_rule = ALLOC(struct serial_rule, total_line_cnt);
s_rule->ref_ctx = c; s_rule->ref_ctx = c;
@@ -191,12 +207,13 @@ TEST(redis_mode, maat_scan_string) {
EXPECT_NE(server_time, -1); EXPECT_NE(server_time, -1);
absolute_expire_time = server_time + 300; absolute_expire_time = server_time + 300;
config_monitor_traverse(0, tmp_iris_full_idx_path, NULL, make_serial_rule, NULL, config_monitor_traverse(0, tmp_iris_full_idx_path, NULL, make_serial_rule,
s_rule, NULL, g_logger); NULL, s_rule, NULL, g_logger);
s_rule->ref_ctx = NULL; s_rule->ref_ctx = NULL;
int success_cnt = 0; int success_cnt = 0;
do { do {
success_cnt = maat_cmd_write_rule(c, s_rule, total_line_cnt, server_time, g_logger); success_cnt = maat_cmd_write_rule(c, s_rule, total_line_cnt,
server_time, g_logger);
} while (success_cnt < 0); } while (success_cnt < 0);
EXPECT_EQ(success_cnt, (int)total_line_cnt); EXPECT_EQ(success_cnt, (int)total_line_cnt);
@@ -209,12 +226,14 @@ TEST(redis_mode, maat_scan_string) {
struct maat_options *opts = maat_options_new(); struct maat_options *opts = maat_options_new();
maat_options_set_redis(opts, redis_ip, redis_port, redis_db); maat_options_set_redis(opts, redis_ip, redis_port, redis_db);
maat_options_set_logger(opts, "./maat_input_mode_gtest.log", LOG_LEVEL_INFO); maat_options_set_logger(opts, "./maat_input_mode_gtest.log",
LOG_LEVEL_INFO);
struct maat *maat_inst = maat_new(opts, table_info_path); struct maat *maat_inst = maat_new(opts, table_info_path);
const char *table_name = "KEYWORDS_TABLE"; const char *table_name = "KEYWORDS_TABLE";
int table_id = maat_get_table_id(maat_inst, table_name); int table_id = maat_get_table_id(maat_inst, table_name);
char scan_data[128] = "string1, string2, string3, string4, string5, string6, string7, string8"; char scan_data[128] =
"string1, string2, string3, string4, string5, string6, string7, string8";
long long results[5] = {0}; long long results[5] = {0};
size_t n_hit_result = 0; size_t n_hit_result = 0;
int thread_id = 0; int thread_id = 0;
@@ -238,7 +257,8 @@ int main(int argc, char ** argv)
g_logger = log_handle_create("./maat_input_mode_gtest.log", 0); g_logger = log_handle_create("./maat_input_mode_gtest.log", 0);
char json_iris_path[NAME_MAX] = {0}; char json_iris_path[NAME_MAX] = {0};
snprintf(json_iris_path, sizeof(json_iris_path), "./%s_iris_tmp", json_filename); snprintf(json_iris_path, sizeof(json_iris_path),
"./%s_iris_tmp", json_filename);
if ((access(json_iris_path, F_OK)) == 0) { if ((access(json_iris_path, F_OK)) == 0) {
system_cmd_rmdir(json_iris_path); system_cmd_rmdir(json_iris_path);
} }

42
tools/maat_redis_tool.cpp
View File

@@ -220,7 +220,8 @@ int make_serial_rule(const char *table_name, const char *line, void *u_para)
} }
const char *redis_rule_key = "TEST_RULE_KEY"; const char *redis_rule_key = "TEST_RULE_KEY";
redisReply *reply = maat_wrap_redis_command(ctx, "INCRBY %s %d", redis_rule_key, 1); redisReply *reply =
maat_wrap_redis_command(ctx, NULL, "INCRBY %s %d", redis_rule_key, 1);
if (reply->type == REDIS_REPLY_NIL) { if (reply->type == REDIS_REPLY_NIL) {
printf("incrby redis_rule_key:%s failed.", redis_rule_key); printf("incrby redis_rule_key:%s failed.", redis_rule_key);
return -1; return -1;
@@ -230,8 +231,8 @@ int make_serial_rule(const char *table_name, const char *line, void *u_para)
reply = NULL; reply = NULL;
} }
maat_set_serial_rule(s_rule + line_idx, MAAT_OP_ADD, s_rule->rule_id, table_name, maat_set_serial_rule(s_rule + line_idx, MAAT_OP_ADD, s_rule->rule_id,
buff, absolute_expire_time); table_name, buff, absolute_expire_time);
(s_rule + line_idx)->ref_ctx = ctx; (s_rule + line_idx)->ref_ctx = ctx;
line_idx++; line_idx++;
FREE(buff); FREE(buff);
@@ -239,27 +240,33 @@ int make_serial_rule(const char *table_name, const char *line, void *u_para)
return 0; return 0;
} }
int write_config_to_redis(redisContext *c, char *json_filename, struct log_handle *logger) int write_config_to_redis(redisContext *c, char *json_filename,
struct log_handle *logger)
{ {
char tmp_iris_path[128] = {0}; char tmp_iris_path[128] = {0};
char *json_buff = NULL; char *json_buff = NULL;
size_t json_buff_sz = 0; size_t json_buff_sz = 0;
snprintf(tmp_iris_path, sizeof(tmp_iris_path), "%s_iris_tmp", json_filename); snprintf(tmp_iris_path, sizeof(tmp_iris_path),
"%s_iris_tmp", json_filename);
int ret = load_file_to_memory(json_filename, (unsigned char **)&json_buff, &json_buff_sz); int ret = load_file_to_memory(json_filename,
(unsigned char **)&json_buff,
&json_buff_sz);
if (ret < 0) { if (ret < 0) {
return -1; return -1;
} }
ret = json2iris(json_buff, json_filename, c, tmp_iris_path, sizeof(tmp_iris_path), NULL, NULL, logger); ret = json2iris(json_buff, json_filename, c, tmp_iris_path,
sizeof(tmp_iris_path), NULL, NULL, logger);
FREE(json_buff); FREE(json_buff);
if (ret < 0) { if (ret < 0) {
return -1; return -1;
} }
size_t total_line_cnt = 0; size_t total_line_cnt = 0;
config_monitor_traverse(0, tmp_iris_path, NULL, count_line_num_cb, NULL, &total_line_cnt, NULL, logger); config_monitor_traverse(0, tmp_iris_path, NULL, count_line_num_cb, NULL,
&total_line_cnt, NULL, logger);
struct serial_rule *s_rule = ALLOC(struct serial_rule, total_line_cnt); struct serial_rule *s_rule = ALLOC(struct serial_rule, total_line_cnt);
s_rule->ref_ctx = c; s_rule->ref_ctx = c;
@@ -269,13 +276,15 @@ int write_config_to_redis(redisContext *c, char *json_filename, struct log_handl
} }
s_rule->timeout = server_time + 300; s_rule->timeout = server_time + 300;
config_monitor_traverse(0, tmp_iris_path, NULL, make_serial_rule, NULL, s_rule, NULL, logger); config_monitor_traverse(0, tmp_iris_path, NULL, make_serial_rule, NULL,
s_rule, NULL, logger);
s_rule->ref_ctx = NULL; s_rule->ref_ctx = NULL;
line_idx = 0; line_idx = 0;
int success_cnt = 0; int success_cnt = 0;
do { do {
success_cnt = maat_cmd_write_rule(c, s_rule, total_line_cnt, server_time, logger); success_cnt = maat_cmd_write_rule(c, s_rule, total_line_cnt,
server_time, logger);
} while (success_cnt < 0); } while (success_cnt < 0);
assert(success_cnt == (int)total_line_cnt); assert(success_cnt == (int)total_line_cnt);
@@ -291,10 +300,12 @@ int write_config_to_redis(redisContext *c, char *json_filename, struct log_handl
int rollback_redis_version(redisContext *c, struct log_handle *logger) int rollback_redis_version(redisContext *c, struct log_handle *logger)
{ {
redisReply *reply = maat_wrap_redis_command(c, "SET MAAT_VERSION 0"); redisReply *reply =
maat_wrap_redis_command(c, logger, "SET MAAT_VERSION 0");
if (NULL == reply) { if (NULL == reply) {
log_fatal(logger, MODULE_REDIS_TOOL, log_fatal(logger, MODULE_REDIS_TOOL,
"[%s:%d] set MAAT_VERSION failed, Redis Communication error: %s", "[%s:%d] set MAAT_VERSION failed, "
"Redis Communication error: %s",
__FUNCTION__, __LINE__, c->errstr); __FUNCTION__, __LINE__, c->errstr);
return -1; return -1;
} }
@@ -307,9 +318,10 @@ int rollback_redis_version(redisContext *c, struct log_handle *logger)
int clear_config_in_redis(redisContext *c, struct log_handle *logger) int clear_config_in_redis(redisContext *c, struct log_handle *logger)
{ {
long long redis_version = 0; long long redis_version = 0;
redisReply *reply = maat_wrap_redis_command(c, "GET MAAT_VERSION"); redisReply *reply = maat_wrap_redis_command(c, logger, "GET MAAT_VERSION");
if (reply != NULL) { if (reply != NULL) {
if (reply->type == REDIS_REPLY_NIL || reply->type == REDIS_REPLY_ERROR) { if (reply->type == REDIS_REPLY_NIL ||
reply->type == REDIS_REPLY_ERROR) {
log_fatal(logger, MODULE_REDIS_TOOL, log_fatal(logger, MODULE_REDIS_TOOL,
"[%s:%d] GET MAAT_VERSION failed, maybe Redis is busy", "[%s:%d] GET MAAT_VERSION failed, maybe Redis is busy",
__FUNCTION__, __LINE__); __FUNCTION__, __LINE__);
@@ -339,7 +351,7 @@ int clear_config_in_redis(redisContext *c, struct log_handle *logger)
freeReplyObject(reply); freeReplyObject(reply);
reply = NULL; reply = NULL;
reply = maat_wrap_redis_command(c, "MULTI"); reply = maat_wrap_redis_command(c, logger, "MULTI");
freeReplyObject(reply); freeReplyObject(reply);
reply = NULL; reply = NULL;