gfwleak/tango-maat
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

third draft

Browse Source
This commit is contained in:
liuwentan
2023-07-06 18:58:15 +08:00
parent 2d6ffdd166
commit 9d373ad454
41 changed files with 81287 additions and 455 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
src/CMakeLists.txt
View File

@@ -57,4 +57,5 @@ target_link_libraries(maat_frame_shared adapter-static)
# install
set(CMAKE_INSTALL_PREFIX /opt/MESA/)
install(FILES ${PROJECT_SOURCE_DIR}/include/maat.h DESTINATION ${CMAKE_INSTALL_PREFIX}/include/MESA/ COMPONENT HEADER)
install(FILES ${PROJECT_SOURCE_DIR}/include/maat_command.h DESTINATION ${CMAKE_INSTALL_PREFIX}/include/MESA/ COMPONENT HEADER)
install(TARGETS maat_frame_shared LIBRARY DESTINATION ${CMAKE_INSTALL_PREFIX}/lib/ COMPONENT LIBRARIES)

130
src/inc_internal/maat_command.h
View File

@@ -1,130 +0,0 @@
/*
**********************************************************************************************
* File: maat_command.h
* Description:
* Authors: Liu WenTan <liuwentan@geedgenetworks.com>
* Date: 2022-10-31
* Copyright: (c) Since 2022 Geedge Networks, Ltd. All rights reserved.
***********************************************************************************************
*/
#ifndef _MAAT_COMMAND_H_
#define _MAAT_COMMAND_H_
#ifdef __cplusplus
extern "C"
{
#endif
#include <limits.h>
#include <sys/queue.h>
#include "maat.h"
#include "maat_table.h"
#include "log/log.h"
#include "uthash/uthash.h"
#include "hiredis/hiredis.h"
#include "maat_limits.h"
enum maat_operation {
MAAT_OP_DEL = 0,
MAAT_OP_ADD,
MAAT_OP_RENEW_TIMEOUT //Rule expire time is changed to now+cmd->expire_after
};
struct maat_cmd_line {
const char *table_name;
const char *table_line;
long long rule_id; // for MAAT_OP_DEL, only rule_id and table_name are necessary.
int expire_after; //expired after $timeout$ seconds, set to 0 for never timeout.
};
struct foreign_key {
int column;
char *key;
size_t key_len;
char *filename;
};
//rm= Redis Maat
struct serial_rule {
enum maat_operation op;//0: delete, 1: add.
long long rule_id;
long long timeout; // absolute unix time.
char table_name[MAX_NAME_STR_LEN];
char *table_line;
int n_foreign;
struct foreign_key *f_keys;
redisContext *ref_ctx;
TAILQ_ENTRY(serial_rule) entries;
UT_hash_handle hh;
};
/**
* @brief write one line to redis
*
* @retval
* success: number of successfully updated rule.
* failed: -1
*/
int maat_cmd_set_line(struct maat *maat_instance, const struct maat_cmd_line *line_rule);
int maat_cmd_set_file(struct maat *maat_instance, const char *key, const char *value,
size_t size, enum maat_operation op);
long long maat_cmd_incrby(struct maat *maat_instance, const char *key, int increment);
long long maat_cmd_get_config_version(struct maat *maat_instance);
/* True(1), False(0) */
int maat_cmd_config_is_updating(struct maat *maat_instance);
char *maat_cmd_str_escape(char *dst, int size, const char *src);
int maat_cmd_flushDB(struct maat *maat_instance);
/* maat command API for internal */
redisContext *maat_cmd_connect_redis(const char *redis_ip, int redis_port,
int redis_db, struct log_handle *logger);
redisReply *maat_cmd_wrap_redis_command(redisContext *c, const char *format, ...);
int maat_cmd_wrap_redis_get_reply(redisContext *c, redisReply **reply);
long long maat_cmd_redis_server_time_s(redisContext *c);
long long maat_cmd_read_redis_integer(const redisReply *reply);
int maat_cmd_get_valid_flag_offset(const char *line, int column_seq);
int maat_cmd_write_rule(redisContext *c, struct serial_rule *s_rule,
size_t serial_rule_num, long long server_time,
struct log_handle *logger);
void maat_cmd_clear_rule_cache(struct serial_rule *s_rule);
int maat_cmd_get_redis_value(redisContext *c, struct serial_rule *rule_list,
int rule_num, int print_process, struct log_handle *logger);
int maat_cmd_get_foreign_keys_by_prefix(redisContext *ctx, struct serial_rule *rule_list,
int rule_num, const char* dir, struct log_handle *logger);
void maat_cmd_get_foreign_conts(redisContext *c, struct serial_rule *rule_list,
int rule_num, int print_fn, struct log_handle *logger);
void maat_cmd_rewrite_table_line_with_foreign(struct serial_rule *s_rule);
void maat_cmd_set_serial_rule(struct serial_rule *rule, enum maat_operation op,
long long rule_id, const char *table_name,
const char *line, long long timeout);
int maat_cmd_get_rm_key_list(redisContext *c, long long instance_version,
long long desired_version, long long *new_version,
struct table_manager *tbl_mgr, struct serial_rule **list,
int *update_type, int cumulative_off, struct log_handle *logger);
#ifdef __cplusplus
}
#endif
#endif

65
src/inc_internal/maat_redis_monitor.h
View File

@@ -17,6 +17,71 @@ extern "C"
#endif
#include <stdint.h>
#include <sys/queue.h>
#include "hiredis/hiredis.h"
#include "maat_command.h"
#include "uthash/uthash.h"
struct foreign_key {
int column;
char *key;
size_t key_len;
char *filename;
};
//rm= Redis Maat
struct serial_rule {
enum maat_operation op;//0: delete, 1: add.
long long rule_id;
long long timeout; // absolute unix time.
char table_name[MAX_NAME_STR_LEN];
char *table_line;
int n_foreign;
struct foreign_key *f_keys;
redisContext *ref_ctx;
TAILQ_ENTRY(serial_rule) entries;
UT_hash_handle hh;
};
void maat_clear_rule_cache(struct serial_rule *s_rule);
void maat_set_serial_rule(struct serial_rule *rule, enum maat_operation op,
long long rule_id, const char *table_name,
const char *line, long long timeout);
redisContext *maat_connect_redis(const char *redis_ip, int redis_port,
int redis_db, struct log_handle *logger);
redisReply *maat_wrap_redis_command(redisContext *c, const char *format, ...);
int maat_wrap_redis_get_reply(redisContext *c, redisReply **reply);
long long maat_redis_server_time_s(redisContext *c);
long long maat_read_redis_integer(const redisReply *reply);
int maat_get_valid_flag_offset(const char *line, int column_seq);
int maat_cmd_write_rule(redisContext *c, struct serial_rule *s_rule,
size_t serial_rule_num, long long server_time,
struct log_handle *logger);
int maat_get_redis_value(redisContext *c, struct serial_rule *rule_list,
int rule_num, int print_process, struct log_handle *logger);
int maat_get_foreign_keys_by_prefix(redisContext *ctx, struct serial_rule *rule_list,
int rule_num, const char* dir, struct log_handle *logger);
void maat_get_foreign_conts(redisContext *c, struct serial_rule *rule_list,
int rule_num, int print_fn, struct log_handle *logger);
void maat_rewrite_table_line_with_foreign(struct serial_rule *s_rule);
int maat_get_rm_key_list(redisContext *c, long long instance_version,
long long desired_version, long long *new_version,
struct table_manager *tbl_mgr, struct serial_rule **list,
int *update_type, int cumulative_off, struct log_handle *logger);
void redis_monitor_traverse(long long version, struct source_redis_ctx *mr_ctx,
void (*start_fn)(long long, int, void *),

1
src/inc_internal/maat_utils.h
View File

@@ -93,6 +93,7 @@ int load_file_to_memory(const char *file_name, unsigned char **pp_out, size_t *o
char *strtok_r_esc(char *s, const char delim, char **save_ptr);
char *str_escape(char *dst, int size, const char *src);
char *str_unescape(char *s);
char *md5_file(const char *filename, char *md5string);

7
src/json2iris.c
View File

@@ -21,6 +21,7 @@
#include "maat_utils.h"
#include "maat_rule.h"
#include "uthash/uthash.h"
#include "maat_redis_monitor.h"
#define MODULE_JSON2IRIS module_name_str("maat.json2iris")
@@ -300,7 +301,7 @@ static int get_group_seq(struct iris_description *iris_cfg)
if (NULL == iris_cfg->redis_write_ctx) {
sequence = iris_cfg->group_cnt;
} else {
data_reply = maat_cmd_wrap_redis_command(iris_cfg->redis_write_ctx,
data_reply = maat_wrap_redis_command(iris_cfg->redis_write_ctx,
"INCRBY %s 1", mr_group_id_var);
sequence = (int)data_reply->integer - 1;
freeReplyObject(data_reply);
@@ -337,8 +338,8 @@ static int get_region_seq(struct iris_description *iris_cfg)
if (NULL == iris_cfg->redis_write_ctx) {
sequence = iris_cfg->region_cnt;
} else {
redisReply *data_reply = maat_cmd_wrap_redis_command(iris_cfg->redis_write_ctx,
"INCRBY %s 1", mr_region_id_var);
redisReply *data_reply = maat_wrap_redis_command(iris_cfg->redis_write_ctx,
"INCRBY %s 1", mr_region_id_var);
sequence = (int)data_reply->integer - 1;
freeReplyObject(data_reply);
}

140
src/maat_command.c
View File

@@ -18,6 +18,7 @@
#include "maat_rule.h"
#include "hiredis/hiredis.h"
#include "maat_config_monitor.h"
#include "maat_redis_monitor.h"
#define MODULE_MAAT_COMMAND module_name_str("maat.command")
@@ -34,7 +35,7 @@ extern const char *mr_label_sset;
extern const int MAAT_REDIS_SYNC_TIME;
redisReply *maat_cmd_wrap_redis_command(redisContext *c, const char *format, ...)
redisReply *maat_wrap_redis_command(redisContext *c, const char *format, ...)
{
va_list ap;
void *reply = NULL;
@@ -54,7 +55,7 @@ redisReply *maat_cmd_wrap_redis_command(redisContext *c, const char *format, ...
return (redisReply *)reply;
}
redisContext *maat_cmd_connect_redis(const char *redis_ip, int redis_port,
redisContext *maat_connect_redis(const char *redis_ip, int redis_port,
int redis_db, struct log_handle *logger)
{
struct timeval connect_timeout;
@@ -80,14 +81,14 @@ redisContext *maat_cmd_connect_redis(const char *redis_ip, int redis_port,
}
redisEnableKeepAlive(c);
redisReply *reply = maat_cmd_wrap_redis_command(c, "select %d", redis_db);
redisReply *reply = maat_wrap_redis_command(c, "select %d", redis_db);
freeReplyObject(reply);
reply = NULL;
return c;
}
long long maat_cmd_read_redis_integer(const redisReply *reply)
long long maat_read_redis_integer(const redisReply *reply)
{
switch (reply->type) {
case REDIS_REPLY_INTEGER:
@@ -112,26 +113,26 @@ static int redis_flushDB(redisContext *ctx, int db_index,
{
long long maat_redis_version = 0;
redisReply *data_reply = maat_cmd_wrap_redis_command(ctx, "WATCH MAAT_VERSION");
redisReply *data_reply = maat_wrap_redis_command(ctx, "WATCH MAAT_VERSION");
freeReplyObject(data_reply);
data_reply = NULL;
data_reply = maat_cmd_wrap_redis_command(ctx, "GET MAAT_VERSION");
data_reply = maat_wrap_redis_command(ctx, "GET MAAT_VERSION");
if (data_reply->type == REDIS_REPLY_NIL) {
maat_redis_version = 0;
} else {
maat_redis_version = maat_cmd_read_redis_integer(data_reply);
maat_redis_version = maat_read_redis_integer(data_reply);
maat_redis_version++;
freeReplyObject(data_reply);
data_reply = NULL;
}
data_reply = maat_cmd_wrap_redis_command(ctx, "DBSIZE");
long long dbsize = maat_cmd_read_redis_integer(data_reply);
data_reply = maat_wrap_redis_command(ctx, "DBSIZE");
long long dbsize = maat_read_redis_integer(data_reply);
freeReplyObject(data_reply);
data_reply = NULL;
data_reply = maat_cmd_wrap_redis_command(ctx, "MULTI");
data_reply = maat_wrap_redis_command(ctx, "MULTI");
freeReplyObject(data_reply);
data_reply = NULL;
@@ -152,7 +153,7 @@ static int redis_flushDB(redisContext *ctx, int db_index,
int ret = 0;
int redis_transaction_success = 1;
for (int i = 0; i < append_cmd_cnt; i++) {
ret = maat_cmd_wrap_redis_get_reply(ctx, &data_reply);
ret = maat_wrap_redis_get_reply(ctx, &data_reply);
if (ret == REDIS_OK) {
if (data_reply->type == REDIS_REPLY_NIL) {
redis_transaction_success = 0;
@@ -175,7 +176,7 @@ static int connect_redis_for_write(struct source_redis_ctx *redis_ctx,
struct log_handle *logger)
{
assert(redis_ctx->write_ctx == NULL);
redis_ctx->write_ctx = maat_cmd_connect_redis(redis_ctx->redis_ip,
redis_ctx->write_ctx = maat_connect_redis(redis_ctx->redis_ip,
redis_ctx->redis_port,
redis_ctx->redis_db, logger);
if (NULL == redis_ctx->write_ctx) {
@@ -215,40 +216,7 @@ int maat_cmd_flushDB(struct maat *maat_inst)
return 0;
}
void maat_cmd_clear_rule_cache(struct serial_rule *s_rule)
{
if (s_rule->table_line != NULL) {
FREE(s_rule->table_line);
}
if (s_rule->n_foreign > 0) {
for (int i = 0; i < s_rule->n_foreign; i++) {
FREE(s_rule->f_keys[i].filename);
FREE(s_rule->f_keys[i].key);
}
FREE(s_rule->f_keys);
}
memset(s_rule, 0, sizeof(struct serial_rule));
}
void maat_cmd_set_serial_rule(struct serial_rule *rule, enum maat_operation op,
long long rule_id, const char *table_name,
const char *line, long long timeout)
{
memset(rule, 0, sizeof(struct serial_rule));
rule->op = op;
rule->rule_id = rule_id;
rule->timeout = timeout;
assert(strlen(table_name) < sizeof(rule->table_name));
strncpy(rule->table_name, table_name, sizeof(rule->table_name));
if (line != NULL) {
rule->table_line = maat_strdup(line);
}
}
int maat_cmd_get_valid_flag_offset(const char *line, int column_seq)
int maat_get_valid_flag_offset(const char *line, int column_seq)
{
size_t offset = 0;
size_t len = 0;
@@ -263,11 +231,11 @@ int maat_cmd_get_valid_flag_offset(const char *line, int column_seq)
return offset;
}
long long maat_cmd_redis_server_time_s(redisContext *c)
long long maat_redis_server_time_s(redisContext *c)
{
long long server_time = 0;
redisReply *data_reply = maat_cmd_wrap_redis_command(c, "TIME");
redisReply *data_reply = maat_wrap_redis_command(c, "TIME");
if (data_reply->type == REDIS_REPLY_ARRAY) {
server_time = atoll(data_reply->element[0]->str);
freeReplyObject(data_reply);
@@ -277,7 +245,7 @@ long long maat_cmd_redis_server_time_s(redisContext *c)
return server_time;
}
int maat_cmd_wrap_redis_get_reply(redisContext *c, redisReply **reply)
int maat_wrap_redis_get_reply(redisContext *c, redisReply **reply)
{
return redisGetReply(c, (void **)reply);
}
@@ -293,7 +261,7 @@ int maat_cmd_set_line(struct maat *maat_inst, const struct maat_cmd_line *line_r
return -1;
}
long long server_time = maat_cmd_redis_server_time_s(write_ctx);
long long server_time = maat_redis_server_time_s(write_ctx);
if(!server_time) {
return -1;
}
@@ -318,7 +286,7 @@ int maat_cmd_set_line(struct maat *maat_inst, const struct maat_cmd_line *line_r
return -1;
}
int valid_offset = maat_cmd_get_valid_flag_offset(line_rule->table_line, valid_column);
int valid_offset = maat_get_valid_flag_offset(line_rule->table_line, valid_column);
if (valid_offset < 0) {
log_error(maat_inst->logger, MODULE_MAAT_COMMAND,
"[%s:%d] Command set line id %lld failed: table %s valid_offset error",
@@ -332,8 +300,8 @@ int maat_cmd_set_line(struct maat *maat_inst, const struct maat_cmd_line *line_r
absolute_expire_time = server_time + line_rule->expire_after;
}
maat_cmd_set_serial_rule(s_rule + i, (enum maat_operation)is_valid, line_rule->rule_id,
line_rule->table_name, line_rule->table_line, absolute_expire_time);
maat_set_serial_rule(s_rule + i, (enum maat_operation)is_valid, line_rule->rule_id,
line_rule->table_name, line_rule->table_line, absolute_expire_time);
int success_cnt = maat_cmd_write_rule(write_ctx, s_rule, 1, server_time, maat_inst->logger);
if (success_cnt != 1) {
@@ -345,7 +313,7 @@ int maat_cmd_set_line(struct maat *maat_inst, const struct maat_cmd_line *line_r
maat_inst->stat->line_cmd_acc_num += success_cnt;
error_out:
maat_cmd_clear_rule_cache(s_rule);
maat_clear_rule_cache(s_rule);
FREE(s_rule);
return ret;
@@ -387,7 +355,7 @@ int maat_cmd_set_file(struct maat *maat_inst, const char *key, const char *value
arg_vec, len_vec);
break;
case MAAT_OP_DEL:
reply = maat_cmd_wrap_redis_command(ctx, "EXPIRE %s %d", key, MAAT_REDIS_SYNC_TIME);
reply = maat_wrap_redis_command(ctx, "EXPIRE %s %d", key, MAAT_REDIS_SYNC_TIME);
break;
default:
return -1;
@@ -416,7 +384,7 @@ long long maat_cmd_incrby(struct maat *maat_inst, const char *key, int increment
return -1;
}
redisReply *data_reply = maat_cmd_wrap_redis_command(write_ctx, "INCRBY %s %d", key, increment);
redisReply *data_reply = maat_wrap_redis_command(write_ctx, "INCRBY %s %d", key, increment);
if (data_reply->type == REDIS_REPLY_INTEGER) {
result = data_reply->integer;
} else {
@@ -427,64 +395,4 @@ long long maat_cmd_incrby(struct maat *maat_inst, const char *key, int increment
data_reply = NULL;
return result;
}
long long maat_cmd_get_config_version(struct maat *maat_inst)
{
long long new_version = -1;
if (maat_inst->new_version != INVALID_VERSION) {
new_version = maat_inst->new_version;
} else {
new_version = maat_inst->maat_version;
}
return new_version;
}
int maat_cmd_config_is_updating(struct maat *maat_inst)
{
int ret = 0;
if (0 == pthread_mutex_trylock(&(maat_inst->background_update_mutex))) {
ret = 0;
pthread_mutex_unlock(&(maat_inst->background_update_mutex));
} else {
ret = 1;
}
return ret;
}
char *maat_cmd_str_escape(char *dst, int size, const char *src)
{
int i = 0, j = 0;
int len = strlen(src);
for (i = 0, j = 0; i < len && j < size; i++) {
switch (src[i]) {
case '&':
dst[j] = '\\';
dst[j+1] = '&';
j += 2;
break;
case ' ':
dst[j] = '\\';
dst[j+1] = 'b';//space,0x20;
j += 2;
break;
case '\\':
dst[j] = '\\';
dst[j+1] = '\\';
j += 2;
break;
default:
dst[j] = src[i];
j++; //undo the followed i++
break;
}
}
dst[j] = '\0';
return dst;
}

135
src/maat_redis_monitor.c
View File

@@ -184,8 +184,8 @@ static int get_foreign_keys_define(redisContext *ctx, struct serial_rule *rule_l
return rule_with_foreign_key;
}
int maat_cmd_get_foreign_keys_by_prefix(redisContext *ctx, struct serial_rule *rule_list,
int rule_num, const char* dir, struct log_handle *logger)
int maat_get_foreign_keys_by_prefix(redisContext *ctx, struct serial_rule *rule_list,
int rule_num, const char* dir, struct log_handle *logger)
{
int j = 0;
int foreign_key_size = 0;
@@ -243,7 +243,7 @@ static int _get_maat_redis_value(redisContext *c, struct serial_rule *rule_list,
}
for (i = 0; i < rule_num; i++) {
ret = maat_cmd_wrap_redis_get_reply(c, &reply);
ret = maat_wrap_redis_get_reply(c, &reply);
if (ret == REDIS_ERR) {
log_error(logger, MODULE_REDIS_MONITOR,
"[%s:%d] Redis GET %s:%s,%lld failed, redis server error",
@@ -289,7 +289,7 @@ static int _get_maat_redis_value(redisContext *c, struct serial_rule *rule_list,
for (i = 0; i < failed_cnt; i++) {
idx = retry_ids[i];
ret = maat_cmd_wrap_redis_get_reply(c, &reply);
ret = maat_wrap_redis_get_reply(c, &reply);
if (ret == REDIS_ERR) {
log_error(logger, MODULE_REDIS_MONITOR,
"[%s:%d] redis command %s failed, redis server error",
@@ -321,8 +321,8 @@ static int _get_maat_redis_value(redisContext *c, struct serial_rule *rule_list,
return 0;
}
int maat_cmd_get_redis_value(redisContext *c, struct serial_rule *rule_list,
int rule_num, int print_process, struct log_handle *logger)
int maat_get_redis_value(redisContext *c, struct serial_rule *rule_list,
int rule_num, int print_process, struct log_handle *logger)
{
int max_redis_batch = 4096;
int success_cnt = 0;
@@ -377,9 +377,9 @@ static int get_inc_key_list(long long instance_version, long long target_version
return 0;
}
redisReply *tmp_reply= maat_cmd_wrap_redis_command(c, "ZSCORE %s %s",
mr_status_sset,
reply->element[0]->str);
redisReply *tmp_reply= maat_wrap_redis_command(c, "ZSCORE %s %s",
mr_status_sset,
reply->element[0]->str);
if (tmp_reply->type != REDIS_REPLY_STRING) {
log_error(logger, MODULE_REDIS_MONITOR,
"[%s:%d] ZSCORE %s %s failed Version: %lld->%lld",
@@ -393,7 +393,7 @@ static int get_inc_key_list(long long instance_version, long long target_version
return -1;
}
long long nearest_rule_version = maat_cmd_read_redis_integer(tmp_reply);
long long nearest_rule_version = maat_read_redis_integer(tmp_reply);
freeReplyObject(tmp_reply);
tmp_reply = NULL;
@@ -444,6 +444,39 @@ static int get_inc_key_list(long long instance_version, long long target_version
return rule_num;
}
void maat_clear_rule_cache(struct serial_rule *s_rule)
{
if (s_rule->table_line != NULL) {
FREE(s_rule->table_line);
}
if (s_rule->n_foreign > 0) {
for (int i = 0; i < s_rule->n_foreign; i++) {
FREE(s_rule->f_keys[i].filename);
FREE(s_rule->f_keys[i].key);
}
FREE(s_rule->f_keys);
}
memset(s_rule, 0, sizeof(struct serial_rule));
}
void maat_set_serial_rule(struct serial_rule *rule, enum maat_operation op,
long long rule_id, const char *table_name,
const char *line, long long timeout)
{
memset(rule, 0, sizeof(struct serial_rule));
rule->op = op;
rule->rule_id = rule_id;
rule->timeout = timeout;
assert(strlen(table_name) < sizeof(rule->table_name));
strncpy(rule->table_name, table_name, sizeof(rule->table_name));
if (line != NULL) {
rule->table_line = maat_strdup(line);
}
}
static void serial_rule_free(struct serial_rule *s_rule)
{
if (NULL == s_rule) {
@@ -559,10 +592,10 @@ static int recovery_history_version(const struct serial_rule *current, int curre
return ret;
}
int maat_cmd_get_rm_key_list(redisContext *c, long long instance_version,
long long desired_version, long long *new_version,
struct table_manager *tbl_mgr, struct serial_rule **list,
int *update_type, int cumulative_off, struct log_handle *logger)
int maat_get_rm_key_list(redisContext *c, long long instance_version,
long long desired_version, long long *new_version,
struct table_manager *tbl_mgr, struct serial_rule **list,
int *update_type, int cumulative_off, struct log_handle *logger)
{
int rule_num = 0;
long long target_version = 0;
@@ -585,7 +618,7 @@ int maat_cmd_get_rm_key_list(redisContext *c, long long instance_version,
return -1;
}
long long redis_version = maat_cmd_read_redis_integer(reply);
long long redis_version = maat_read_redis_integer(reply);
if (redis_version < 0) {
if (reply->type == REDIS_REPLY_ERROR) {
log_error(logger, MODULE_REDIS_MONITOR,
@@ -667,12 +700,12 @@ FULL_UPDATE:
size_t i = 0;
//consume reply "OK" and "QUEUED".
for (i = 0; i < append_cmd_cnt; i++) {
maat_cmd_wrap_redis_get_reply(c, &reply);
maat_wrap_redis_get_reply(c, &reply);
freeReplyObject(reply);
reply = NULL;
}
reply = maat_cmd_wrap_redis_command(c, "EXEC");
reply = maat_wrap_redis_command(c, "EXEC");
if (NULL == reply) {
log_error(logger, MODULE_REDIS_MONITOR,
"[%s:%d] Redis Communication error: %s",
@@ -689,7 +722,7 @@ FULL_UPDATE:
return -1;
}
*new_version = maat_cmd_read_redis_integer(reply->element[0]);
*new_version = maat_read_redis_integer(reply->element[0]);
redisReply *sub_reply = reply->element[1];
if (sub_reply->type != REDIS_REPLY_ARRAY) {
log_error(logger, MODULE_REDIS_MONITOR,
@@ -829,7 +862,7 @@ static void _get_foreign_conts(redisContext *c, struct serial_rule *rule_list,
redisReply *reply = NULL;
for (i = 0; i < key_num; i++) {
ret = maat_cmd_wrap_redis_get_reply(c, &reply);
ret = maat_wrap_redis_get_reply(c, &reply);
if (ret == REDIS_ERR) {
log_error(logger, MODULE_REDIS_MONITOR,
"[%s:%d] Get %s,%lld foreign key %s content failed, redis server error",
@@ -874,8 +907,8 @@ static void _get_foreign_conts(redisContext *c, struct serial_rule *rule_list,
return;
}
void maat_cmd_get_foreign_conts(redisContext *c, struct serial_rule *rule_list,
int rule_num, int print_fn, struct log_handle *logger)
void maat_get_foreign_conts(redisContext *c, struct serial_rule *rule_list,
int rule_num, int print_fn, struct log_handle *logger)
{
int max_redis_batch = 4096;
int success_cnt = 0;
@@ -893,7 +926,7 @@ static int invalidate_line(char *line, int column_seq)
return -1;
}
int offset = maat_cmd_get_valid_flag_offset(line, column_seq);
int offset = maat_get_valid_flag_offset(line, column_seq);
if (offset < 0) {
return -1;
}
@@ -903,7 +936,7 @@ static int invalidate_line(char *line, int column_seq)
return 0;
}
void maat_cmd_rewrite_table_line_with_foreign(struct serial_rule *s_rule)
void maat_rewrite_table_line_with_foreign(struct serial_rule *s_rule)
{
int i = 0;
size_t fn_size = 0;
@@ -950,8 +983,8 @@ static int redlock_try_lock(redisContext *c, const char *lock_name,
{
int ret = 0;
redisReply *reply = maat_cmd_wrap_redis_command(c, "SET %s locked NX PX %lld",
lock_name, expire);
redisReply *reply = maat_wrap_redis_command(c, "SET %s locked NX PX %lld",
lock_name, expire);
if (reply->type == REDIS_REPLY_NIL) {
ret = 0;
} else {
@@ -979,8 +1012,8 @@ static long long exec_serial_rule_begin(redisContext* c, size_t rule_num,
}
if (rule_num > renew_rule_num) {
data_reply = maat_cmd_wrap_redis_command(c, "INCRBY MAAT_PRE_VER 1");
*transaction_version = maat_cmd_read_redis_integer(data_reply);
data_reply = maat_wrap_redis_command(c, "INCRBY MAAT_PRE_VER 1");
*transaction_version = maat_read_redis_integer(data_reply);
freeReplyObject(data_reply);
data_reply = NULL;
if (*transaction_version < 0) {
@@ -989,7 +1022,7 @@ static long long exec_serial_rule_begin(redisContext* c, size_t rule_num,
}
if (*renew_allowed == 1 || rule_num > renew_rule_num) {
data_reply = maat_cmd_wrap_redis_command(c, "MULTI");
data_reply = maat_wrap_redis_command(c, "MULTI");
freeReplyObject(data_reply);
data_reply = NULL;
ret = 0;
@@ -1000,7 +1033,7 @@ static long long exec_serial_rule_begin(redisContext* c, size_t rule_num,
static void redlock_unlock(redisContext *c, const char *lock_name)
{
redisReply *reply = maat_cmd_wrap_redis_command(c, "DEL %s", lock_name);
redisReply *reply = maat_wrap_redis_command(c, "DEL %s", lock_name);
freeReplyObject(reply);
reply = NULL;
}
@@ -1027,7 +1060,7 @@ static redisReply* exec_serial_rule_end(redisContext *c, const char *transaction
}
if (strlen(transaction_list) > 0) {
data_reply = maat_cmd_wrap_redis_command(c, "eval %s 4 MAAT_VERSION %s %s %s %lld",
data_reply = maat_wrap_redis_command(c, "eval %s 4 MAAT_VERSION %s %s %s %lld",
lua_exec_done,
mr_status_sset,
mr_version_sset,
@@ -1039,7 +1072,7 @@ static redisReply* exec_serial_rule_end(redisContext *c, const char *transaction
(*cnt)++;
}
data_reply = maat_cmd_wrap_redis_command(c, "EXEC");
data_reply = maat_wrap_redis_command(c, "EXEC");
return data_reply;
}
@@ -1154,7 +1187,7 @@ static void exec_serial_rule(redisContext *c, const char *transaction_list,
}
for (i = 0; i < append_cmd_cnt; i++) {
maat_cmd_wrap_redis_get_reply(c, &data_reply);
maat_wrap_redis_get_reply(c, &data_reply);
freeReplyObject(data_reply);
data_reply=NULL;
}
@@ -1260,7 +1293,7 @@ int maat_cmd_write_rule(redisContext *c, struct serial_rule *s_rule,
}
if (transaction_version > 0) {
transaction_finished_version = maat_cmd_read_redis_integer(transaction_reply->element[multi_cmd_cnt - 1]);
transaction_finished_version = maat_read_redis_integer(transaction_reply->element[multi_cmd_cnt - 1]);
log_info(logger, MODULE_REDIS_MONITOR,
"Redis transaction MAAT_PRE_VER = %lld , MAAT_VERSION = %lld",
transaction_version, transaction_finished_version);
@@ -1297,12 +1330,12 @@ static void cleanup_update_status(redisContext *c, struct log_handle *logger)
long long version_num = 0;
long long entry_num = 0;
long long server_time = maat_cmd_redis_server_time_s(c);
long long server_time = maat_redis_server_time_s(c);
if (!server_time) {
return;
}
redisReply *reply = maat_cmd_wrap_redis_command(c, "MULTI");
redisReply *reply = maat_wrap_redis_command(c, "MULTI");
freeReplyObject(reply);
reply = NULL;
@@ -1317,13 +1350,13 @@ static void cleanup_update_status(redisContext *c, struct log_handle *logger)
//consume reply "OK" and "QUEUED".
for(int i = 0; i < append_cmd_cnt; i++) {
maat_cmd_wrap_redis_get_reply(c, &reply);
maat_wrap_redis_get_reply(c, &reply);
freeReplyObject(reply);
reply = NULL;
}
redisReply *sub_reply = NULL;
reply = maat_cmd_wrap_redis_command(c, "EXEC");
reply = maat_wrap_redis_command(c, "EXEC");
if (reply->type != REDIS_REPLY_ARRAY) {
goto error_out;
}
@@ -1338,16 +1371,16 @@ static void cleanup_update_status(redisContext *c, struct log_handle *logger)
goto error_out;
}
version_lower_bound = maat_cmd_read_redis_integer(sub_reply->element[0]);
version_upper_bound = maat_cmd_read_redis_integer(sub_reply->element[sub_reply->elements-1]);
version_lower_bound = maat_read_redis_integer(sub_reply->element[0]);
version_upper_bound = maat_read_redis_integer(sub_reply->element[sub_reply->elements-1]);
freeReplyObject(reply);
reply = NULL;
//To deal with maat_version reset to 0, do NOT use -inf as lower bound intentionally.
reply = maat_cmd_wrap_redis_command(c, "ZREMRANGEBYSCORE %s %lld %lld",
reply = maat_wrap_redis_command(c, "ZREMRANGEBYSCORE %s %lld %lld",
mr_status_sset, version_lower_bound,
version_upper_bound);
entry_num = maat_cmd_read_redis_integer(reply);
entry_num = maat_read_redis_integer(reply);
freeReplyObject(reply);
reply = NULL;
@@ -1365,12 +1398,12 @@ static void check_maat_expiration(redisContext *c, struct log_handle *logger)
{
UNUSED int ret = 0;
long long server_time = maat_cmd_redis_server_time_s(c);
long long server_time = maat_redis_server_time_s(c);
if (!server_time) {
return;
}
redisReply *data_reply= maat_cmd_wrap_redis_command(c, "ZRANGEBYSCORE %s -inf %lld",
redisReply *data_reply= maat_wrap_redis_command(c, "ZRANGEBYSCORE %s -inf %lld",
mr_expire_sset, server_time);
if (data_reply->type != REDIS_REPLY_ARRAY || 0 == data_reply->elements) {
freeReplyObject(data_reply);
@@ -1442,7 +1475,7 @@ void redis_monitor_traverse(long long version, struct source_redis_ctx *mr_ctx,
}
log_info(maat_inst->logger, MODULE_REDIS_MONITOR, "Reconnecting...");
mr_ctx->read_ctx = maat_cmd_connect_redis(mr_ctx->redis_ip,
mr_ctx->read_ctx = maat_connect_redis(mr_ctx->redis_ip,
mr_ctx->redis_port,
mr_ctx->redis_db,
maat_inst->logger);
@@ -1457,7 +1490,7 @@ void redis_monitor_traverse(long long version, struct source_redis_ctx *mr_ctx,
long long new_version = 0;
int update_type = MAAT_UPDATE_TYPE_INC;
int rule_num = maat_cmd_get_rm_key_list(mr_ctx->read_ctx, version,
int rule_num = maat_get_rm_key_list(mr_ctx->read_ctx, version,
maat_inst->load_specific_version,
&new_version, maat_inst->tbl_mgr,
&rule_list, &update_type,
@@ -1477,8 +1510,8 @@ void redis_monitor_traverse(long long version, struct source_redis_ctx *mr_ctx,
}
if (rule_num > 0) {
ret = maat_cmd_get_redis_value(mr_ctx->read_ctx, rule_list, rule_num,
0, maat_inst->logger);
ret = maat_get_redis_value(mr_ctx->read_ctx, rule_list, rule_num,
0, maat_inst->logger);
//redis communication error
if (ret < 0) {
redisFree(mr_ctx->read_ctx);
@@ -1504,8 +1537,8 @@ void redis_monitor_traverse(long long version, struct source_redis_ctx *mr_ctx,
ret = get_foreign_keys_define(mr_ctx->read_ctx, rule_list, rule_num,
maat_inst, maat_inst->opts.foreign_cont_dir);
if (ret > 0) {
maat_cmd_get_foreign_conts(mr_ctx->read_ctx, rule_list, rule_num, 0,
maat_inst->logger);
maat_get_foreign_conts(mr_ctx->read_ctx, rule_list, rule_num, 0,
maat_inst->logger);
}
}
@@ -1539,7 +1572,7 @@ void redis_monitor_traverse(long long version, struct source_redis_ctx *mr_ctx,
}
if (rule_list[i].n_foreign > 0) {
maat_cmd_rewrite_table_line_with_foreign(rule_list+i);
maat_rewrite_table_line_with_foreign(rule_list+i);
}
update_fn(rule_list[i].table_name, rule_list[i].table_line, u_param);
@@ -1556,7 +1589,7 @@ void redis_monitor_traverse(long long version, struct source_redis_ctx *mr_ctx,
clean_up:
for (i = 0; i < rule_num; i++) {
maat_cmd_clear_rule_cache(rule_list + i);
maat_clear_rule_cache(rule_list + i);
}
FREE(rule_list);

2
src/maat_rule.c
View File

@@ -276,7 +276,7 @@ void maat_read_full_config(struct maat *maat_inst)
log_info(maat_inst->logger, MODULE_MAAT_RULE,
"Maat initiate from Redis %s:%hu db%d",
redis_ctx->redis_ip, redis_ctx->redis_port, redis_ctx->redis_db);
redis_ctx->read_ctx = maat_cmd_connect_redis(redis_ctx->redis_ip,
redis_ctx->read_ctx = maat_connect_redis(redis_ctx->redis_ip,
redis_ctx->redis_port,
redis_ctx->redis_db,
maat_inst->logger);

33
src/maat_utils.c
View File

@@ -169,6 +169,39 @@ char *strtok_r_esc(char *s, const char delim, char **save_ptr)
return s;
}
char *str_escape(char *dst, int size, const char *src)
{
int i = 0, j = 0;
int len = strlen(src);
for (i = 0, j = 0; i < len && j < size; i++) {
switch (src[i]) {
case '&':
dst[j] = '\\';
dst[j+1] = '&';
j += 2;
break;
case ' ':
dst[j] = '\\';
dst[j+1] = 'b';//space,0x20;
j += 2;
break;
case '\\':
dst[j] = '\\';
dst[j+1] = '\\';
j += 2;
break;
default:
dst[j] = src[i];
j++; //undo the followed i++
break;
}
}
dst[j] = '\0';
return dst;
}
char *str_unescape(char *s)
{
size_t i=0;

1
src/version.map
View File

@@ -16,6 +16,7 @@ global:
maat_state_*;
maat_helper*;
maat_stream_*;
maat_cmd_*;
};
local: *;
};