gfwleak/tango-maat
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

redis内部key改为unsigned long，适配性能测试用例。

Browse Source
This commit is contained in:
zhengchao
2020-06-18 21:59:44 +08:00
parent af27d7197c
commit 73d27d983c
7 changed files with 226 additions and 153 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
inc/Maat_command.h
View File

@@ -185,8 +185,8 @@ int Maat_command_batch_set_group2compile(struct Maat_command_batch* batch, enum
int Maat_command_batch_set_compile(struct Maat_command_batch* batch, enum MAAT_OPERATION op, const struct Maat_rule_t* compile, const char* table_name, const char * huge_service_defined, int clause_num, int label_id, int expire_after);
int Maat_command_batch_commit(struct Maat_command_batch* batch);
int Maat_cmd_get_new_group_id(Maat_feather_t feather);
int Maat_cmd_get_new_region_id(Maat_feather_t feather);
int Maat_command_get_new_group_id(Maat_feather_t feather);
int Maat_command_get_new_region_id(Maat_feather_t feather);
#ifdef __cplusplus
} //end extern"C"

43
src/entry/Maat_command.cpp
View File

@@ -97,7 +97,7 @@ int connect_redis_for_write(struct source_redis_ctx* mr_ctx, void* logger)
return 0;
}
}
redisContext* get_redis_ctx_for_write(_Maat_feather_t * feather)
redisContext* get_redis_ctx_for_write(struct _Maat_feather_t * feather)
{
int ret=0;
if(feather->mr_ctx.write_ctx==NULL)
@@ -361,7 +361,7 @@ void empty_serial_rules(struct serial_rule_t* rule)
memset(rule,0,sizeof(struct serial_rule_t));
return;
}
void set_serial_rule(struct serial_rule_t* rule, enum MAAT_OPERATION op,int rule_id,int label_id,const char* table_name,const char* line, long long timeout)
void set_serial_rule(struct serial_rule_t* rule, enum MAAT_OPERATION op, unsigned long rule_id,int label_id,const char* table_name,const char* line, long long timeout)
{
memset(rule, 0, sizeof(struct serial_rule_t));
rule->op=op;
@@ -430,7 +430,7 @@ int get_inc_key_list(long long instance_version, long long target_version, redis
for(i=0, j=0;i<reply->elements;i++)
{
assert(reply->element[i]->type==REDIS_REPLY_STRING);
ret=sscanf(reply->element[i]->str,"%[^,],%[^,],%ld",op_str,s_rule[j].table_name,&(s_rule[j].rule_id));
ret=sscanf(reply->element[i]->str,"%[^,],%[^,],%lu",op_str,s_rule[j].table_name,&(s_rule[j].rule_id));
if(ret!=3||s_rule[i].rule_id<0)
{
MESA_handle_runtime_log(logger, RLOG_LV_FATAL, maat_redis_monitor,
@@ -989,7 +989,7 @@ void _exec_serial_rule(redisContext* ctx, const char* transaction_list, struct s
switch(s_rule[i].op)
{
case MAAT_OP_ADD:
redisAppendCommand(ctx,"SET %s:%s,%d %s",
redisAppendCommand(ctx,"SET %s:%s,%lu %s",
mr_key_prefix[MAAT_OP_ADD],
s_rule[i].table_name,
s_rule[i].rule_id,
@@ -998,7 +998,7 @@ void _exec_serial_rule(redisContext* ctx, const char* transaction_list, struct s
(*cnt)++;
append_cmd_cnt++;
//Allowing add duplicated members for rule id recycling.
redisAppendCommand(ctx,"RPUSH %s ADD,%s,%d",
redisAppendCommand(ctx,"RPUSH %s ADD,%s,%lu",
transaction_list,
s_rule[i].table_name,
s_rule[i].rule_id);
@@ -1007,7 +1007,7 @@ void _exec_serial_rule(redisContext* ctx, const char* transaction_list, struct s
append_cmd_cnt++;
if(s_rule[i].timeout>0)
{
redisAppendCommand(ctx,"ZADD %s %lld %s,%d",
redisAppendCommand(ctx,"ZADD %s %lld %s,%lu",
mr_expire_sset,
s_rule[i].timeout,
s_rule[i].table_name,
@@ -1019,7 +1019,7 @@ void _exec_serial_rule(redisContext* ctx, const char* transaction_list, struct s
}
if(s_rule[i].label_id>0)
{
redisAppendCommand(ctx,"ZADD %s %d %s,%d",
redisAppendCommand(ctx,"ZADD %s %d %s,%lu",
mr_label_sset,
s_rule[i].label_id,
s_rule[i].table_name,
@@ -1033,7 +1033,7 @@ void _exec_serial_rule(redisContext* ctx, const char* transaction_list, struct s
}
break;
case MAAT_OP_DEL:
redisAppendCommand(ctx,"RENAME %s:%s,%d %s:%s,%d",
redisAppendCommand(ctx,"RENAME %s:%s,%lu %s:%s,%lu",
mr_key_prefix[MAAT_OP_ADD],
s_rule[i].table_name,
s_rule[i].rule_id,
@@ -1045,7 +1045,7 @@ void _exec_serial_rule(redisContext* ctx, const char* transaction_list, struct s
(*cnt)++;
append_cmd_cnt++;
redisAppendCommand(ctx,"EXPIRE %s:%s,%d %d",
redisAppendCommand(ctx,"EXPIRE %s:%s,%lu %d",
mr_key_prefix[MAAT_OP_DEL],
s_rule[i].table_name,
s_rule[i].rule_id,
@@ -1055,7 +1055,7 @@ void _exec_serial_rule(redisContext* ctx, const char* transaction_list, struct s
append_cmd_cnt++;
//NX: Don't update already exisiting elements. Always add new elements.
redisAppendCommand(ctx,"RPUSH %s DEL,%s,%d",
redisAppendCommand(ctx,"RPUSH %s DEL,%s,%lu",
transaction_list,
s_rule[i].table_name,
s_rule[i].rule_id);
@@ -1064,7 +1064,7 @@ void _exec_serial_rule(redisContext* ctx, const char* transaction_list, struct s
append_cmd_cnt++;
// Try to remove from expiration sorted set, no matter wheather it exists or not.
redisAppendCommand(ctx,"ZREM %s %s,%d",
redisAppendCommand(ctx,"ZREM %s %s,%lu",
mr_expire_sset,
s_rule[i].table_name,
s_rule[i].rule_id);
@@ -1073,7 +1073,7 @@ void _exec_serial_rule(redisContext* ctx, const char* transaction_list, struct s
append_cmd_cnt++;
// Try to remove from label sorted set, no matter wheather it exists or not.
redisAppendCommand(ctx,"ZREM %s %s,%d",
redisAppendCommand(ctx,"ZREM %s %s,%lu",
mr_label_sset,
s_rule[i].table_name,
s_rule[i].rule_id);
@@ -1087,7 +1087,7 @@ void _exec_serial_rule(redisContext* ctx, const char* transaction_list, struct s
continue;
}
//s_rule[i].timeout>0 was checked by caller.
redisAppendCommand(ctx,"ZADD %s %lld %s,%d",
redisAppendCommand(ctx,"ZADD %s %lld %s,%lu",
mr_expire_sset,
s_rule[i].timeout,
s_rule[i].table_name,
@@ -2051,13 +2051,13 @@ long long Maat_cmd_incrby(Maat_feather_t feather,const char* key, int increment)
}
freeReplyObject(data_reply);
return result;
}
}
int Maat_command_get_new_group_id(Maat_feather_t feather)
{
int group_id=0;
group_id=(int) Maat_cmd_incrby(feather, mr_group_id_var, 1);
return group_id;
}
}
int Maat_command_get_new_region_id(Maat_feather_t feather)
{
int region_id=0;
@@ -2212,6 +2212,7 @@ int Maat_command_batch_set_region(struct Maat_command_batch* batch, enum MAAT_O
{
struct serial_rule_t* s_rule=ALLOC(struct serial_rule_t, 1);
long long absolute_expire_time=0;
char line[MAX_TABLE_LINE_SIZE];
serialize_region(region, group_id, line, sizeof(line));
@@ -2221,17 +2222,19 @@ int Maat_command_batch_set_region(struct Maat_command_batch* batch, enum MAAT_O
batch->batch_size++;
return 0;
}
#define TO_GROUP2X_KEY(group_id, parent_id) ((unsigned long)group_id<<32|parent_id)
int Maat_command_batch_set_group2group(struct Maat_command_batch* batch, enum MAAT_OPERATION op, const struct Maat_cmd_group2group* g2g)
{
struct serial_rule_t* s_rule=ALLOC(struct serial_rule_t, 1);
long long absolute_expire_time=0;
char line[MAX_TABLE_LINE_SIZE];
serialize_group2group(op, g2g, line, sizeof(line));
set_serial_rule(s_rule, op, TO_GROUP2X_KEY(g2g->group_id, g2g->superior_group_id), 0, g2g->table_name,
line, absolute_expire_time);
TAILQ_INSERT_TAIL(&batch->queue, s_rule, entries);
batch->batch_size++;
@@ -2243,9 +2246,7 @@ int Maat_command_batch_set_group2compile(struct Maat_command_batch* batch, enum
long long absolute_expire_time=0;
char line[MAX_TABLE_LINE_SIZE];
serialize_group2compile(op, g2c, line, sizeof(line));
assert(g2c->group_id<1024*1024);
serialize_group2compile(op, g2c, line, sizeof(line));
set_serial_rule(s_rule, op, TO_GROUP2X_KEY(g2c->group_id, g2c->compile_id), 0, g2c->table_name,
line, absolute_expire_time);
TAILQ_INSERT_TAIL(&batch->queue, s_rule, entries);
@@ -2265,6 +2266,7 @@ int Maat_command_batch_set_compile(struct Maat_command_batch* batch, enum MAAT_O
absolute_expire_time=batch->server_time+expire_after;
}
set_serial_rule(s_rule, op, compile->config_id, label_id, table_name,
line, absolute_expire_time);
TAILQ_INSERT_TAIL(&batch->queue, s_rule, entries);
batch->batch_size++;
@@ -2275,6 +2277,7 @@ int Maat_command_batch_commit(struct Maat_command_batch* batch)
struct serial_rule_t* s_rule_array=ALLOC(struct serial_rule_t, batch->batch_size);
int i=0;
redisContext* write_ctx=get_redis_ctx_for_write(batch->feather);
struct serial_rule_t * tmp = TAILQ_FIRST(&batch->queue);
while(tmp != NULL)
{

2
src/entry/json2iris.cpp
View File

@@ -1200,7 +1200,7 @@ int write_iris(cJSON *json, struct iris_description_t *p_iris, void* logger)
"compile rule %d have no groups.",compile_id);
return -1;
}
i=1;
i=0;
cJSON_ArrayForEach(group_obj, group_array)
{
ret=write_group_rule(group_obj, compile_id, PARENT_TYPE_COMPILE, compile_id, i, p_iris, logger);

4
src/inc_internal/Maat_rule_internal.h
View File

@@ -348,7 +348,7 @@ struct foreign_key
struct serial_rule_t //rm= Redis Maat
{
enum MAAT_OPERATION op;//0: delete, 1: add.
long rule_id;
unsigned long rule_id;
int label_id;
char with_error;
long long timeout; // absolute unix time.
@@ -386,7 +386,7 @@ MAAT_RULE_EX_DATA rule_ex_data_new(const struct Maat_rule_head * rule_head, cons
void rule_ex_data_free(const struct Maat_rule_head * rule_head, const char* srv_def, MAAT_RULE_EX_DATA *ad, const struct compile_ex_data_idx* ex_desc);
void set_serial_rule(struct serial_rule_t* rule,enum MAAT_OPERATION op,int rule_id,int label_id,const char* table_name,const char* line, long long timeout);
void set_serial_rule(struct serial_rule_t* rule,enum MAAT_OPERATION op, unsigned long rule_id,int label_id,const char* table_name,const char* line, long long timeout);
void empty_serial_rules(struct serial_rule_t* rule);
int exec_serial_rule(redisContext* ctx,struct serial_rule_t* s_rule,unsigned int serial_rule_num, long long server_time, void* logger);
long long redis_server_time(redisContext* ctx);

4
test/CMakeLists.txt
View File

@@ -7,8 +7,8 @@ target_link_libraries(test_igraph igraph-static)
add_executable(test_maatframe test_maatframe.cpp)
target_link_libraries(test_maatframe maat_frame_shared gtest)
#add_executable(perf_test_maatframe perf_test_maatframe.cpp)
#target_link_libraries(perf_test_maatframe maat_frame_shared gtest)
add_executable(perf_test_maatframe perf_test_maatframe.cpp)
target_link_libraries(perf_test_maatframe maat_frame_shared gtest)
configure_file(table_info.conf table_info.conf COPYONLY)
configure_file(file_test_tableinfo.conf file_test_tableinfo.conf COPYONLY)

191
test/perf_test_maatframe.cpp
View File

@@ -3,24 +3,93 @@
#include "Maat_command.h"
#include <MESA/MESA_handle_logger.h>
#include <gtest/gtest.h>
int test_add_expr_command_copy(Maat_feather_t feather,const char* region_table,int config_id, int timeout,int label_id, const char* keywords)
#include <stdlib.h>
#include <arpa/inet.h>
void ipv4_addr_set_copy(struct ipaddr *ipv4_addr, struct stream_tuple4_v4* v4_addr,
const char* src_ip, unsigned short sport, const char* dest_ip, unsigned short dport)
{
ipv4_addr->addrtype=ADDR_TYPE_IPV4;
inet_pton(AF_INET, src_ip, &(v4_addr->saddr));
v4_addr->source=htons(sport);
inet_pton(AF_INET, dest_ip, &(v4_addr->daddr));
v4_addr->dest=htons(dport);
ipv4_addr->v4=v4_addr;
return;
}
void random_keyword_generate(char* keyword_buf, size_t sz)
{
#define MIN_KEYWORD_LEN 4
size_t i=0, len=0;
len=random()%(sz-1-MIN_KEYWORD_LEN)+MIN_KEYWORD_LEN;
for(i=0; i<len; i++)
{
keyword_buf[i]='0'+random()%('~' - '0');
}
keyword_buf[i]='\0';
return;
}
void random_ipv4_addr_generate(char* ip_buff, size_t sz)
{
unsigned int ip=random();
inet_ntop(AF_INET, &ip, ip_buff, sz);
return;
}
int test_add_ip_command(struct Maat_command_batch* batch, const char* region_table, const char* src_ip, const char* dst_ip, int config_id)
{
struct Maat_cmd_group2compile g2c;
struct Maat_cmd_group2group g2g;
struct Maat_rule_t rule;
struct Maat_cmd_region region;
int group_num=1,ret=0;
memset(&rule,0,sizeof(rule));
rule.config_id=config_id;
strcpy(rule.service_defined,"maat_command");
Maat_command_raw_set_compile(feather, MAAT_OP_ADD, &rule, "COMPILE", NULL, 1, timeout, label_id);
memset(g2c, 0, sizeof(g2c));
Maat_command_batch_set_compile(batch, MAAT_OP_ADD, &rule, "COMPILE", NULL, 1, 0, 0);
memset(&g2c, 0, sizeof(g2c));
g2c.group_id=config_id;
g2c.compile_id=config_id;
g2c.table_name="GROUP2COMPILE";
Maat_command_raw_set_group2compile(feather, MAAT_OP_ADD, &g2c);
Maat_command_batch_set_group2compile(batch, MAAT_OP_ADD, &g2c);
struct Maat_cmd_region region;
memset(&region, 0, sizeof(region));
region.region_type=REGION_IP;
region.table_name=region_table;
region.ip_rule.addr_type=ADDR_TYPE_IPv4;
region.ip_rule.direction=ADDR_DIR_DOUBLE;
region.ip_rule.src_ip=src_ip;
region.ip_rule.mask_src_ip="255.255.255.255";
region.ip_rule.src_port=0;
region.ip_rule.mask_src_port=0;//means any port should hit.
region.ip_rule.dst_ip=dst_ip;
region.ip_rule.mask_dst_ip="255.255.255.255";
region.ip_rule.dst_port=0;
region.ip_rule.mask_dst_port=65535;
region.ip_rule.protocol=0;//means any protocol should hit.
Maat_command_batch_set_region(batch, MAAT_OP_ADD, &region, g2c.group_id);
return 0;
}
int test_add_expr_command_copy(struct Maat_command_batch* batch, const char* region_table,int config_id, const char* keywords)
{
struct Maat_cmd_group2compile g2c;
struct Maat_rule_t rule;
struct Maat_cmd_region region;
memset(&rule,0,sizeof(rule));
rule.config_id=config_id;
strcpy(rule.service_defined,"maat_command");
Maat_command_batch_set_compile(batch, MAAT_OP_ADD, &rule, "COMPILE", NULL, 1, 0, 0);
memset(&g2c, 0, sizeof(g2c));
g2c.group_id=config_id;
g2c.compile_id=config_id;
g2c.table_name="GROUP2COMPILE";
Maat_command_batch_set_group2compile(batch, MAAT_OP_ADD, &g2c);
memset(&region,0,sizeof(region));
region.region_id=config_id;
region.region_type=REGION_EXPR;
region.table_name=region_table;
region.expr_rule.district=NULL;
@@ -28,9 +97,8 @@ int test_add_expr_command_copy(Maat_feather_t feather,const char* region_table,i
region.expr_rule.expr_type=EXPR_TYPE_AND;
region.expr_rule.match_method=MATCH_METHOD_SUB;
region.expr_rule.hex_bin=UNCASE_PLAIN;
Maat_command_raw_set_region(feather, MAAT_OP_ADD, &region, g2c->group_id);
Maat_command_batch_set_region(batch, MAAT_OP_ADD, &region, g2c.group_id);
return 0;
}
void wait_for_cmd_effective_copy(Maat_feather_t feather, long long version_before)
{
@@ -63,7 +131,7 @@ protected:
int scan_interval_ms=500;
int effective_interval_ms=0;
logger=MESA_create_runtime_log_handle("test_maat_redis.log",0);
logger=MESA_create_runtime_log_handle("maat_perf_test.log",0);
_shared_feather=Maat_feather(g_iThreadNum, table_info_path, logger);
Maat_set_feather_opt(_shared_feather,MAAT_OPT_INSTANCE_NAME,"perf", strlen("perf")+1);
@@ -96,83 +164,62 @@ void* MaatCMDPerfTest::logger;
//Following tests must be coded/tested at last, for they stalled the maat update thread and interrupt other tests.
TEST_F(MaatCMDPerfTest, SetExpr200K)
{
const int CMD_EXPR_NUM=200*1000;
const char* table_name="HTTP_URL";
const int CMD_EXPR_NUM=1*1000*1000;
const int CMD_IP_NUM=1*1000*1000;
const char* expr_table_name="HTTP_URL";
const char* ip_table_name="IP_CONFIG";
const char* keywords1="Hiredis";
const char* keywords2="C Client";
char escape_buff1[256],escape_buff2[256];
char keywords[256];
char keyword_buf[128];
char src_ip_buff[32], dst_ip_buff[32];
int label_id=5210, config_id=0,ret=0, output_id_cnt=0;
int config_id=0, ret=0;
Maat_feather_t feather=MaatCMDPerfTest::_shared_feather;
long long version_before=0;
ret=Maat_read_state(feather, MAAT_STATE_VERSION, &version_before, sizeof(version_before));
Maat_str_escape(escape_buff1, sizeof(escape_buff1),keywords1);
Maat_str_escape(escape_buff2, sizeof(escape_buff2),keywords2);
snprintf(keywords,sizeof(keywords),"%s&%s",escape_buff1,escape_buff2);
config_id=(int)Maat_cmd_incrby(feather, "TEST_SEQ", CMD_EXPR_NUM);
config_id=(int)Maat_cmd_incrby(feather, "TEST_SEQ", CMD_EXPR_NUM+CMD_IP_NUM);
int i=0;
struct Maat_command_batch* batch=NULL;
batch=Maat_command_batch_new(feather);
for(i=0; i<CMD_EXPR_NUM;i++)
{
test_add_expr_command_copy(feather,table_name,config_id-i, 0, label_id, keywords);
random_keyword_generate(keyword_buf, sizeof(keyword_buf));
test_add_expr_command_copy(batch, expr_table_name, config_id-i, keyword_buf);
}
struct Maat_rule_t rule;
memset(&rule,0,sizeof(rule));
for(i=0; i<CMD_EXPR_NUM;i++)
for(i=0; i<CMD_IP_NUM; i++)
{
memset(&rule,0,sizeof(rule));
rule.config_id=output_ids[i];
cmd=Maat_create_cmd(&rule, 0);
ret=Maat_cmd_append(feather, cmd, MAAT_OP_DEL);
EXPECT_EQ(ret, 0);
Maat_free_cmd(cmd);
random_ipv4_addr_generate(src_ip_buff, sizeof(src_ip_buff));
random_ipv4_addr_generate(dst_ip_buff, sizeof(dst_ip_buff));
test_add_ip_command(batch, ip_table_name, src_ip_buff, dst_ip_buff, config_id-CMD_EXPR_NUM-i);
}
ret=Maat_cmd_commit(feather);
EXPECT_EQ(ret, CMD_EXPR_NUM);
free(output_ids);
}
TEST_F(MaatCMDPerfTest, SetLinesOneMillion)
{
const int TEST_CMD_LINE_NUM=1000*1000;
Maat_feather_t feather=MaatCMDPerfTest::_shared_feather;
struct Maat_cmd_line **p_line=(struct Maat_cmd_line **)calloc(sizeof(struct Maat_cmd_line *), TEST_CMD_LINE_NUM);
struct Maat_cmd_line *line_rule=(struct Maat_cmd_line *)calloc(sizeof(struct Maat_cmd_line), TEST_CMD_LINE_NUM);
int i=0;
const char* line="1\t192.168.0.1\t4444444444\t1";
int seq=(int)Maat_cmd_incrby(feather,"TEST_PLUG_SEQ", TEST_CMD_LINE_NUM);
for(i=0;i<TEST_CMD_LINE_NUM;i++)
{
line_rule[i].label_id=0;
line_rule[i].rule_id=seq-i;
line_rule[i].table_name="QD_ENTRY_INFO";
line_rule[i].table_line=line;
// asprintf(&(line_rule[i].table_line),"1\t192.168.0.1\t%d\t1",100+i);
line_rule[i].expire_after=0;
p_line[i]=line_rule+i;
}
long long version_before=0;
Maat_read_state(feather,MAAT_STATE_VERSION, &version_before, sizeof(version_before));
Maat_cmd_set_lines(feather,(const struct Maat_cmd_line **)p_line,TEST_CMD_LINE_NUM, MAAT_OP_ADD);
Maat_command_batch_commit(batch);
wait_for_cmd_effective_copy(feather, version_before);
for(i=0;i<TEST_CMD_LINE_NUM;i++)
{
line_rule[i].table_line=NULL;
}
Maat_cmd_set_lines(feather, (const struct Maat_cmd_line **)p_line,TEST_CMD_LINE_NUM, MAAT_OP_DEL);
free(p_line);
free(line_rule);
struct Maat_rule_t result;
scan_status_t mid=NULL;
int table_id=0;
table_id=Maat_table_register(feather, expr_table_name);
ASSERT_GT(table_id, 0);
memset(&result, 0, sizeof(result));
ret=Maat_full_scan_string(feather, table_id, CHARSET_GBK, keyword_buf, strlen(keyword_buf),
&result, NULL, 1,
&mid, 0);
EXPECT_EQ(ret, 1);
Maat_clean_status(&mid);
struct ipaddr ipv4_addr;
struct stream_tuple4_v4 v4_addr;
ipv4_addr_set_copy(&ipv4_addr, &v4_addr, src_ip_buff, 50001, dst_ip_buff, 80);
table_id=Maat_table_register(feather, ip_table_name);
ASSERT_GT(table_id, 0);
memset(&result, 0, sizeof(result));
ret=Maat_scan_proto_addr(feather, table_id, &ipv4_addr, 6, &result, 1, &mid, 0);
EXPECT_EQ(ret, 1);
Maat_clean_status(&mid);
return;
}

115
test/test_maatframe.cpp
View File

@@ -2076,7 +2076,7 @@ Maat_feather_t MaatCmdTest::_shared_feather;
void* MaatCmdTest::logger;
int MaatCmdTest::linger_timeout;
int test_add_expr_command(Maat_feather_t feather,const char* region_table,int config_id, int timeout,int label_id, const char* keywords)
int test_add_expr_command(struct Maat_command_batch* batch, Maat_feather_t feather, const char* region_table, int config_id, int timeout, int label_id, const char* keywords)
{
char huge_serv_def[1024*2];
memset(huge_serv_def,'s',sizeof(huge_serv_def)-1);
@@ -2085,11 +2085,11 @@ int test_add_expr_command(Maat_feather_t feather,const char* region_table,int co
memset(&compile, 0, sizeof(compile));
compile.config_id=config_id;
strcpy(compile.service_defined, "maat_command");
Maat_command_raw_set_compile(feather, MAAT_OP_ADD, &compile, "COMPILE", huge_serv_def, 1, label_id, timeout);
Maat_command_batch_set_compile(batch, MAAT_OP_ADD, &compile, "COMPILE", huge_serv_def, 1, label_id, timeout);
struct Maat_cmd_region region;
memset(&region,0,sizeof(region));
region.region_id=Maat_cmd_get_new_region_id(feather);
region.region_id=Maat_command_get_new_region_id(feather);
region.region_type=REGION_EXPR;
region.table_name=region_table;
region.expr_rule.district=NULL;
@@ -2100,15 +2100,24 @@ int test_add_expr_command(Maat_feather_t feather,const char* region_table,int co
struct Maat_cmd_group2compile g2c;
memset(&g2c, 0, sizeof(g2c));
g2c.group_id=Maat_cmd_get_new_group_id(feather);
g2c.group_id=Maat_command_get_new_group_id(feather);;
g2c.table_name="GROUP2COMPILE";
g2c.compile_id=config_id;
g2c.clause_index=1;
Maat_command_raw_set_group2compile(feather, MAAT_OP_ADD, &g2c);
Maat_command_raw_set_region(feather, MAAT_OP_ADD, &region, g2c.group_id);
Maat_command_batch_set_group2compile(batch, MAAT_OP_ADD, &g2c);
Maat_command_batch_set_region(batch, MAAT_OP_ADD, &region, g2c.group_id);
return 0;
}
int test_add_expr_command_simple(Maat_feather_t feather, const char* region_table, int config_id, int timeout, int label_id, const char* keywords)
{
struct Maat_command_batch* batch=NULL;
batch=Maat_command_batch_new(feather);
test_add_expr_command(batch, feather, region_table, config_id, timeout, label_id, keywords);
Maat_command_batch_commit(batch);
return 0;
}
int del_command(Maat_feather_t feather, int config_id)
{
int ret=0;
@@ -2133,20 +2142,21 @@ TEST_F(MaatCmdTest, SetIP)
compile.config_id=config_id;
strcpy(compile.service_defined, "maat_command");
struct Maat_command_batch* batch=NULL;
batch=Maat_command_batch_new(feather);
//MUST acqire by function, because Maat_cmd_t has some hidden members.
Maat_command_raw_set_compile(feather, MAAT_OP_ADD, &compile, "COMPILE", NULL, 1, 0, timeout);
Maat_command_batch_set_compile(batch, MAAT_OP_ADD, &compile, "COMPILE", NULL, 1, 0, timeout);
struct Maat_cmd_group2compile g2c;
memset(&g2c, 0, sizeof(g2c));
g2c.group_id=Maat_cmd_get_new_group_id(feather);
g2c.group_id=Maat_command_get_new_group_id(feather);
g2c.compile_id=compile.config_id;
g2c.clause_index=1;
g2c.table_name="GROUP2COMPILE";
Maat_command_raw_set_group2compile(feather, MAAT_OP_ADD, &g2c);
Maat_command_batch_set_group2compile(batch, MAAT_OP_ADD, &g2c);
ret=Maat_read_state(feather,MAAT_STATE_VERSION, &version_before, sizeof(version_before));
EXPECT_EQ(ret, 0);
struct Maat_cmd_region region;
@@ -2165,9 +2175,14 @@ TEST_F(MaatCmdTest, SetIP)
region.ip_rule.dst_port=80;
region.ip_rule.mask_dst_port=65535;
region.ip_rule.protocol=0;//means any protocol should hit.
ret=Maat_command_raw_set_region(feather, MAAT_OP_ADD, &region, g2c.group_id);
ret=Maat_command_batch_set_region(batch, MAAT_OP_ADD, &region, g2c.group_id);
EXPECT_GE(ret, 0);
ret=Maat_read_state(feather,MAAT_STATE_VERSION, &version_before, sizeof(version_before));
EXPECT_EQ(ret, 0);
Maat_command_batch_commit(batch);
wait_for_cmd_effective(feather, version_before);
struct ipaddr ipv4_addr;
@@ -2221,7 +2236,7 @@ TEST_F(MaatCmdTest, SetExpr)
struct Maat_rule_t result;
int timeout=0;//seconds
int label_id=5210;
long long version_before=0,version_after=0;
long long version_before=0;
Maat_feather_t feather=MaatCmdTest::_shared_feather;
Maat_str_escape(escape_buff1, sizeof(escape_buff1),keywords1);
@@ -2230,14 +2245,15 @@ TEST_F(MaatCmdTest, SetExpr)
config_id=(int)Maat_cmd_incrby(feather, "TEST_SEQ", 2);
ret=Maat_read_state(feather,MAAT_STATE_VERSION, &version_before, sizeof(version_before));
test_add_expr_command(feather,table_name,config_id-1, 0, label_id, keywords);
test_add_expr_command(feather,table_name,config_id, 0, label_id, keywords);
struct Maat_command_batch* batch=NULL;
batch=Maat_command_batch_new(feather);
usleep(WAIT_FOR_EFFECTIVE_US);//waiting for commands go into effect
test_add_expr_command(batch, feather, table_name, config_id-1, 0, label_id, keywords);
test_add_expr_command(batch, feather, table_name, config_id, 0, label_id, keywords);
ret=Maat_read_state(feather,MAAT_STATE_VERSION, &version_before, sizeof(version_before));
Maat_command_batch_commit(batch);
ret=Maat_read_state(feather,MAAT_STATE_VERSION, &version_after, sizeof(version_after));
EXPECT_EQ(ret, 0);
EXPECT_GT(version_after, version_before);
wait_for_cmd_effective(feather, version_before);
table_id=Maat_table_register(feather, table_name);
ASSERT_GT(table_id, 0);
@@ -2268,7 +2284,9 @@ TEST_F(MaatCmdTest, SetExpr)
timeout=1;
config_id=(int)Maat_cmd_incrby(feather, "TEST_SEQ", 1);
test_add_expr_command(feather,table_name,config_id, timeout, label_id, keywords);
batch=Maat_command_batch_new(feather);
test_add_expr_command(batch, feather, table_name, config_id, timeout, label_id, keywords);
Maat_command_batch_commit(batch);
usleep(timeout*1000*1000+WAIT_FOR_EFFECTIVE_US);
ret=Maat_full_scan_string(feather, table_id,CHARSET_GBK, scan_data, strlen(scan_data),
@@ -2293,7 +2311,8 @@ TEST_F(MaatCmdTest, RuleIDRecycle)
int rule_id=(int)Maat_cmd_incrby(feather, "TEST_SEQ", 1);
int label_id=5211, ret=0;
test_add_expr_command(feather,table_name,rule_id, 0, label_id, keywords);
test_add_expr_command_simple(feather,table_name,rule_id, 0, label_id, keywords);
usleep(WAIT_FOR_EFFECTIVE_US);//waiting for commands go into effect
@@ -2312,7 +2331,7 @@ TEST_F(MaatCmdTest, RuleIDRecycle)
Maat_clean_status(&mid);
EXPECT_EQ(ret, -2);
test_add_expr_command(feather,table_name,rule_id, 0, label_id, keywords);
test_add_expr_command_simple(feather,table_name,rule_id, 0, label_id, keywords);
usleep(WAIT_FOR_EFFECTIVE_US);//waiting for commands go into effect
memset(&result, 0, sizeof(result));
@@ -2343,12 +2362,16 @@ TEST_F(MaatCmdTest, ReturnRuleIDWithDescendingOrder)
int rule_id=(int)Maat_cmd_incrby(feather, "TEST_SEQ", repeat_times);
int label_id=5211, ret=0;
struct Maat_command_batch* batch=NULL;
batch=Maat_command_batch_new(feather);
for(i=0; i<repeat_times; i++)
{
//add in ascending order
expect_ruleid[i]=rule_id+1-repeat_times+i;
test_add_expr_command(feather,table_name,rule_id+1-repeat_times+i, 0, label_id, keywords);
test_add_expr_command(batch, feather, table_name,rule_id+1-repeat_times+i, 0, label_id, keywords);
}
Maat_command_batch_commit(batch);
usleep(WAIT_FOR_EFFECTIVE_US);//waiting for commands go into effect
memset(&result, 0, sizeof(result));
ret=Maat_full_scan_string(feather, table_id,CHARSET_GBK, scan_data, strlen(scan_data),
@@ -2395,7 +2418,7 @@ TEST_F(MaatCmdTest, SubGroup)
//group1->compile1
memset(&group1, 0, sizeof(group1));
group1.group_id=Maat_cmd_get_new_group_id(feather);
group1.group_id=Maat_command_get_new_group_id(feather);
group1.table_name=g2c_tn;
group1.compile_id=compile1.config_id;
group1.clause_index=1;
@@ -2412,7 +2435,7 @@ TEST_F(MaatCmdTest, SubGroup)
\__compile2
*/
memset(&group2, 0, sizeof(group2));
group2.group_id=Maat_cmd_get_new_group_id(feather);
group2.group_id=Maat_command_get_new_group_id(feather);
group2.table_name=g2g_tn;
group2.superior_group_id=group1.group_id;
@@ -2424,13 +2447,13 @@ TEST_F(MaatCmdTest, SubGroup)
\_compile2
*/
memset(&region1, 0, sizeof(region1));
region1.region_id=Maat_cmd_get_new_region_id(feather);
region1.region_id=Maat_command_get_new_region_id(feather);
region1.region_type=REGION_EXPR;
region1.table_name=table_name;
region1.expr_rule.keywords=keyword1;
region1.expr_rule.expr_type=EXPR_TYPE_AND;
Maat_command_raw_set_region(feather, MAAT_OP_ADD, &region1, group2.group_id);
sleep(1);
sleep(4);
struct Maat_rule_t result[4];
memset(&result, 0, sizeof(result));
@@ -2491,13 +2514,13 @@ TEST_F(MaatCmdTest, SubGroup)
*/
memset(&group3, 0, sizeof(group3));
group3.group_id=Maat_cmd_get_new_group_id(feather);
group3.group_id=Maat_command_get_new_group_id(feather);
group3.superior_group_id=group1.group_id;
group3.table_name=g2g_tn;
Maat_command_raw_set_group2group(feather, MAAT_OP_ADD, &group3);
memset(&region2, 0, sizeof(region2));
region2.region_id=Maat_cmd_get_new_region_id(feather);
region2.region_id=Maat_command_get_new_region_id(feather);
region2.region_type=REGION_EXPR;
region2.table_name=table_name;
region2.expr_rule.keywords=keyword2;
@@ -2562,7 +2585,7 @@ TEST_F(MaatCmdTest, RefGroup)
//group1->compile1
memset(&group1, 0, sizeof(group1));
group1.group_id=Maat_cmd_get_new_group_id(feather);
group1.group_id=Maat_command_get_new_group_id(feather);
group1.table_name=g2c_tn;
group1.compile_id=compile1.config_id;
Maat_command_raw_set_group2compile(feather, MAAT_OP_ADD, &group1);
@@ -2571,7 +2594,7 @@ TEST_F(MaatCmdTest, RefGroup)
//region1->group1->compile1
memset(&region1, 0, sizeof(region1));
region1.region_id=Maat_cmd_get_new_region_id(feather);
region1.region_id=Maat_command_get_new_region_id(feather);
region1.region_type=REGION_EXPR;
region1.table_name=table_name;
region1.expr_rule.keywords=keyword1;
@@ -2589,12 +2612,12 @@ TEST_F(MaatCmdTest, RefGroup)
// /
// region2->group2
memset(&group2, 0, sizeof(group2));
group2.group_id=Maat_cmd_get_new_group_id(feather);
group2.group_id=Maat_command_get_new_group_id(feather);
group2.table_name=g2c_tn;
group2.compile_id=compile1.config_id;
memset(&region2, 0, sizeof(region2));
region2.region_id=Maat_cmd_get_new_region_id(feather);
region2.region_id=Maat_command_get_new_region_id(feather);
region2.region_type=REGION_EXPR;
region2.table_name=table_name;
region2.expr_rule.keywords=keyword2;
@@ -2644,7 +2667,7 @@ TEST_F(MaatCmdTest, VirtualTable)
//group1->compile1
memset(&group1, 0, sizeof(group1));
group1.group_id=Maat_cmd_get_new_group_id(feather);
group1.group_id=Maat_command_get_new_group_id(feather);
group1.table_name=g2c_tn;
group1.virtual_table_name="HTTP_REQUEST_HEADER";
group1.compile_id=compile1.config_id;
@@ -2654,7 +2677,7 @@ TEST_F(MaatCmdTest, VirtualTable)
//region1->group1->compile1
memset(&region1, 0, sizeof(region1));
region1.region_id=Maat_cmd_get_new_region_id(feather);
region1.region_id=Maat_command_get_new_region_id(feather);
region1.region_type=REGION_EXPR;
region1.table_name=region_table_name;
region1.expr_rule.district="User-Agent";
@@ -2666,7 +2689,7 @@ TEST_F(MaatCmdTest, VirtualTable)
// /
// group2-/
memset(&group2, 0, sizeof(group2));
group2.group_id=Maat_cmd_get_new_group_id(feather);
group2.group_id=Maat_command_get_new_group_id(feather);
group2.table_name=g2c_tn;
group2.virtual_table_name="HTTP_RESPONSE_HEADER";
group2.compile_id=compile1.config_id;
@@ -2679,7 +2702,7 @@ TEST_F(MaatCmdTest, VirtualTable)
// region2->group2-/
memset(&region2, 0, sizeof(region2));
region2.region_id=Maat_cmd_get_new_region_id(feather);
region2.region_id=Maat_command_get_new_region_id(feather);
region2.region_type=REGION_EXPR;
region2.table_name=region_table_name;
region2.expr_rule.district="Cookie";
@@ -3087,7 +3110,7 @@ TEST_F(MaatCmdTest, HitPath)
//group1->compile1
memset(&group1, 0, sizeof(group1));
group1.group_id=Maat_cmd_get_new_group_id(feather);
group1.group_id=Maat_command_get_new_group_id(feather);
group1.table_name=g2c_tn;
group1.virtual_table_name="HTTP_REQUEST_HEADER";
group1.compile_id=compile1.config_id;
@@ -3098,7 +3121,7 @@ TEST_F(MaatCmdTest, HitPath)
//region1->group1->compile1
memset(&region1, 0, sizeof(region1));
region1.region_id=Maat_cmd_get_new_region_id(feather);
region1.region_id=Maat_command_get_new_region_id(feather);
region1.region_type=REGION_EXPR;
region1.table_name=table_http_sig;
region1.expr_rule.district="URL";
@@ -3112,7 +3135,7 @@ TEST_F(MaatCmdTest, HitPath)
// group21/
memset(&group21, 0, sizeof(group21));
group21.group_id=Maat_cmd_get_new_group_id(feather);
group21.group_id=Maat_command_get_new_group_id(feather);
group21.table_name=g2c_tn;
group21.virtual_table_name="HTTP_RESPONSE_HEADER";
group21.compile_id=compile1.config_id;
@@ -3125,7 +3148,7 @@ TEST_F(MaatCmdTest, HitPath)
// group2->group21/
memset(&group2, 0, sizeof(group2));
group2.group_id=Maat_cmd_get_new_group_id(feather);
group2.group_id=Maat_command_get_new_group_id(feather);
group2.table_name=g2g_tn;
group2.superior_group_id=group21.group_id;
Maat_command_raw_set_group2group(feather, MAAT_OP_ADD, &group2);
@@ -3137,7 +3160,7 @@ TEST_F(MaatCmdTest, HitPath)
//region2->group2->group21/
memset(&region2, 0, sizeof(region2));
region2.region_id=Maat_cmd_get_new_region_id(feather);
region2.region_id=Maat_command_get_new_region_id(feather);
region2.region_type=REGION_EXPR;
region2.table_name=table_http_sig;
region2.expr_rule.district="Cookie";
@@ -3147,7 +3170,7 @@ TEST_F(MaatCmdTest, HitPath)
//region3->group3, group3 is not referenced by any compile.
memset(&region3, 0, sizeof(region3));
region3.region_id=Maat_cmd_get_new_region_id(feather);
region3.region_id=Maat_command_get_new_region_id(feather);
region3.region_type=REGION_IP;
region3.table_name=table_ip;
region3.ip_rule.addr_type=ADDR_TYPE_IPv4;
@@ -3163,20 +3186,20 @@ TEST_F(MaatCmdTest, HitPath)
region3.ip_rule.mask_dst_port=65535;
region3.ip_rule.protocol=0;//means any protocol should hit.
group3.group_id=Maat_cmd_get_new_group_id(feather);
group3.group_id=Maat_command_get_new_group_id(feather);
Maat_command_raw_set_region(feather, MAAT_OP_ADD, &region3, group3.group_id);
char temp[1024]={0};
//region4->group4, group4 is not referenced by any compile.
memset(&region4, 0, sizeof(region4));
region4.region_id=Maat_cmd_get_new_region_id(feather);
region4.region_id=Maat_command_get_new_region_id(feather);
region4.region_type=REGION_EXPR;
region4.table_name=table_keywords;
region4.expr_rule.district=NULL;
region4.expr_rule.keywords= Maat_str_escape(temp, sizeof(temp), "a finite or infinite");
region4.expr_rule.expr_type=EXPR_TYPE_STRING;
group4.group_id=Maat_cmd_get_new_group_id(feather);
group4.group_id=Maat_command_get_new_group_id(feather);
Maat_command_raw_set_region(feather, MAAT_OP_ADD, &region4, group4.group_id);