gfwleak/tango-maat
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

interrupt execution if table schema has error

Browse Source
This commit is contained in:
liuwentan
2023-03-23 19:16:23 +08:00
parent 2ce749d9bc
commit 7b49d7d52f
9 changed files with 149 additions and 186 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
include/maat.h
View File

@@ -9,7 +9,7 @@
* Authors: Liu WenTan <liuwentan@geedgenetworks.com> * Authors: Liu WenTan <liuwentan@geedgenetworks.com>
* Date: 2022-10-31 * Date: 2022-10-31
* Copyright: (c) 2018-2022 Geedge Networks, Inc. All rights reserved. * Copyright: (c) 2018-2023 Geedge Networks, Inc. All rights reserved.
*********************************************************************************************** ***********************************************************************************************
*/ */
@@ -67,6 +67,7 @@ struct ipv4_tuple {
unsigned int dip; /* network order */ unsigned int dip; /* network order */
unsigned short sport; /* network order */ unsigned short sport; /* network order */
unsigned short dport; /* network order */ unsigned short dport; /* network order */
int protocol;
}; };
struct ipv6_tuple { struct ipv6_tuple {
@@ -74,6 +75,7 @@ struct ipv6_tuple {
unsigned int dip[4] ; /* network order */ unsigned int dip[4] ; /* network order */
unsigned short sport; /* network order */ unsigned short sport; /* network order */
unsigned short dport; /* network order */ unsigned short dport; /* network order */
int protocol;
}; };
/* update_type: MAAT_UPDATE_TYPE_FULL or MAAT_UPDATE_TYPE_INC */ /* update_type: MAAT_UPDATE_TYPE_FULL or MAAT_UPDATE_TYPE_INC */

22
scanner/adapter_hs/adapter_hs.cpp
View File

@@ -804,18 +804,18 @@ int adapter_hs_scan_stream(struct adapter_hs_stream *hs_stream, const char *data
return -1; return -1;
} }
size_t n_item = HASH_COUNT(hs_stream->matched_pat_container.pat_hash); size_t n_pattern_id = HASH_COUNT(hs_stream->matched_pat_container.pat_hash);
if (0 == n_item) { if (0 == n_pattern_id) {
*n_hit_result = 0; *n_hit_result = 0;
return 0; return 0;
} }
if (n_item > MAX_SCANNER_HIT_PATTERN_NUM) { if (n_pattern_id > MAX_SCANNER_HIT_PATTERN_NUM) {
n_item = MAX_SCANNER_HIT_PATTERN_NUM; n_pattern_id = MAX_SCANNER_HIT_PATTERN_NUM;
} }
unsigned long long item_ids[MAX_SCANNER_HIT_PATTERN_NUM]; unsigned long long pattern_ids[MAX_SCANNER_HIT_PATTERN_NUM];
memset(item_ids, 0, sizeof(unsigned long long) * MAX_SCANNER_HIT_PATTERN_NUM); memset(pattern_ids, 0, sizeof(unsigned long long) * MAX_SCANNER_HIT_PATTERN_NUM);
int i = 0; int i = 0;
struct matched_pattern *pat = NULL, *tmp_pat = NULL; struct matched_pattern *pat = NULL, *tmp_pat = NULL;
@@ -823,15 +823,15 @@ int adapter_hs_scan_stream(struct adapter_hs_stream *hs_stream, const char *data
if (i >= MAX_SCANNER_HIT_PATTERN_NUM) { if (i >= MAX_SCANNER_HIT_PATTERN_NUM) {
break; break;
} }
item_ids[i++] = pat->pattern_id; pattern_ids[i++] = pat->pattern_id;
} }
qsort(item_ids, n_item, sizeof(unsigned long long), cmp_ull_p); qsort(pattern_ids, n_pattern_id, sizeof(unsigned long long), cmp_ull_p);
int ret = 0; int ret = 0;
int real_matched_index = 0; int real_matched_index = 0;
struct hs_tag *hs_tag = NULL; struct hs_tag *hs_tag = NULL;
struct bool_expr_match *bool_matcher_results = ALLOC(struct bool_expr_match, hs_stream->n_expr); struct bool_expr_match *bool_matcher_results = ALLOC(struct bool_expr_match, hs_stream->n_expr);
int bool_matcher_ret = bool_matcher_match(hs_stream->hs_rt->bm, item_ids, n_item, int bool_matcher_ret = bool_matcher_match(hs_stream->hs_rt->bm, pattern_ids, n_pattern_id,
bool_matcher_results, hs_stream->n_expr); bool_matcher_results, hs_stream->n_expr);
if (bool_matcher_ret < 0) { if (bool_matcher_ret < 0) {
ret = -1; ret = -1;
@@ -879,6 +879,10 @@ int adapter_hs_scan(struct adapter_hs *hs_instance, int thread_id,
} }
struct adapter_hs_stream *hs_stream = adapter_hs_stream_open(hs_instance, thread_id); struct adapter_hs_stream *hs_stream = adapter_hs_stream_open(hs_instance, thread_id);
if (NULL == hs_stream) {
return -1;
}
int ret = adapter_hs_scan_stream(hs_stream, data, data_len, results, n_result, n_hit_result); int ret = adapter_hs_scan_stream(hs_stream, data, data_len, results, n_result, n_hit_result);
adapter_hs_stream_close(hs_stream); adapter_hs_stream_close(hs_stream);

1
src/inc_internal/maat_rule.h
View File

@@ -151,6 +151,7 @@ struct serial_rule {
char *table_line; char *table_line;
int n_foreign; int n_foreign;
struct foreign_key *f_keys; struct foreign_key *f_keys;
redisContext *ref_ctx;
TAILQ_ENTRY(serial_rule) entries; TAILQ_ENTRY(serial_rule) entries;
UT_hash_handle hh; UT_hash_handle hh;
}; };

13
src/maat_api.c
View File

@@ -957,8 +957,7 @@ int flag_scan(struct table_manager *tbl_mgr, int thread_id, long long flag,
int physical_table_id, int vtable_id, struct maat_state *state) int physical_table_id, int vtable_id, struct maat_state *state)
{ {
enum table_type table_type = table_manager_get_table_type(tbl_mgr, physical_table_id); enum table_type table_type = table_manager_get_table_type(tbl_mgr, physical_table_id);
if ((table_type == TABLE_TYPE_FLAG_PLUS) && if (table_type == TABLE_TYPE_FLAG_PLUS && DISTRICT_FLAG_UNSET == state->is_set_district) {
(NULL == state || DISTRICT_FLAG_UNSET == state->is_set_district)) {
return -1; return -1;
} }
@@ -989,8 +988,7 @@ int interval_scan(struct table_manager *tbl_mgr, int thread_id, long long intege
{ {
enum table_type table_type = table_manager_get_table_type(tbl_mgr, physical_table_id); enum table_type table_type = table_manager_get_table_type(tbl_mgr, physical_table_id);
if ((table_type == TABLE_TYPE_INTERVAL_PLUS) && if (table_type == TABLE_TYPE_INTERVAL_PLUS && DISTRICT_FLAG_UNSET == state->is_set_district) {
(NULL == state || DISTRICT_FLAG_UNSET == state->is_set_district)) {
// maat_instance->scan_err_cnt++; // maat_instance->scan_err_cnt++;
return -1; return -1;
} }
@@ -1075,8 +1073,7 @@ int string_scan(struct table_manager *tbl_mgr, int thread_id, const char *data,
int physical_table_id, int vtable_id, struct maat_state *state) int physical_table_id, int vtable_id, struct maat_state *state)
{ {
enum table_type table_type = table_manager_get_table_type(tbl_mgr, physical_table_id); enum table_type table_type = table_manager_get_table_type(tbl_mgr, physical_table_id);
if ((table_type == TABLE_TYPE_EXPR_PLUS) && if (table_type == TABLE_TYPE_EXPR_PLUS && DISTRICT_FLAG_UNSET == state->is_set_district) {
(NULL == state || DISTRICT_FLAG_UNSET == state->is_set_district)) {
// maat_instance->scan_err_cnt++; // maat_instance->scan_err_cnt++;
return -1; return -1;
} }
@@ -1112,8 +1109,7 @@ int expr_stream_scan(struct maat_stream *stream, const char *data, size_t data_l
struct table_manager *tbl_mgr = stream->ref_maat_instance->tbl_mgr; struct table_manager *tbl_mgr = stream->ref_maat_instance->tbl_mgr;
enum table_type table_type = table_manager_get_table_type(tbl_mgr, stream->physical_table_id); enum table_type table_type = table_manager_get_table_type(tbl_mgr, stream->physical_table_id);
if ((table_type == TABLE_TYPE_EXPR_PLUS) && if (table_type == TABLE_TYPE_EXPR_PLUS && DISTRICT_FLAG_UNSET == state->is_set_district) {
(NULL == state || DISTRICT_FLAG_UNSET == state->is_set_district)) {
// maat_instance->scan_err_cnt++; // maat_instance->scan_err_cnt++;
return -1; return -1;
} }
@@ -1698,7 +1694,6 @@ void maat_state_free(struct maat_state *state)
assert(state->compile_state != NULL); assert(state->compile_state != NULL);
maat_compile_state_free(state->compile_state); maat_compile_state_free(state->compile_state);
state->compile_state = NULL; state->compile_state = NULL;
//alignment_int64_array_add(mid->maat_instance->compile_state_cnt, mid->thread_id, -1);
state->maat_instance = NULL; state->maat_instance = NULL;
free(state); free(state);
} }

66
src/maat_table.c
View File

@@ -490,12 +490,7 @@ table_manager_create(const char *table_info_path, const char *accept_tags,
"[%s:%d] register_tablename2id failed", __FUNCTION__, __LINE__); "[%s:%d] register_tablename2id failed", __FUNCTION__, __LINE__);
FREE(json_buff); FREE(json_buff);
cJSON_Delete(root); cJSON_Delete(root);
maat_kv_store_free(tbl_mgr->tablename2id_map); table_manager_destroy(tbl_mgr);
for (int idx = 0; idx < tbl_mgr->n_accept_tag; idx++) {
FREE(tbl_mgr->accept_tags[idx].tag_name);
FREE(tbl_mgr->accept_tags[idx].tag_val);
}
FREE(tbl_mgr);
return NULL; return NULL;
} }
} }
@@ -512,7 +507,8 @@ table_manager_create(const char *table_info_path, const char *accept_tags,
if (json != NULL && json->type == cJSON_Object) { if (json != NULL && json->type == cJSON_Object) {
struct maat_table *maat_tbl = maat_table_new(json, reserved_word_map, logger); struct maat_table *maat_tbl = maat_table_new(json, reserved_word_map, logger);
if (NULL == maat_tbl) { if (NULL == maat_tbl) {
continue; ret = -1;
goto next;
} }
maat_tbl->schema = maat_table_schema_new(json, maat_tbl->table_name, maat_tbl->schema = maat_table_schema_new(json, maat_tbl->table_name,
@@ -521,8 +517,8 @@ table_manager_create(const char *table_info_path, const char *accept_tags,
log_error(logger, MODULE_TABLE, log_error(logger, MODULE_TABLE,
"[%s:%d] Maat table schema new failed, table_name:%s", "[%s:%d] Maat table schema new failed, table_name:%s",
__FUNCTION__, __LINE__, maat_tbl->table_name); __FUNCTION__, __LINE__, maat_tbl->table_name);
maat_table_free(maat_tbl); ret = -1;
continue; goto next;
} }
log_info(logger, MODULE_TABLE, "successfully register table[%s]->table_id:%d", log_info(logger, MODULE_TABLE, "successfully register table[%s]->table_id:%d",
maat_tbl->table_name, maat_tbl->table_id); maat_tbl->table_name, maat_tbl->table_id);
@@ -547,11 +543,16 @@ table_manager_create(const char *table_info_path, const char *accept_tags,
log_info(logger, MODULE_TABLE, "default compile table id: %d", default_compile_table_id); log_info(logger, MODULE_TABLE, "default compile table id: %d", default_compile_table_id);
log_info(logger, MODULE_TABLE, "group2group table id: %d", g2g_table_id); log_info(logger, MODULE_TABLE, "group2group table id: %d", g2g_table_id);
next:
FREE(json_buff); FREE(json_buff);
maat_kv_store_free(reserved_word_map); maat_kv_store_free(reserved_word_map);
cJSON_Delete(root); cJSON_Delete(root);
if (ret < 0) {
table_manager_destroy(tbl_mgr);
return NULL;
}
return tbl_mgr; return tbl_mgr;
} }
@@ -628,13 +629,12 @@ void table_manager_runtime_destroy(struct table_manager *tbl_mgr)
for(size_t i = 0; i < MAX_TABLE_NUM; i++) { for(size_t i = 0; i < MAX_TABLE_NUM; i++) {
void *runtime = table_manager_get_runtime(tbl_mgr, i); void *runtime = table_manager_get_runtime(tbl_mgr, i);
if (NULL == runtime) { if (runtime != NULL) {
continue; enum table_type table_type = table_manager_get_table_type(tbl_mgr, i);
assert(table_type != TABLE_TYPE_INVALID);
maat_table_runtime_free(runtime, table_type);
tbl_mgr->tbl[i]->runtime = NULL;
} }
enum table_type table_type = table_manager_get_table_type(tbl_mgr, i);
maat_table_runtime_free(runtime, table_type);
tbl_mgr->tbl[i]->runtime = NULL;
} }
/* free district map */ /* free district map */
@@ -647,30 +647,28 @@ void table_manager_destroy(struct table_manager *tbl_mgr)
return; return;
} }
for (size_t i = 0; i < MAX_TABLE_NUM; i++) { size_t i = 0;
void *runtime = table_manager_get_runtime(tbl_mgr, i);
assert(NULL == runtime);
void *schema = table_manager_get_schema(tbl_mgr, i);
if (NULL == schema) {
continue;
}
enum table_type table_type = table_manager_get_table_type(tbl_mgr, i);
maat_table_schema_free(schema, table_type);
tbl_mgr->tbl[i]->schema = NULL;
for (i = 0; i < MAX_TABLE_NUM; i++) {
maat_table_free(tbl_mgr->tbl[i]); maat_table_free(tbl_mgr->tbl[i]);
tbl_mgr->tbl[i] = NULL; tbl_mgr->tbl[i] = NULL;
} }
for (size_t i = 0; i < tbl_mgr->n_accept_tag; i++) { if (tbl_mgr->accept_tags != NULL) {
FREE(tbl_mgr->accept_tags[i].tag_name); for (i = 0; i < tbl_mgr->n_accept_tag; i++) {
FREE(tbl_mgr->accept_tags[i].tag_val); if (tbl_mgr->accept_tags[i].tag_name != NULL) {
} FREE(tbl_mgr->accept_tags[i].tag_name);
FREE(tbl_mgr->accept_tags); }
if (tbl_mgr->accept_tags[i].tag_val != NULL) {
FREE(tbl_mgr->accept_tags[i].tag_val);
}
}
FREE(tbl_mgr->accept_tags);
}
maat_kv_store_free(tbl_mgr->tablename2id_map); maat_kv_store_free(tbl_mgr->tablename2id_map);
tbl_mgr->tablename2id_map = NULL;
FREE(tbl_mgr); FREE(tbl_mgr);
} }

79
test/maat_framework_gtest.cpp
View File

@@ -49,35 +49,28 @@ long long absolute_expire_time=0;
int make_serial_rule(const char *table_name, const char *line, void *u_para) int make_serial_rule(const char *table_name, const char *line, void *u_para)
{ {
struct serial_rule *s_rule=(struct serial_rule *)u_para; struct serial_rule *s_rule=(struct serial_rule *)u_para;
int rule_id = 0; redisContext *ctx = s_rule->ref_ctx;
char *buff = ALLOC(char, strlen(line) + 1); char *buff = ALLOC(char, strlen(line) + 1);
memcpy(buff, line, strlen(line) + 1); memcpy(buff, line, strlen(line) + 1);
while(buff[strlen(line) - 1] == '\n' || buff[strlen(line) - 1] == '\t') {
while (buff[strlen(buff) - 1] == '\n' || buff[strlen(buff) - 1] == '\t') { buff[strlen(line) - 1] = '\0';
buff[strlen(buff) - 1] = '\0';
} }
int j = 0; const char *redis_rule_key = "TEST_RULE_KEY";
char *str1 = NULL; redisReply *reply = maat_cmd_wrap_redis_command(ctx, "INCRBY %s %d", redis_rule_key, 1);
char *token = NULL; if (reply->type == REDIS_REPLY_NIL) {
char *saveptr1 = NULL; printf("incrby redis_rule_key:%s failed.", redis_rule_key);
return -1;
for (j = 0,str1 = buff; ; j++, str1 = NULL) { } else {
token = strtok_r(str1, "\t ", &saveptr1); s_rule->rule_id = maat_cmd_read_redis_integer(reply);
if (token == NULL) freeReplyObject(reply);
break; reply = NULL;
if (j == 0) {
sscanf(token,"%d", &rule_id);
}
}
memcpy(buff, line, strlen(line)+1);
while(buff[strlen(buff)-1]=='\n'||buff[strlen(buff)-1]=='\t') {
buff[strlen(buff)-1]='\0';
} }
maat_cmd_set_serial_rule(s_rule + line_idx, MAAT_OP_ADD, rule_id, table_name, buff, absolute_expire_time); maat_cmd_set_serial_rule(s_rule + line_idx, MAAT_OP_ADD, s_rule->rule_id, table_name,
buff, absolute_expire_time);
(s_rule + line_idx)->ref_ctx = ctx;
line_idx++; line_idx++;
FREE(buff); FREE(buff);
@@ -130,6 +123,7 @@ int write_config_to_redis(char *redis_ip, int redis_port, int redis_db,
config_monitor_traverse(0, tmp_iris_full_idx_path, NULL, count_line_num_cb, NULL, &total_line_cnt, logger); config_monitor_traverse(0, tmp_iris_full_idx_path, NULL, count_line_num_cb, NULL, &total_line_cnt, logger);
struct serial_rule *s_rule = ALLOC(struct serial_rule, total_line_cnt); struct serial_rule *s_rule = ALLOC(struct serial_rule, total_line_cnt);
s_rule->ref_ctx = c;
long long server_time = maat_cmd_redis_server_time_s(c); long long server_time = maat_cmd_redis_server_time_s(c);
if (server_time < 0) { if (server_time < 0) {
return -1; return -1;
@@ -137,6 +131,7 @@ int write_config_to_redis(char *redis_ip, int redis_port, int redis_db,
absolute_expire_time = server_time + 300; absolute_expire_time = server_time + 300;
config_monitor_traverse(0, tmp_iris_full_idx_path, NULL, make_serial_rule, NULL, s_rule, logger); config_monitor_traverse(0, tmp_iris_full_idx_path, NULL, make_serial_rule, NULL, s_rule, logger);
s_rule->ref_ctx = NULL;
line_idx = 0; line_idx = 0;
absolute_expire_time = 0; absolute_expire_time = 0;
@@ -722,12 +717,13 @@ TEST_F(MaatStringScan, ExprPlusWithOffset)
maat_state_free(state); maat_state_free(state);
state = NULL; state = NULL;
} }
#if 0
TEST_F(MaatStringScan, ExprPlusWithHex) { TEST_F(MaatStringScan, ExprPlusWithHex) {
long long results[ARRAY_SIZE] = {0}; long long results[ARRAY_SIZE] = {0};
size_t n_hit_result = 0; size_t n_hit_result = 0;
struct maat_state *state = NULL; int thread_id = 0;
struct maat *maat_instance = MaatStringScan::_shared_maat_instance; struct maat *maat_instance = MaatStringScan::_shared_maat_instance;
struct maat_state *state = maat_state_new(maat_instance, thread_id);
const char *scan_data1 = "text/html; charset=UTF-8"; const char *scan_data1 = "text/html; charset=UTF-8";
const char *scan_data2 = "Batman\\:Take me Home.Superman/:Fine,stay with me."; const char *scan_data2 = "Batman\\:Take me Home.Superman/:Fine,stay with me.";
const char *region_name1 = "Content-Type"; const char *region_name1 = "Content-Type";
@@ -736,27 +732,28 @@ TEST_F(MaatStringScan, ExprPlusWithHex) {
int table_id = maat_get_table_id(maat_instance, "HTTP_SIGNATURE"); int table_id = maat_get_table_id(maat_instance, "HTTP_SIGNATURE");
ASSERT_GT(table_id, 0); ASSERT_GT(table_id, 0);
int ret = maat_state_set_scan_district(maat_instance, &state, region_name1, strlen(region_name1)); int ret = maat_state_set_scan_district(state, region_name1, strlen(region_name1));
ASSERT_EQ(ret, 0); ASSERT_EQ(ret, 0);
ret = maat_scan_string(maat_instance, table_id, 0, scan_data1, strlen(scan_data1), ret = maat_scan_string(maat_instance, table_id, thread_id, scan_data1, strlen(scan_data1),
results, ARRAY_SIZE, &n_hit_result, &state); results, ARRAY_SIZE, &n_hit_result, state);
EXPECT_EQ(ret, MAAT_SCAN_HIT); EXPECT_EQ(ret, MAAT_SCAN_HIT);
EXPECT_EQ(results[0], 156); EXPECT_EQ(results[0], 156);
ret = maat_state_set_scan_district(maat_instance, &state, region_name2, strlen(region_name2)); ret = maat_state_set_scan_district(state, region_name2, strlen(region_name2));
ASSERT_EQ(ret, 0); ASSERT_EQ(ret, 0);
ret = maat_scan_string(maat_instance, table_id, 0, scan_data1, strlen(scan_data1), ret = maat_scan_string(maat_instance, table_id, thread_id, scan_data1, strlen(scan_data1),
results, ARRAY_SIZE, &n_hit_result, &state); results, ARRAY_SIZE, &n_hit_result, state);
EXPECT_EQ(ret, MAAT_SCAN_HALF_HIT); EXPECT_EQ(ret, MAAT_SCAN_OK); //maat-v3 consider as half hit, it's unreasonable
table_id = maat_get_table_id(maat_instance, "KEYWORDS_TABLE"); table_id = maat_get_table_id(maat_instance, "KEYWORDS_TABLE");
ret = maat_scan_string(maat_instance, table_id, 0, scan_data2, strlen(scan_data2), ret = maat_scan_string(maat_instance, table_id, thread_id, scan_data2, strlen(scan_data2),
results, ARRAY_SIZE, &n_hit_result, &state); results, ARRAY_SIZE, &n_hit_result, state);
EXPECT_EQ(ret, MAAT_SCAN_HIT); EXPECT_EQ(ret, MAAT_SCAN_HIT);
EXPECT_EQ(results[0], 132); EXPECT_EQ(results[0], 132);
maat_state_free(&state); maat_state_free(state);
state = NULL;
} }
#endif
TEST_F(MaatStringScan, ExprAndExprPlus) { TEST_F(MaatStringScan, ExprAndExprPlus) {
long long results[ARRAY_SIZE] = {0}; long long results[ARRAY_SIZE] = {0};
size_t n_hit_result = 0; size_t n_hit_result = 0;
@@ -788,7 +785,7 @@ TEST_F(MaatStringScan, ExprAndExprPlus) {
maat_state_free(state); maat_state_free(state);
state = NULL; state = NULL;
} }
#if 0
TEST_F(MaatStringScan, ShouldNotHitExprPlus) { TEST_F(MaatStringScan, ShouldNotHitExprPlus) {
long long results[ARRAY_SIZE] = {0}; long long results[ARRAY_SIZE] = {0};
size_t n_hit_result = 0; size_t n_hit_result = 0;
@@ -818,11 +815,11 @@ TEST_F(MaatStringScan, ShouldNotHitExprPlus) {
ret = maat_scan_string(maat_instance, table_id, thread_id, (char *)udp_payload_not_hit, sizeof(udp_payload_not_hit), ret = maat_scan_string(maat_instance, table_id, thread_id, (char *)udp_payload_not_hit, sizeof(udp_payload_not_hit),
results, ARRAY_SIZE, &n_hit_result, state); results, ARRAY_SIZE, &n_hit_result, state);
EXPECT_EQ(ret, MAAT_SCAN_HALF_HIT); EXPECT_EQ(ret, MAAT_SCAN_OK); //maat-v3 consider as half hit, it's unreasonable
maat_state_free(state); maat_state_free(state);
state = NULL; state = NULL;
} }
#endif
TEST_F(MaatStringScan, Expr8) { TEST_F(MaatStringScan, Expr8) {
const char *table_name = "KEYWORDS_TABLE"; const char *table_name = "KEYWORDS_TABLE";
int thread_id = 0; int thread_id = 0;
@@ -957,7 +954,7 @@ TEST_F(MaatStringScan, PrefixAndSuffix) {
state = NULL; state = NULL;
} }
#endif #endif
#if 0
TEST_F(MaatStringScan, MaatUnescape) { TEST_F(MaatStringScan, MaatUnescape) {
const char *scan_data = "Batman\\:Take me Home.Superman/:Fine,stay with me."; const char *scan_data = "Batman\\:Take me Home.Superman/:Fine,stay with me.";
const char *table_name = "KEYWORDS_TABLE"; const char *table_name = "KEYWORDS_TABLE";
@@ -978,7 +975,7 @@ TEST_F(MaatStringScan, MaatUnescape) {
maat_state_free(state); maat_state_free(state);
state = NULL; state = NULL;
} }
#endif
#if 0 #if 0
TEST_F(MaatStringScan, RegexWithNotContains) { TEST_F(MaatStringScan, RegexWithNotContains) {
const char *should_NOT_hit_scan_data = "new.qq.com/rain/a/TWF2021042600418000"; const char *should_NOT_hit_scan_data = "new.qq.com/rain/a/TWF2021042600418000";

46
test/maat_input_mode_gtest.cpp
View File

@@ -7,6 +7,8 @@
#include <gtest/gtest.h> #include <gtest/gtest.h>
#define MODULE_INPUT_MODE_GTEST module_name_str("maat.input_mode_gtest")
const char *table_info_path = "./table_info.conf"; const char *table_info_path = "./table_info.conf";
const char *json_filename = "maat_json.json"; const char *json_filename = "maat_json.json";
struct log_handle *g_logger = NULL; struct log_handle *g_logger = NULL;
@@ -124,39 +126,31 @@ long long absolute_expire_time=0;
int make_serial_rule(const char *table_name, const char *line, void *u_para) int make_serial_rule(const char *table_name, const char *line, void *u_para)
{ {
struct serial_rule *s_rule=(struct serial_rule *)u_para; struct serial_rule *s_rule=(struct serial_rule *)u_para;
int rule_id = 0; redisContext *ctx = s_rule->ref_ctx;
char *buff = ALLOC(char, strlen(line) + 1); char *buff = ALLOC(char, strlen(line) + 1);
memcpy(buff, line, strlen(line) + 1); memcpy(buff, line, strlen(line) + 1);
while(buff[strlen(line) - 1] == '\n' || buff[strlen(line) - 1] == '\t') {
while (buff[strlen(buff) - 1] == '\n' || buff[strlen(buff) - 1] == '\t') { buff[strlen(line) - 1] = '\0';
buff[strlen(buff) - 1] = '\0';
} }
int j = 0; const char *redis_rule_key = "TEST_RULE_KEY";
char *str1 = NULL; redisReply *reply = maat_cmd_wrap_redis_command(ctx, "INCRBY %s %d", redis_rule_key, 1);
char *token = NULL; if (reply->type == REDIS_REPLY_NIL) {
char *saveptr1 = NULL; printf("incrby redis_rule_key:%s failed.", redis_rule_key);
return -1;
for (j = 0,str1 = buff; ; j++, str1 = NULL) { } else {
token = strtok_r(str1, "\t ", &saveptr1); s_rule->rule_id = maat_cmd_read_redis_integer(reply);
if (token == NULL) freeReplyObject(reply);
break; reply = NULL;
if (j == 0) {
sscanf(token,"%d", &rule_id);
}
}
memcpy(buff, line, strlen(line)+1);
while(buff[strlen(buff)-1]=='\n'||buff[strlen(buff)-1]=='\t') {
buff[strlen(buff)-1]='\0';
} }
maat_cmd_set_serial_rule(s_rule + line_idx, MAAT_OP_ADD, rule_id, table_name, maat_cmd_set_serial_rule(s_rule + line_idx, MAAT_OP_ADD, s_rule->rule_id, table_name,
buff, absolute_expire_time); buff, absolute_expire_time);
(s_rule + line_idx)->ref_ctx = ctx;
line_idx++; line_idx++;
FREE(buff); FREE(buff);
return 0; return 0;
} }
@@ -196,12 +190,14 @@ TEST(redis_mode, maat_scan_string) {
&total_line_cnt, g_logger); &total_line_cnt, g_logger);
struct serial_rule *s_rule = ALLOC(struct serial_rule, total_line_cnt); struct serial_rule *s_rule = ALLOC(struct serial_rule, total_line_cnt);
s_rule->ref_ctx = c;
long long server_time = maat_cmd_redis_server_time_s(c); long long server_time = maat_cmd_redis_server_time_s(c);
EXPECT_NE(server_time, -1); EXPECT_NE(server_time, -1);
absolute_expire_time = server_time + 300; absolute_expire_time = server_time + 300;
config_monitor_traverse(0, tmp_iris_full_idx_path, NULL, make_serial_rule, NULL, config_monitor_traverse(0, tmp_iris_full_idx_path, NULL, make_serial_rule, NULL,
s_rule, g_logger); s_rule, g_logger);
s_rule->ref_ctx = NULL;
int success_cnt = 0; int success_cnt = 0;
do { do {
success_cnt = maat_cmd_write_rule(c, s_rule, total_line_cnt, server_time, g_logger); success_cnt = maat_cmd_write_rule(c, s_rule, total_line_cnt, server_time, g_logger);

37
test/table_info.conf
View File

@@ -450,43 +450,6 @@
}, },
{ {
"table_id":40, "table_id":40,
"table_name":"PORT_CONFIG",
"table_type":"port",
"valid_column":7,
"custom": {
"item_id":1,
"group_id":2,
"port_format":3,
"port1":4,
"port2":5,
"proto":6
}
},
{
"table_id":41,
"table_name":"VIRTUAL_PORT_SOURCE",
"table_type":"virtual",
"physical_table": "PORT_CONFIG"
},
{
"table_id":42,
"table_name":"VIRTUAL_PORT_DESTINATION",
"table_type":"virtual",
"physical_table": "PORT_CONFIG"
},
{
"table_id":43,
"table_name":"COMPOSITION_CONFIG",
"table_type":"composition",
"custom": {
"source_ip":"VIRTUAL_IP_PLUS_SOURCE",
"dest_ip":"VIRTUAL_IP_PLUS_DESTINATION",
"source_port":"VIRTUAL_PORT_SOURCE",
"dest_port":"VIRTUAL_PORT_DESTINATION"
}
},
{
"table_id":44,
"table_name":"TEST_PLUGIN_KEY_TYPE_TABLE", "table_name":"TEST_PLUGIN_KEY_TYPE_TABLE",
"table_type":"plugin", "table_type":"plugin",
"valid_column":4, "valid_column":4,

67
tools/maat_redis_tool.cpp
View File

@@ -13,6 +13,8 @@
#include "json2iris.h" #include "json2iris.h"
#include "hiredis/hiredis.h" #include "hiredis/hiredis.h"
#define MODULE_REDIS_TOOL module_name_str("maat.redis_tool")
const char *redis_dump_dir = "./redis_dump"; const char *redis_dump_dir = "./redis_dump";
const char *default_table_info = "./table_info.conf"; const char *default_table_info = "./table_info.conf";
@@ -205,34 +207,30 @@ long long absolute_expire_time=0;
int make_serial_rule(const char *table_name, const char *line, void *u_para) int make_serial_rule(const char *table_name, const char *line, void *u_para)
{ {
struct serial_rule *s_rule=(struct serial_rule *)u_para; struct serial_rule *s_rule=(struct serial_rule *)u_para;
int rule_id = 0; redisContext *ctx = s_rule->ref_ctx;
//int is_valid = 0;
char *buff = ALLOC(char, strlen(line) + 1); char *buff = ALLOC(char, strlen(line) + 1);
memcpy(buff, line, strlen(line) + 1);
while (buff[strlen(buff) - 1] == '\n' || buff[strlen(buff) - 1] == '\t') { memcpy(buff, line, strlen(line) + 1);
buff[strlen(buff) - 1] = '\0'; while(buff[strlen(line) - 1] == '\n' || buff[strlen(line) - 1] == '\t') {
buff[strlen(line) - 1] = '\0';
} }
int j = 0; const char *redis_rule_key = "TEST_RULE_KEY";
char *str1 = NULL; redisReply *reply = maat_cmd_wrap_redis_command(ctx, "INCRBY %s %d", redis_rule_key, 1);
char *token = NULL; if (reply->type == REDIS_REPLY_NIL) {
char *saveptr1 = NULL; printf("incrby redis_rule_key:%s failed.", redis_rule_key);
return -1;
} else {
s_rule->rule_id = maat_cmd_read_redis_integer(reply);
freeReplyObject(reply);
reply = NULL;
}
for (j = 0,str1 = buff; ; j++, str1 = NULL) { maat_cmd_set_serial_rule(s_rule + line_idx, MAAT_OP_ADD, s_rule->rule_id, table_name,
token = strtok_r(str1, "\t ", &saveptr1); buff, absolute_expire_time);
if (token == NULL) (s_rule + line_idx)->ref_ctx = ctx;
break;
if (j == 0) {
sscanf(token,"%d", &rule_id);
}
}
maat_cmd_set_serial_rule(s_rule + line_idx, MAAT_OP_ADD, rule_id, table_name, buff, absolute_expire_time);
line_idx++; line_idx++;
FREE(buff);
FREE(buff);
return 0; return 0;
} }
@@ -298,38 +296,46 @@ int main(int argc, char * argv[])
char tmp_iris_path[128] = {0}; char tmp_iris_path[128] = {0};
if (model == WORK_MODE_DUMP) { if (model == WORK_MODE_DUMP) {
printf("Reading key list from %s:%d db%d.\n", redis_ip, redis_port, redis_db); log_info(logger, MODULE_REDIS_TOOL, "Reading key list from %s:%d db%d.",
redis_ip, redis_port, redis_db);
read_rule_from_redis(c, desired_version, dump_dir, logger); read_rule_from_redis(c, desired_version, dump_dir, logger);
} else if(model == WORK_MODE_JSON) { } else if(model == WORK_MODE_JSON) {
char *json_buff = NULL; char *json_buff = NULL;
size_t json_buff_sz = 0; size_t json_buff_sz = 0;
int ret = load_file_to_memory(json_file, (unsigned char**)&json_buff, &json_buff_sz); int ret = load_file_to_memory(json_file, (unsigned char**)&json_buff, &json_buff_sz);
if (ret < 0) { if (ret < 0) {
printf("open %s failed.\n", json_file); log_error(logger, MODULE_REDIS_TOOL, "open file:%s failed.", json_file);
return -1;
} }
ret = json2iris(json_buff, json_file, c, tmp_iris_path, sizeof(tmp_iris_path), NULL, NULL, logger); ret = json2iris(json_buff, json_file, c, tmp_iris_path, sizeof(tmp_iris_path), NULL, NULL, logger);
FREE(json_buff); FREE(json_buff);
if (ret < 0) { if (ret < 0) {
printf("Invalid json format.\n"); log_error(logger, MODULE_REDIS_TOOL, "Invalid json format.");
return -1;
} }
size_t total_line_cnt = 0; size_t total_line_cnt = 0;
config_monitor_traverse(0, tmp_iris_path, NULL, count_line_num_cb, NULL, &total_line_cnt, logger); config_monitor_traverse(0, tmp_iris_path, NULL, count_line_num_cb, NULL, &total_line_cnt, logger);
printf("Serialize %s to %zu lines, write temp file to %s .\n", json_file, total_line_cnt, tmp_iris_path); log_error(logger, MODULE_REDIS_TOOL, "Serialize %s to %zu lines, write temp file to %s .",
json_file, total_line_cnt, tmp_iris_path);
struct serial_rule *s_rule = ALLOC(struct serial_rule, total_line_cnt); struct serial_rule *s_rule = ALLOC(struct serial_rule, total_line_cnt);
s_rule->ref_ctx = c;
long long server_time = maat_cmd_redis_server_time_s(c); long long server_time = maat_cmd_redis_server_time_s(c);
if (!server_time) { if (!server_time) {
printf("Get Redis Time failed.\n"); log_error(logger, MODULE_REDIS_TOOL, "Get Redis Time failed.");
FREE(s_rule);
return -1;
} }
if (timeout > 0) { if (timeout > 0) {
absolute_expire_time = server_time + timeout; absolute_expire_time = server_time + timeout;
} }
config_monitor_traverse(0, tmp_iris_path, NULL, make_serial_rule, NULL, s_rule, logger); config_monitor_traverse(0, tmp_iris_path, NULL, make_serial_rule, NULL, s_rule, logger);
printf("Timeout = %lld\n", absolute_expire_time); s_rule->ref_ctx = NULL;
log_info(logger, MODULE_REDIS_TOOL, "Timeout = %lld\n", absolute_expire_time);
ret = 0; ret = 0;
int success_cnt = 0; int success_cnt = 0;
@@ -338,7 +344,8 @@ int main(int argc, char * argv[])
} while(success_cnt < 0); } while(success_cnt < 0);
if (success_cnt != (int)total_line_cnt) { if (success_cnt != (int)total_line_cnt) {
printf("Only Add %d of %zu, rule id maybe conflicts.\n", success_cnt, total_line_cnt); log_error(logger, MODULE_REDIS_TOOL, "Only Add %d of %zu, rule id maybe conflicts.",
success_cnt, total_line_cnt);
} }
for (size_t i = 0; i < total_line_cnt; i++) { for (size_t i = 0; i < total_line_cnt; i++) {