gfwleak/tango-maat
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix clause update bug and stream scan bug

Browse Source
This commit is contained in:
liuwentan
2023-03-29 14:29:34 +08:00
parent cca03b6faf
commit 658625fde3
6 changed files with 141 additions and 61 deletions
Download Patch File Download Diff File Expand all files Collapse all files

26
scanner/adapter_hs/adapter_hs.cpp
View File

@@ -522,8 +522,6 @@ int matched_event_cb(unsigned int id, unsigned long long from,
}
int ret = 0;
long long start_offset = -1;
long long end_offset = -1;
struct pattern_attribute pat_attr = matched_pat->ref_hs_attr[id];
switch (pat_attr.match_mode) {
case HS_MATCH_MODE_EXACTLY:
@@ -532,20 +530,28 @@ int matched_event_cb(unsigned int id, unsigned long long from,
}
break;
case HS_MATCH_MODE_SUB:
if (pat_attr.offset.start == -1 &&
pat_attr.offset.end == -1) {
ret = 1;
break;
}
if (pat_attr.offset.start == -1) {
start_offset = 0;
} else {
start_offset = pat_attr.offset.start;
if ((long long)(to - 1) <= pat_attr.offset.end) {
ret = 1;
break;
}
}
if (pat_attr.offset.end == -1) {
end_offset = matched_pat->scan_data_len;
} else {
end_offset = pat_attr.offset.end;
if ((long long)from >= pat_attr.offset.start) {
ret = 1;
break;
}
}
if (start_offset <= (long long)from &&
end_offset >= (long long)(to - 1)) {
if ((long long)from >= pat_attr.offset.start &&
(long long)(to - 1) <= pat_attr.offset.end) {
ret = 1;
}
break;

68
src/maat_compile.c
View File

@@ -72,7 +72,7 @@ struct compile_runtime {
struct maat_compile *compile_hash; // <compile_id, struct maat_compile>
struct maat_runtime *ref_maat_rt;
uint32_t rule_num;
struct maat_clause *clause_by_literals_hash;
pthread_rwlock_t rwlock; /* TODO: replaced with mutex? */
struct bool_expr_match *expr_match_buff;
@@ -499,6 +499,7 @@ void *compile_runtime_new(void *compile_schema, int max_thread_num,
struct compile_runtime *compile_rt = ALLOC(struct compile_runtime, 1);
compile_rt->expr_match_buff = ALLOC(struct bool_expr_match, max_thread_num * MAX_SCANNER_HIT_COMPILE_NUM);
compile_rt->clause_by_literals_hash = NULL;
compile_rt->logger = logger;
compile_rt->ref_garbage_bin = garbage_bin;
pthread_rwlock_init(&compile_rt->rwlock, NULL);
@@ -541,6 +542,18 @@ void maat_compile_hash_free(struct maat_compile **compile_hash)
assert(*compile_hash == NULL);
}
static void maat_clause_hash_free(struct maat_clause **clause_hash)
{
struct maat_clause *clause = NULL, *tmp_clause = NULL;
HASH_ITER (hh, *clause_hash, clause, tmp_clause) {
HASH_DEL(*clause_hash, clause);
FREE(clause->literal_ids);
clause->n_literal_id = 0;
FREE(clause);
}
}
void compile_runtime_free(void *compile_runtime)
{
if (NULL == compile_runtime) {
@@ -553,12 +566,18 @@ void compile_runtime_free(void *compile_runtime)
if (compile_rt->bm != NULL) {
bool_matcher_free(compile_rt->bm);
compile_rt->bm = NULL;
}
if (compile_rt->compile_hash != NULL) {
maat_compile_hash_free(&(compile_rt->compile_hash));
compile_rt->compile_hash = NULL;
}
if (compile_rt->clause_by_literals_hash != NULL) {
maat_clause_hash_free(&compile_rt->clause_by_literals_hash);
compile_rt->clause_by_literals_hash = NULL;
}
if (compile_rt->expr_match_buff != NULL) {
FREE(compile_rt->expr_match_buff);
}
@@ -723,6 +742,7 @@ struct maat_compile *maat_compile_new(long long compile_id)
for(int i = 0; i < MAX_ITEMS_PER_BOOL_EXPR; i++) {
utarray_new(compile->clause_states[i].ut_literal_ids, &ut_literal_id_icd);
compile->clause_states[i].in_use=0;
compile->clause_states[i].clause_id = -1;
}
return compile;
@@ -751,8 +771,7 @@ int maat_compile_hash_add(struct maat_compile **compile_hash, long long compile_
HASH_ADD(hh, *compile_hash, compile_id, sizeof(long long), compile);
//TODO:mytest need to delete
#if 0
if (compile_id == 141)
{
size_t compile_cnt = HASH_COUNT(*compile_hash);
struct maat_compile *compile1 = NULL, *tmp_compile1 = NULL;
HASH_ITER(hh, *compile_hash, compile1, tmp_compile1)
@@ -760,7 +779,7 @@ int maat_compile_hash_add(struct maat_compile **compile_hash, long long compile_
printf("<maat_compile_hash_add> compile_id:%lld, compile_cnt:%zu\n",
compile1->compile_id, compile_cnt);
}
}
#endif
return ret;
}
@@ -879,40 +898,28 @@ int maat_compile_clause_remove_literal(struct maat_compile *compile,
}
static const struct maat_clause *
maat_clause_hash_fetch_clause(struct maat_clause **clause_hash,
struct maat_runtime *ref_maat_rt,
maat_clause_hash_fetch_clause(struct compile_runtime *compile_rt,
struct maat_literal_id *literal_ids,
size_t n_literal_id)
{
struct maat_clause *clause = NULL;
HASH_FIND(hh, *clause_hash, literal_ids, n_literal_id * sizeof(struct maat_literal_id), clause);
HASH_FIND(hh, compile_rt->clause_by_literals_hash, literal_ids,
n_literal_id * sizeof(struct maat_literal_id), clause);
if (!clause) {
clause = ALLOC(struct maat_clause, 1);
clause->clause_id = maat_runtime_get_sequence(ref_maat_rt, "clause_id");
clause->clause_id = maat_runtime_get_sequence(compile_rt->ref_maat_rt, "clause_id");
clause->n_literal_id = n_literal_id;
clause->literal_ids = ALLOC(struct maat_literal_id, n_literal_id);
memcpy(clause->literal_ids, literal_ids, n_literal_id * sizeof(struct maat_literal_id));
HASH_ADD_KEYPTR(hh, *clause_hash, clause->literal_ids,
HASH_ADD_KEYPTR(hh, compile_rt->clause_by_literals_hash, clause->literal_ids,
n_literal_id * sizeof(struct maat_literal_id), clause);
}
return clause;
}
static void maat_clause_hash_free(struct maat_clause **clause_hash)
{
struct maat_clause *clause = NULL, *tmp_clause = NULL;
HASH_ITER (hh, *clause_hash, clause, tmp_clause) {
HASH_DEL(*clause_hash, clause);
FREE(clause->literal_ids);
clause->n_literal_id = 0;
FREE(clause);
}
}
struct bool_matcher *maat_compile_bool_matcher_new(struct compile_runtime *compile_rt)
{
if (NULL == compile_rt) {
@@ -924,7 +931,7 @@ struct bool_matcher *maat_compile_bool_matcher_new(struct compile_runtime *compi
struct bool_matcher *bm = NULL;
struct maat_clause_state *clause_state = NULL;
const struct maat_clause *clause = NULL;
struct maat_clause *clause_hash = NULL; // <literal_id, struct maat_clause>
//struct maat_clause *clause_hash = NULL; // <literal_id, struct maat_clause>
pthread_rwlock_rdlock(&compile_rt->rwlock);
//STEP 1, update clause_id of each compile and literal
@@ -943,8 +950,7 @@ struct bool_matcher *maat_compile_bool_matcher_new(struct compile_runtime *compi
has_clause_num++;
literal_ids = (struct maat_literal_id *)utarray_eltptr(clause_state->ut_literal_ids, 0);
n_literal_id = utarray_len(clause_state->ut_literal_ids);
clause = maat_clause_hash_fetch_clause(&clause_hash, compile_rt->ref_maat_rt,
literal_ids, n_literal_id);
clause = maat_clause_hash_fetch_clause(compile_rt, literal_ids, n_literal_id);
clause_state->clause_id = clause->clause_id;
}
assert(has_clause_num == compile->actual_clause_num);
@@ -1023,13 +1029,7 @@ struct bool_matcher *maat_compile_bool_matcher_new(struct compile_runtime *compi
}
error:
if (clause_hash != NULL) {
maat_clause_hash_free(&clause_hash);
clause_hash = NULL;
}
FREE(bool_expr_array);
return bm;
}
@@ -1392,7 +1392,7 @@ void maat_compile_state_update_hit_clause(struct maat_compile_state *compile_sta
if (!clause_state->in_use) {
continue;
}
size_t new_clause_idx = utarray_len(compile_state->this_scan_hit_clauses);
tmp = (struct maat_literal_id *)utarray_find(clause_state->ut_literal_ids,
&literal_id, compare_literal_id);
@@ -1416,6 +1416,12 @@ void maat_compile_state_update_hit_clause(struct maat_compile_state *compile_sta
utarray_push_back(compile_state->all_hit_clauses, clause_id);
}
utarray_sort(compile_state->all_hit_clauses, compare_clause_id);
// printf("<update_hit_clause> all_hit_clause:");
// for (i = 0; i < utarray_len(compile_state->all_hit_clauses); i++) {
// long long *tmp_clause_id = (long long *)utarray_eltptr(compile_state->all_hit_clauses, i);
// printf(" %lld ", *tmp_clause_id);
// }
// printf("\n");
}
}
}

27
test/adapter_hs_gtest.cpp
View File

@@ -241,7 +241,6 @@ TEST(adapter_hs_scan, literal_sub_has_normal_offset)
int ret = parse_config_file("./literal_expr.conf", expr_array, &n_expr_array);
EXPECT_EQ(ret, 0);
EXPECT_EQ(n_expr_array, 12);
struct adapter_hs *hs_instance = adapter_hs_new(1, expr_array, n_expr_array, g_logger);
EXPECT_TRUE(hs_instance != NULL);
@@ -653,6 +652,32 @@ TEST(adapter_hs_scan, same_pattern_different_offset)
hs_instance = NULL;
}
TEST(adapter_hs_scan, long_scan_data)
{
struct hs_expr expr_array[64] = {0};
size_t n_expr_array = 0;
int ret = parse_config_file("./literal_expr.conf", expr_array, &n_expr_array);
EXPECT_EQ(ret, 0);
struct adapter_hs *hs_instance = adapter_hs_new(1, expr_array, n_expr_array, g_logger);
EXPECT_TRUE(hs_instance != NULL);
expr_array_free(expr_array, n_expr_array);
const char* scan_data = "A directed path in a directed graph is a finite or infinite\
sequence of edges which joins a sequence of distinct vertices, but with the added restriction\
that the edges be all directed in the same direction.";
struct hs_scan_result result[64] = {0};
size_t n_result = 0;
ret = adapter_hs_scan(hs_instance, 0, scan_data, strlen(scan_data), result, 64, &n_result);
EXPECT_EQ(ret, 0);
EXPECT_EQ(n_result, 1);
EXPECT_EQ(result[0].item_id, 113);
adapter_hs_free(hs_instance);
hs_instance = NULL;
}
int main(int argc, char **argv)
{
int ret = 0;

12
test/literal_expr.conf
View File

@@ -161,6 +161,18 @@
"offset": "16~20"
}
]
},
{
"expr_id": 113,
"pattern_num": 1,
"patterns": [
{
"match_method": "sub",
"case_sensitive": "yes",
"is_hexbin": "no",
"pattern": "a finite or infinite"
}
]
}
]
}

36
test/maat_framework_gtest.cpp
View File

@@ -1286,7 +1286,7 @@ TEST_F(MaatIPScan, IPv4_IPPort) {
uint32_t sip;
int ret = inet_pton(AF_INET, ip_str, &sip);
EXPECT_EQ(ret, 1);
uint16_t port = htons(65529);
uint16_t port = htons(65530);
int proto = 6;
long long results[ARRAY_SIZE] = {0};
@@ -1313,7 +1313,7 @@ TEST_F(MaatIPScan, IPv4_Port) {
uint32_t sip;
int ret = inet_pton(AF_INET, ip_str, &sip);
EXPECT_EQ(ret, 1);
uint16_t port = htons(65528);
uint16_t port = htons(20303);
int proto = 6;
long long results[ARRAY_SIZE] = {0};
@@ -1324,12 +1324,12 @@ TEST_F(MaatIPScan, IPv4_Port) {
EXPECT_EQ(ret, MAAT_SCAN_OK);
maat_state_reset(state);
port = htons(65529);
port = htons(64999);
ret = maat_scan_ipv4(maat_instance, table_id, sip, port, proto,
results, ARRAY_SIZE, &n_hit_result, state);
EXPECT_EQ(ret, MAAT_SCAN_HIT);
EXPECT_EQ(n_hit_result, 1);
EXPECT_EQ(results[0], 154);
EXPECT_EQ(results[0], 169);
maat_state_free(state);
state = NULL;
@@ -2929,7 +2929,6 @@ protected:
maat_cmd_flushDB(_shared_maat_instance);
maat_free(_shared_maat_instance);
maat_options_set_deferred_load_on(opts);
_shared_maat_instance = maat_new(opts, table_info_path);
maat_options_free(opts);
}
@@ -4270,7 +4269,7 @@ TEST_F(MaatCmdTest, GroupInMassCompiles) {
maat_state_free(state);
state = NULL;
}
#if 0
TEST_F(MaatCmdTest, HitPath) {
const char *g2g_table_name = "GROUP2GROUP";
const char *g2c_table_name = "GROUP2COMPILE";
@@ -4343,7 +4342,7 @@ TEST_F(MaatCmdTest, HitPath) {
NULL, 0, 0, 0, 0); /*EXPR_TYPE_STRING MATCH_METHOD_SUB*/
EXPECT_EQ(ret, 1);
sleep(WAIT_FOR_EFFECTIVE_S);
sleep(WAIT_FOR_EFFECTIVE_S * 2);
const char* http_url = "en.wikipedia.org/wiki/Path_(graph_theory)";
const char* http_resp_hdr_cookie = "laptop=thinkpad X1 extrem;time=2020-02-11T15:34:00;main[XWJOKE]=hoho; Hm_lvt_bbac0322e6ee13093f98d5c4b5a10912=1578874808;";
@@ -4419,7 +4418,7 @@ that the edges be all directed in the same direction.";
struct maat_stream *stream = maat_stream_new(maat_instance, table_id, state);
Nth_scan++;
ret = maat_stream_scan(stream, keywords1, strlen(keywords1), results, ARRAY_SIZE,
&n_hit_result, state);
&n_hit_result, state);
EXPECT_EQ(ret, MAAT_SCAN_HALF_HIT);
n_read = maat_state_get_hit_paths(state, hit_path, sizeof(hit_path));
EXPECT_EQ(n_read, 3);
@@ -4456,7 +4455,7 @@ that the edges be all directed in the same direction.";
EXPECT_EQ(hit_path[path_idx].vtable_id, 0);
EXPECT_EQ(hit_path[path_idx].compile_id, -1);
Nth_scan++;
Nth_scan++;
ret = maat_stream_scan(stream, keywords2, strlen(keywords2), results, ARRAY_SIZE,
&n_hit_result, state);
EXPECT_EQ(ret, MAAT_SCAN_HALF_HIT);
@@ -4502,19 +4501,19 @@ TEST_F(MaatCmdTest, SameScanStatusWhenClauseUpdate_TSG6419) {
long long item11_id = maat_cmd_incrby(maat_instance, "SEQUENCE_REGION", 1);
ret = ip_table_set_line(maat_instance, ip_table_name, MAAT_OP_ADD, item11_id, group11_id,
IPv4, "192.168.2.1", "192.168.2.4", 0, 0, 0);
IPv4, "192.168.2.1", "192.168.2.4", 0, 65535, 0);
EXPECT_EQ(ret, 1);
long long group21_id = maat_cmd_incrby(maat_instance, "SEQUENCE_GROUP", 1);
ret = group2compile_table_set_line(maat_instance, g2c_table_name, MAAT_OP_ADD, group21_id,
compile1_id, 0, "null", 1, 0);
compile1_id, 0, "null", 2, 0);
EXPECT_EQ(ret, 1);
long long item21_id = maat_cmd_incrby(maat_instance, "SEQUENCE_REGION", 1);
ret = intval_table_set_line(maat_instance, app_id_table_name, MAAT_OP_ADD, item21_id,
group21_id, 31, 31, NULL, 0);
EXPECT_EQ(ret, 1);
sleep(WAIT_FOR_EFFECTIVE_S);
sleep(WAIT_FOR_EFFECTIVE_S * 2);
long long results[ARRAY_SIZE] = {0};
size_t n_hit_result = 0;
@@ -4555,7 +4554,7 @@ TEST_F(MaatCmdTest, SameScanStatusWhenClauseUpdate_TSG6419) {
group22_id, 32, 32, NULL, 0);
EXPECT_EQ(ret, 1);
sleep(WAIT_FOR_EFFECTIVE_S);
sleep(WAIT_FOR_EFFECTIVE_S * 2);
table_id = maat_get_table_id(maat_instance, app_id_table_name);
ret = maat_scan_integer(maat_instance, table_id, 31, results, ARRAY_SIZE,
@@ -4595,7 +4594,7 @@ TEST_F(MaatCmdTest, GroupEdit) {
long long item11_id = maat_cmd_incrby(maat_instance, "SEQUENCE_REGION", 1);
ret = ip_table_set_line(maat_instance, ip_table_name, MAAT_OP_ADD, item11_id, group11_id,
IPv4, "192.168.3.1", "192.168.3.4", 0, 0, 0);
IPv4, "192.168.3.1", "192.168.3.4", 0, 65535, 0);
EXPECT_EQ(ret, 1);
long long group21_id = maat_cmd_incrby(maat_instance, "SEQUENCE_GROUP", 1);
@@ -4747,11 +4746,11 @@ TEST_F(MaatCmdTest, CompileDelete_TSG6548) {
while (now - update_time < 3) {
ret = maat_scan_ipv4(maat_instance, table_id, ip_addr, port, 6, results, ARRAY_SIZE,
&n_hit_result, state);
if (ret > 0) {
if (ret == MAAT_SCAN_HIT) {
hit_cnt++;
EXPECT_EQ(results[0], compile1_id);
} else {
EXPECT_EQ(ret, MAAT_SCAN_HALF_HIT);
}
if (ret == MAAT_SCAN_HALF_HIT) {
miss_cnt++;
}
now = time(NULL);
@@ -4761,6 +4760,7 @@ TEST_F(MaatCmdTest, CompileDelete_TSG6548) {
maat_state_free(state);
}
#if 0
TEST_F(MaatCmdTest, UpdateDeadLockDetection) {
const char* g2c_table_name = "GROUP2COMPILE";
const char* compile_table_name = "COMPILE";
@@ -4878,7 +4878,7 @@ TEST_F(MaatCmdTest, StreamScanSegfaultWhenVersionRollBack_TSG6324) {
EXPECT_EQ(results[0], compile1_id);
//DON'T DO THIS!!!
//Roll back version, trigger full udpate.
//Roll back version, trigger full update.
//This operation generates FATAL logs in test_maat_redis.log.yyyy-mm-dd.
//For example: Add group 22 vt_id 0 to clause 2 of compile 979 failed, group is already exisited
maat_cmd_incrby(maat_instance, "MAAT_VERSION", -100);

33
test/maat_json.json
View File

@@ -1064,7 +1064,7 @@
"ip1": "10.0.7.100",
"ip2": "10.0.7.106",
"port_format": "range",
"port1": "65529",
"port1": "65530",
"port2": "65535",
"protocol": 6
}
@@ -1506,6 +1506,37 @@
}
]
},
{
"compile_id": 169,
"service": 0,
"action": 0,
"do_blacklist": 0,
"do_log": 0,
"effective_rage": 0,
"user_region": "IPScan.IPv4_Any",
"is_valid": "yes",
"groups": [
{
"regions": [
{
"table_type": "ip_plus",
"table_name": "IP_PLUS_CONFIG",
"table_content": {
"addr_type": "ipv4",
"addr_format": "CIDR",
"ip1": "0.0.0.0",
"ip2": "0",
"port_format": "range",
"port1": "64000",
"port2": "64999",
"protocol": 6
}
}
],
"not_flag" : 0
}
]
},
{
"compile_id": 170,
"service": 0,