gfwleak/tango-maat
Archived
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

support expr stream scan

Browse Source
This commit is contained in:
liuwentan
2023-03-17 11:32:13 +08:00
parent c669eb5619
commit 68533f9d43
32 changed files with 748 additions and 56 deletions
Download Patch File Download Diff File Expand all files Collapse all files

120
scanner/adapter_hs/adapter_hs.cpp
View File

@@ -63,16 +63,7 @@ struct adapter_hs {
size_t n_patterns;
struct adapter_hs_runtime *hs_rt;
struct hs_tag *tag_map;
};
struct adapter_hs_stream {
int thread_id;
size_t n_expr;
size_t n_patterns;
hs_stream_t *literal_stream;
hs_stream_t *regex_stream;
struct adapter_hs_runtime *hs_rt;
UT_array *pattern_id_set;
struct log_handle *logger;
};
struct matched_pattern {
@@ -90,6 +81,16 @@ struct matched_pattern_container {
struct matched_pattern *pat_hash;
};
struct adapter_hs_stream {
int thread_id;
size_t n_expr;
size_t n_patterns;
hs_stream_t *literal_stream;
hs_stream_t *regex_stream;
struct adapter_hs_runtime *hs_rt;
struct matched_pattern_container matched_pat_container;
};
struct pattern_attribute {
unsigned long long pattern_id;
enum hs_match_mode match_mode;
@@ -180,9 +181,9 @@ static int adpt_hs_build_database(struct adapter_hs_runtime *hs_rt,
return -1;
}
} else {
err = hs_compile_ext_multi((const char *const *)compile_data->patterns, compile_data->flags,
compile_data->ids, NULL, compile_data->n_patterns,
scan_mode, NULL, &hs_rt->regex_db, &compile_err);
err = hs_compile_multi((const char *const *)compile_data->patterns, compile_data->flags,
compile_data->ids, compile_data->n_patterns, scan_mode, NULL,
&hs_rt->regex_db, &compile_err);
if (err != HS_SUCCESS) {
if (compile_err) {
log_error(logger, MODULE_ADAPTER_HS, "[%s:%d] compile error: %s",
@@ -305,6 +306,7 @@ struct adapter_hs *adapter_hs_initialize(enum hs_scan_mode scan_mode,
uint32_t pattern_index = 0;
struct adapter_hs *hs_instance = ALLOC(struct adapter_hs, 1);
hs_instance->tag_map = NULL;
hs_instance->logger = logger;
struct bool_expr *bool_exprs = ALLOC(struct bool_expr, n_expr);
/* populate adpt_hs_compile_data and bool_expr */
@@ -324,7 +326,7 @@ struct adapter_hs *adapter_hs_initialize(enum hs_scan_mode scan_mode,
compile_data->ids[pattern_index] = pattern_index;
if (pattern_type == HS_PATTERN_TYPE_STR) {
compile_data->flags[pattern_index] = HS_FLAG_SOM_LEFTMOST;
compile_data->flags[pattern_index] |= HS_FLAG_SOM_LEFTMOST;
}
if (exprs[i].patterns[j].case_sensitive == HS_CASE_INSESITIVE) {
@@ -333,7 +335,7 @@ struct adapter_hs *adapter_hs_initialize(enum hs_scan_mode scan_mode,
pat_len = exprs[i].patterns[j].pat_len;
compile_data->pattern_lens[pattern_index] = pat_len;
compile_data->patterns[pattern_index] = ALLOC(char, pat_len);
compile_data->patterns[pattern_index] = ALLOC(char, pat_len + 1);
memcpy(compile_data->patterns[pattern_index], exprs[i].patterns[j].pat,
exprs[i].patterns[j].pat_len);
@@ -357,11 +359,8 @@ struct adapter_hs *adapter_hs_initialize(enum hs_scan_mode scan_mode,
hs_instance->hs_rt = ALLOC(struct adapter_hs_runtime, 1);
//mytest
// for (size_t i = 0; i < n_expr_array; i++) {
// printf("exprs[%zu] expr_id:%llu, item_num:%zu\n", i, exprs[i].expr_id, exprs[i].item_num);
// for (size_t j = 0; j < exprs[i].item_num; j++) {
// printf("item[%zu] item_id: %llu\n", j, exprs[i].items[j].item_id);
// }
// for (size_t i = 0; i < n_expr; i++) {
// printf("hs_instance:%p exprs[%zu] expr_id:%llu, item_num:%zu\n", hs_instance, i, bool_exprs[i].expr_id, bool_exprs[i].item_num);
// }
/* create bool matcher */
@@ -577,12 +576,20 @@ int adapter_hs_scan(struct adapter_hs *hs_instance, int thread_id,
err_count++;
}
}
if (2 == err_count) {
if (err_count > 0) {
utarray_free(matched_pat_container.pat_ids);
return -1;
}
size_t matched_pattern_ids_cnt = utarray_len(matched_pat_container.pat_ids);
if (0 == matched_pattern_ids_cnt) {
*n_hit_result = 0;
utarray_free(matched_pat_container.pat_ids);
assert(matched_pat_container.pat_hash == NULL);
return 0;
}
size_t i = 0;
unsigned long long items[matched_pattern_ids_cnt];
memset(items, 0, sizeof(unsigned long long) * matched_pattern_ids_cnt);
@@ -646,28 +653,25 @@ struct adapter_hs_stream *adapter_hs_stream_open(struct adapter_hs *hs_instance,
hs_stream->n_expr = hs_instance->n_expr;
hs_stream->n_patterns = hs_instance->n_patterns;
hs_stream->hs_rt = hs_instance->hs_rt;
utarray_new(hs_stream->pattern_id_set, &ut_pattern_id_icd);
utarray_reserve(hs_stream->pattern_id_set, hs_stream->n_patterns);
utarray_new(hs_stream->matched_pat_container.pat_ids, &ut_pattern_id_icd);
utarray_reserve(hs_stream->matched_pat_container.pat_ids, hs_stream->n_patterns);
int err_count = 0;
if (hs_instance->hs_rt->literal_db != NULL) {
err = hs_open_stream(hs_instance->hs_rt->literal_db, 0, &hs_stream->literal_stream);
if (err != HS_SUCCESS) {
err_count++;
log_error(hs_instance->logger, MODULE_ADAPTER_HS, "hs_open_stream failed, hs err:%d", err);
return NULL;
}
}
if (hs_instance->hs_rt->regex_db != NULL) {
err = hs_open_stream(hs_instance->hs_rt->regex_db, 0, &hs_stream->regex_stream);
if (err != HS_SUCCESS) {
err_count++;
log_error(hs_instance->logger, MODULE_ADAPTER_HS, "hs_open_stream failed, hs err:%d", err);
return NULL;
}
}
if (2 == err_count) {
return NULL;
}
return hs_stream;
}
@@ -681,13 +685,23 @@ int adapter_hs_scan_stream(struct adapter_hs_stream *hs_stream, const char *data
return -1;
}
/*
In streaming mode, a non-zero return from the user-specified event-handler
function has consequences for the rest of that stream's lifetime: when a
non-zero return occurs, it signals that no more of the stream should be
scanned. Consequently if the user makes a subsequent call to
`hs_scan_stream` on a stream whose processing was terminated in this way,
hs_scan_stream will return `HS_SCAN_TERMINATED`. This case has not been
demonstrated in pcapscan, as its callback always returns 0.
*/
int err_count = 0;
int thread_id = hs_stream->thread_id;
if (hs_stream->literal_stream != NULL) {
err = hs_scan_stream(hs_stream->literal_stream, data, data_len,
0, hs_stream->hs_rt->scratchs[thread_id],
matched_event_cb, hs_stream->pattern_id_set);
if (err != HS_SUCCESS) {
matched_event_cb, &hs_stream->matched_pat_container);
if (err != HS_SUCCESS && err != HS_SCAN_TERMINATED) {
err_count++;
}
}
@@ -695,21 +709,26 @@ int adapter_hs_scan_stream(struct adapter_hs_stream *hs_stream, const char *data
if (hs_stream->regex_stream != NULL) {
err = hs_scan_stream(hs_stream->regex_stream, data, data_len,
0, hs_stream->hs_rt->scratchs[thread_id],
matched_event_cb, hs_stream->pattern_id_set);
if (err != HS_SUCCESS) {
matched_event_cb, &hs_stream->matched_pat_container);
if (err != HS_SUCCESS && err != HS_SCAN_TERMINATED) {
err_count++;
}
}
if (2 == err_count) {
if (err_count > 0) {
return -1;
}
size_t pattern_set_size = utarray_len(hs_stream->pattern_id_set);
unsigned long long items[pattern_set_size];
memset(items, 0, sizeof(unsigned long long) * pattern_set_size);
for (size_t i = 0; i < pattern_set_size; i++) {
items[i] = *(unsigned long long *)utarray_eltptr(hs_stream->pattern_id_set, i);
size_t matched_pattern_ids_cnt = utarray_len(hs_stream->matched_pat_container.pat_ids);
if (0 == matched_pattern_ids_cnt) {
*n_hit_result = 0;
return 0;
}
unsigned long long items[matched_pattern_ids_cnt];
memset(items, 0, sizeof(unsigned long long) * matched_pattern_ids_cnt);
for (size_t i = 0; i < matched_pattern_ids_cnt; i++) {
items[i] = *(unsigned long long *)utarray_eltptr(hs_stream->matched_pat_container.pat_ids, i);
}
int ret = 0;
@@ -717,7 +736,7 @@ int adapter_hs_scan_stream(struct adapter_hs_stream *hs_stream, const char *data
struct bool_expr_match *bool_matcher_results = NULL;
bool_matcher_results = ALLOC(struct bool_expr_match, hs_stream->n_expr);
int bool_matcher_ret = bool_matcher_match(hs_stream->hs_rt->bm, items, pattern_set_size,
int bool_matcher_ret = bool_matcher_match(hs_stream->hs_rt->bm, items, matched_pattern_ids_cnt,
bool_matcher_results, hs_stream->n_expr);
if (bool_matcher_ret < 0) {
ret = -1;
@@ -735,6 +754,14 @@ int adapter_hs_scan_stream(struct adapter_hs_stream *hs_stream, const char *data
*n_hit_result = bool_matcher_ret;
next:
FREE(bool_matcher_results);
struct matched_pattern *pattern = NULL, *tmp_pattern = NULL;
HASH_ITER(hh, hs_stream->matched_pat_container.pat_hash, pattern, tmp_pattern) {
HASH_DELETE(hh, hs_stream->matched_pat_container.pat_hash, pattern);
FREE(pattern);
}
utarray_clear(hs_stream->matched_pat_container.pat_ids);
return ret;
}
@@ -762,7 +789,12 @@ void adapter_hs_stream_close(struct adapter_hs_stream *hs_stream)
}
}
utarray_free(hs_stream->pattern_id_set);
struct matched_pattern *pattern = NULL, *tmp_pattern = NULL;
HASH_ITER(hh, hs_stream->matched_pat_container.pat_hash, pattern, tmp_pattern) {
HASH_DELETE(hh, hs_stream->matched_pat_container.pat_hash, pattern);
FREE(pattern);
}
utarray_free(hs_stream->matched_pat_container.pat_ids);
/* hs_stream->hs_rt point to hs_instance->hs_rt which will call free */
hs_stream->hs_rt = NULL;

13
src/maat_expr.c
View File

@@ -667,13 +667,13 @@ struct hs_expr *expr_item_to_expr_rule(struct expr_item *expr_item, void *user_d
}
if (region_string != NULL) {
expr_rule->patterns[i].pat = ALLOC(char, region_str_len);
expr_rule->patterns[i].pat = ALLOC(char, region_str_len + 1);
memcpy(expr_rule->patterns[i].pat, region_string, region_str_len);
expr_rule->patterns[i].pat_len = region_str_len;
FREE(region_string);
} else {
sub_key_len = strlen(sub_key_array[i]);
expr_rule->patterns[i].pat = ALLOC(char, sub_key_len);
expr_rule->patterns[i].pat = ALLOC(char, sub_key_len + 1);
memcpy(expr_rule->patterns[i].pat, sub_key_array[i], sub_key_len);
expr_rule->patterns[i].pat_len = sub_key_len;
}
@@ -792,9 +792,12 @@ int expr_runtime_commit(void *expr_runtime, const char *table_name)
rules = ALLOC(struct hs_expr, rule_cnt);
for (size_t i = 0; i < rule_cnt; i++) {
rules[i] = *(struct hs_expr *)ex_data_array[i];
//printf("expr_id:%lld\n", rules[i].expr_id);
// for (size_t j = 0; j < rules[i].n_patterns; j++) {
// printf("rules[%zu].patterns[%zu]:%s\n", i, j, rules[i].patterns[j].pat);
// if (rules[i].expr_id == 13)
// {
// for (size_t j = 0; j < rules[i].n_patterns; j++)
// {
// printf("rules[%zu].patterns[%zu]:%s\n", i, j, rules[i].patterns[j].pat);
// }
// }
}
}

193
test/file_test_tableinfo.conf Normal file
View File

@@ -0,0 +1,193 @@
[
{
"table_id":0,
"table_name":["NTC_COMPILE", "WHITE_LIST_COMPILE"],
"table_type":"compile",
"valid_column":8,
"custom": {
"compile_id":1,
"tags":6,
"clause_num":9,
"evaluation_order":10
}
},
{
"table_id":1,
"table_name":"NTC_GROUP2GROUP",
"table_type":"group2group",
"valid_column":3,
"custom": {
"group_id":1,
"super_group_id":2
}
},
{
"table_id":2,
"table_name":"NTC_GROUP2COMPILE",
"table_type":"group2compile",
"associated_compile_table_id":0,
"valid_column":3,
"custom": {
"group_id":1,
"compile_id":2,
"not_flag":4,
"virtual_table_name":5,
"clause_index":6
}
},
{
"table_id":3,
"table_name":"NTC_UNIVERSAL_IP",
"table_type":"ip_plus",
"valid_column":18,
"custom": {
"item_id":1,
"group_id":2,
"addr_type":3,
"saddr_format":4,
"sip1":5,
"sip2":6
}
},
{
"table_id":4,
"table_name":"NTC_UNIVERSAL_PROTO_TYPE",
"table_type":"intval",
"valid_column":5,
"custom": {
"item_id":1,
"group_id":2,
"low_bound":3,
"up_bound":4
}
},
{
"table_id":5,
"table_name":"WHITE_LIST_IP",
"table_type":"ip_plus",
"valid_column":18,
"custom": {
"item_id":1,
"group_id":2,
"addr_type":3,
"saddr_format":4,
"sip1":5,
"sip2":6
}
},
{
"table_id":7,
"table_name":["NTC_HTTP_URL", "WHITE_LIST_DOMAIN"],
"table_type":"expr",
"valid_column":7,
"custom": {
"scan_mode":"block",
"pattern_type":"literal",
"item_id":1,
"group_id":2,
"keywords":3,
"expr_type":4,
"match_method":5,
"is_hexbin":6
}
},
{
"table_id":8,
"table_name":["NTC_HTTP_REQ_HDR", "NTC_HTTP_RES_HDR"],
"table_type":"expr_plus",
"valid_column":8,
"custom": {
"scan_mode":"block",
"pattern_type":"literal",
"item_id":1,
"group_id":2,
"district":3,
"keywords":4,
"expr_type":5,
"match_method":6,
"is_hexbin":7
}
},
{
"table_id":9,
"table_name":["NTC_HTTP_REQ_BODY", "NTC_HTTP_RES_BODY"],
"table_type":"expr",
"valid_column":7,
"custom": {
"scan_mode":"block",
"pattern_type":"literal",
"item_id":1,
"group_id":2,
"keywords":3,
"expr_type":4,
"match_method":5,
"is_hexbin":6
}
},
{
"table_id":11,
"table_name":"NTC_MAIL_HDR",
"table_type":"expr_plus",
"valid_column":8,
"custom": {
"scan_mode":"block",
"pattern_type":"literal",
"item_id":1,
"group_id":2,
"district":3,
"keywords":4,
"expr_type":5,
"match_method":6,
"is_hexbin":7
}
},
{
"table_id":12,
"table_name":"NTC_MAIL_BODY",
"table_type":"expr_plus",
"valid_column":8,
"custom": {
"scan_mode":"block",
"pattern_type":"literal",
"item_id":1,
"group_id":2,
"district":3,
"keywords":4,
"expr_type":5,
"match_method":6,
"is_hexbin":7
}
},
{
"table_id":13,
"table_name":"NTC_FTP_URL",
"table_type":"expr",
"valid_column":7,
"custom": {
"scan_mode":"block",
"pattern_type":"literal",
"item_id":1,
"group_id":2,
"keywords":3,
"expr_type":4,
"match_method":5,
"is_hexbin":6
}
},
{
"table_id":14,
"table_name":"NTC_FTP_CONTENT",
"table_type":"expr",
"valid_column":7,
"custom": {
"scan_mode":"block",
"pattern_type":"literal",
"item_id":1,
"group_id":2,
"keywords":3,
"expr_type":4,
"match_method":5,
"is_hexbin":6
}
}
]

35
test/maat_framework_gtest.cpp
View File

@@ -560,7 +560,7 @@ TEST_F(MaatStringScan, Expr8) {
EXPECT_NE(n_read, 0);
maat_state_free(&state);
}
#if 0
TEST_F(MaatStringScan, Regex) {
int ret = 0;
long long results[ARRAY_SIZE] = {0};
@@ -599,7 +599,7 @@ TEST_F(MaatStringScan, Regex) {
}
#endif
}
#endif
TEST_F(MaatStringScan, ExprPlus) {
long long results[ARRAY_SIZE] = {0};
size_t n_hit_result = 0;
@@ -662,7 +662,7 @@ TEST_F(MaatStringScan, ExprAndExprPlus) {
EXPECT_EQ(results[0], 195);
maat_state_free(&state);
}
#if 0
TEST_F(MaatStringScan, StreamInput) {
long long results[ARRAY_SIZE] = {0};
size_t n_hit_result = 0;
@@ -675,7 +675,7 @@ TEST_F(MaatStringScan, StreamInput) {
ASSERT_GT(table_id, 0);
struct maat_stream *sp = maat_scan_stream_open(maat_instance, table_id, 0);
ASSERT_FALSE(sp==NULL);
ASSERT_TRUE(sp != NULL);
int ret = maat_scan_stream(&sp, "www.cyberessays.com", strlen("www.cyberessays.com"),
results, ARRAY_SIZE, &n_hit_result, &state);
@@ -689,7 +689,7 @@ TEST_F(MaatStringScan, StreamInput) {
EXPECT_EQ(results[0], 125);
maat_state_free(&state);
}
#endif
//TODO:
#if 0
TEST_F(MaatStringScan, ShouldNotHitExprPlus) {
@@ -2456,6 +2456,29 @@ TEST_F(TableInfo, Conjunction) {
maat_state_free(&state);
}
class MaatFileTest : public testing::Test
{
protected:
static void SetUpTestCase() {
struct maat_options *opts = maat_options_new();
char json_path[PATH_MAX] = {0};
const char *table_info = "./file_test_tableinfo.conf";
snprintf(json_path, sizeof(json_path), "./%s", json_filename);
maat_options_set_json_file(opts, json_path);
maat_options_set_logger_path(opts, "./maat_input_mode_gtest.log");
struct maat *maat_instance = maat_new(opts, table_info);
EXPECT_TRUE(maat_instance != NULL);
}
static void TearDownTestCase() {
}
static struct maat *_shared_maat_instance;
};
class MaatCmdTest : public testing::Test
{
protected:
@@ -2483,7 +2506,7 @@ protected:
maat_free(_shared_maat_instance);
}
static struct maat *_shared_maat_instance;
static struct maat *_shared_maat_instance;
};
struct maat *MaatCmdTest::_shared_maat_instance;

22
test/ntcrule/full/2018-10-09/APP_COMPILE.0000050997 Normal file
View File

@@ -0,0 +1,22 @@
0000000021
193214 35 16 1 1 0 PROTO_ID=12 1 3 0
193198 35 16 1 1 0 PROTO_ID=8 1 3 0
193240 35 16 1 1 0 PROTO_ID=16 1 3 0
193069 1152 1 1 1 0 DOMAIN_ID=46002 1 1 0
193172 35 16 1 2 0 PROTO_ID=16 1 3 0
193212 35 16 1 1 0 PROTO_ID=24 1 3 0
193169 35 16 1 2 0 PROTO_ID=8 1 3 0
193235 35 16 1 1 0 PROTO_ID=5 1 3 0
193059 1028 1 1 1 0 APP_ID=90001 1 1 0
170505 36 16 1 1 0 PROTO_ID=13;BEHAV_ID=2 1 1 0
193218 1025 1 1 1 0 APP_ID=90001 1 1 0
170507 36 16 1 1 0 PROTO_ID=19;BEHAV_ID=2 1 1 0
170500 36 16 1 1 0 PROTO_ID=61;BEHAV_ID=1 1 1 0
14 1028 1 1 1 0 APP_ID=103301 1 1 0
193228 35 16 1 1 0 PROTO_ID=18 1 3 0
170503 36 16 1 1 0 PROTO_ID=15;BEHAV_ID=2 1 1 0
170502 36 16 1 1 0 PROTO_ID=15;BEHAV_ID=1 1 1 0
170504 36 16 1 1 0 PROTO_ID=13;BEHAV_ID=1 1 1 0
170506 36 16 1 1 0 PROTO_ID=19;BEHAV_ID=1 1 1 0
233 1028 1 1 1 0 APP_ID=102501 1 1 0
170501 36 16 1 1 0 PROTO_ID=61;BEHAV_ID=2 1 1 0

38
test/ntcrule/full/2018-10-09/APP_GROUP.0000050997 Normal file
View File

@@ -0,0 +1,38 @@
0000000037
922 193172 1
199 233 1
986 193240 1
570 170503 1
571 170502 1
799 193069 1
573 170504 1
783 193059 1
958 193212 1
984 193240 1
976 193235 1
971 193228 1
964 193218 1
978 193235 1
961 193214 1
950 193198 1
959 193212 1
575 170506 1
970 193228 1
568 170500 1
574 170507 1
963 193214 1
985 193240 1
949 193198 1
972 193228 1
962 193214 1
914 193169 1
913 193169 1
960 193212 1
915 193169 1
567 170501 1
921 193172 1
977 193235 1
13 14 1
951 193198 1
572 170505 1
920 193172 1

2
test/ntcrule/full/2018-10-09/APP_PAYLOAD.0000050997 Normal file
View File

@@ -0,0 +1,2 @@
0000000001
979 964 L2_header c4b8b44a1fce246e96c98a800800 0 0 1 1

17
test/ntcrule/full/2018-10-09/APP_POLICY.0000050997 Normal file
View File

@@ -0,0 +1,17 @@
0000000016
590 574 PROTO_ID=19&BEHAV_ID=2 1 0 0 1
584 568 PROTO_ID=61&BEHAV_ID=1 1 0 0 1
966 949 PROTO_ID=8 0 0 0 1
937 920 PROTO_ID=16 0 0 0 1
586 570 PROTO_ID=15&BEHAV_ID=2 1 0 0 1
591 575 PROTO_ID=19&BEHAV_ID=1 1 0 0 1
999 984 PROTO_ID=16 0 0 0 1
587 571 PROTO_ID=15&BEHAV_ID=1 1 0 0 1
991 976 PROTO_ID=5 0 0 0 1
589 573 PROTO_ID=13&BEHAV_ID=1 1 0 0 1
930 913 PROTO_ID=8 0 0 0 1
985 970 PROTO_ID=18 0 0 0 1
973 958 PROTO_ID=24 0 0 0 1
976 961 PROTO_ID=12 0 0 0 1
583 567 PROTO_ID=61&BEHAV_ID=2 1 0 0 1
588 572 PROTO_ID=13&BEHAV_ID=2 1 0 0 1

2
test/ntcrule/full/2018-10-09/DDOS_PROTECT_TARGET_IP_CB.0000050997 Normal file
View File

@@ -0,0 +1,2 @@
0000000001
270 0 4 0.0.0.0 255.255.255.255 0 65535 127.127.127.127 255.255.255.255 127 65535 6 0 1 32 5

2
test/ntcrule/full/2018-10-09/MM_AV_URL.0000050997 Normal file
View File

@@ -0,0 +1,2 @@
0000000001
10 10 www.sohu.com 0 0 0 1

2
test/ntcrule/full/2018-10-09/MM_COMPILE.0000050997 Normal file
View File

@@ -0,0 +1,2 @@
0000000001
5 272 16 1 2 0 0 1 1 0

2
test/ntcrule/full/2018-10-09/MM_GROUP.0000050997 Normal file
View File

@@ -0,0 +1,2 @@
0000000001
10 5 1

2
test/ntcrule/full/2018-10-09/NTC_ASN_IP.0000050997 Normal file
View File

@@ -0,0 +1,2 @@
0000000001
958 941 4 0.0.0.0 255.255.255.255 0 65535 0.0.0.1 255.255.255.255 0 65535 0 0 1

11
test/ntcrule/full/2018-10-09/NTC_BGP_AS.0000050997 Normal file
View File

@@ -0,0 +1,11 @@
0000000010
621 605 100 0 3 0 1
741 725 100 0 3 0 1
744 728 100 0 3 0 1
630 614 100 0 3 0 1
627 611 100 0 3 0 1
20 20 90 0 3 0 1
614 598 100 0 3 0 1
631 615 100 0 3 0 1
624 608 100 0 3 0 1
422 409 110 0 3 0 1

79
test/ntcrule/full/2018-10-09/NTC_COMPILE.0000050997 Normal file
View File

@@ -0,0 +1,79 @@
0000000078
193131 130 1 1 2 0 0 1 1 0
192977 132 1 1 2 0 0 1 1 0
193147 133 1 1 2 0 0 1 1 0
193138 129 1 1 1 0 0 1 1 0
193234 129 1 1 1 0 0 1 1 0
193119 129 1 1 1 0 0 1 1 0
193000 132 1 1 2 0 0 1 1 0
193155 17 16 1 2 0 0 1 1 0
193252 129 1 1 1 0 0 1 1 0
193128 130 1 1 2 0 0 1 1 0
192973 129 1 1 1 0 0 1 1 0
193091 132 1 1 1 0 0 1 1 0
170486 20 16 1 2 0 0 1 1 0
193132 18 16 1 2 0 DNS_STRATEGY=0 1 1 0
193140 129 1 1 2 0 0 1 1 0
192968 31 16 1 2 0 0 1 1 0
192978 132 1 1 1 0 0 1 1 0
193236 129 1 1 1 0 0 1 2 0
193289 129 1 1 1 0 0 1 2 0
193107 129 1 1 1 0 0 1 1 0
121 18 16 1 1 0 DNS_STRATEGY=0 1 1 0
192959 143 1 1 1 0 0 1 1 0
193126 130 1 1 2 0 0 1 1 0
193110 129 1 1 1 0 0 1 1 0
193294 20 16 1 1 0 0 1 1 0
170435 130 1 1 2 0 0 1 1 0
193076 132 1 1 1 0 0 1 1 0
193077 132 1 1 1 0 0 1 1 0
193121 129 1 1 1 0 0 1 1 0
192999 132 1 1 2 0 0 1 1 0
193139 129 1 1 2 0 0 1 1 0
193237 132 1 1 1 0 0 1 2 0
193258 129 1 1 1 0 0 1 2 0
116 130 1 1 1 0 0 1 1 0
32 143 1 1 2 0 0 1 1 0
120 18 16 1 1 0 DNS_STRATEGY=0 1 1 0
193133 129 1 1 1 0 0 1 1 0
193088 132 1 1 1 0 0 1 1 0
193149 21 16 1 2 0 0 1 1 0
193098 129 1 1 2 0 0 1 1 0
193102 18 16 1 2 0 DNS_STRATEGY=0 1 2 0
12 18 16 1 2 0 DNS_STRATEGY=101 1 1 0
193099 129 1 1 1 0 0 1 1 0
193145 129 1 1 1 0 0 1 1 0
193134 133 1 1 1 0 0 1 1 0
193039 31 16 1 1 0 0 1 3 0
193112 21 16 1 2 0 0 1 2 0
170436 18 16 1 2 0 DNS_STRATEGY=0 1 1 0
11 18 16 1 2 0 DNS_STRATEGY=0 1 1 0
192965 143 1 1 1 0 0 1 3 0
441 143 1 1 2 0 0 1 1 0
193101 132 1 1 1 0 0 1 1 0
193040 31 16 1 1 0 0 1 3 0
193108 129 1 1 1 0 0 1 2 0
193150 133 1 1 1 0 0 1 1 0
192976 132 1 1 2 0 0 1 2 0
193171 17 16 1 2 0 0 1 1 0
192960 143 1 1 1 0 0 1 3 0
193116 20 16 1 2 0 0 1 2 0
192966 143 1 1 1 0 0 1 3 0
193103 18 16 1 2 0 DNS_STRATEGY=0 1 2 0
193106 19 16 1 2 0 0 1 2 0
193154 129 1 1 2 0 0 1 1 0
170487 20 16 1 2 0 0 1 1 0
193113 129 1 1 1 0 0 1 1 0
193148 133 1 1 2 0 0 1 1 0
193105 129 1 1 1 0 0 1 1 0
193144 129 1 1 1 0 0 1 1 0
193127 18 16 1 2 0 DNS_STRATEGY=0 1 1 0
193114 21 16 1 2 0 0 1 2 0
193115 20 16 1 2 0 0 1 2 0
193129 130 1 1 2 0 0 1 1 0
118 130 1 1 1 0 0 1 1 0
193120 129 1 1 2 0 0 1 1 0
193002 132 1 1 2 0 0 1 1 0
170485 20 16 1 2 0 0 1 1 0
193130 18 16 1 2 0 DNS_STRATEGY=0 1 1 0
192967 143 1 1 1 0 0 1 3 0

4
test/ntcrule/full/2018-10-09/NTC_DNS_FAKE_IP_CB.0000050997 Normal file
View File

@@ -0,0 +1,4 @@
0000000003
9 0 4 0.0.0.0 255.255.255.255 0 65535 11.11.11.11 255.255.255.255 0 65535 0 0 1 64
10 133 4 0.0.0.0 255.255.255.255 0 65535 22.22.22.22 255.255.255.255 0 65535 0 0 1 64
193104 0 6 :: FFFF:FFFF:FFFF:FFFF:FFFF:FFFF:FFFF:FFFF 0 65535 fe80::6770:f9e7:add5:ed1c FFFF:FFFF:FFFF:FFFF:FFFF:FFFF:FFFF:FFFF 0 65535 0 0 1 64

16
test/ntcrule/full/2018-10-09/NTC_DNS_REGION.0000050997 Normal file
View File

@@ -0,0 +1,16 @@
0000000015
886 869 QNAME www.bing.com 0 0 0 1
68 68 QNAME book.qq.com 0 0 0 1
885 868 QNAME www.bing.com 0 0 0 1
67 67 QNAME www.cz88.net 0 0 0 1
883 866 QNAME youdao.com 0 0 0 1
881 864 QNAME hk.entertainment.appledaily.com 0 0 0 1
884 867 QNAME www.sina.com 0 0 0 1
70 70 QNAME chuangshi.qq.com 0 0 0 1
445 431 QNAME finance.eastmoney.com 0 0 0 1
8 8 QNAME www.sina.com 0 0 0 1
66 66 QNAME www.ip138.com 0 0 0 1
7 7 QNAME www.sohu.com 0 0 0 1
446 432 QNAME stock.eastmoney.com 0 0 0 1
882 865 QNAME youdao.com 0 0 0 1
880 863 QNAME hk.entertainment.appledaily.com 0 0 0 1

3
test/ntcrule/full/2018-10-09/NTC_DNS_RES_STRATEGY.0000050997 Normal file
View File

@@ -0,0 +1,3 @@
0000000002
8 101 dns_response1_policy 133 1 0 0 0 0 0 0 0 0 10 30 1 65
193222 105 STRATEGY_NAME 143 89 0 0 0 0 0 0 0 0 12 24 1 65

6
test/ntcrule/full/2018-10-09/NTC_FTP_URL.0000050997 Normal file
View File

@@ -0,0 +1,6 @@
0000000005
902 885 blockchain 0 0 0 1
901 884 aaaftpbbbtestccc 0 0 0 1
879 862 斩首 0 0 0 1
903 886 movie 0 0 0 1
900 883 blockchain_guide 0 0 0 1

105
test/ntcrule/full/2018-10-09/NTC_GROUP2COMPILE.0000050997 Normal file
View File

@@ -0,0 +1,105 @@
0000000103
828 193099 1 0 null 1
648 192999 1 0 null 1
869 193126 1 0 null 1
847 193114 1 0 null 1
620 192973 1 0 null 1
834 193105 1 0 null 1
551 170487 1 0 null 1
836 193106 1 0 null 1
723 193040 1 0 null 1
867 193128 1 0 null 1
607 192965 1 0 null 1
624 192976 1 0 null 1
991 193252 1 0 null 1
827 193098 1 0 null 1
861 193133 1 0 null 1
843 193112 1 0 null 1
854 193119 1 0 null 1
611 192967 1 0 null 1
890 193154 1 0 null 1
728 193039 1 0 null 1
980 193237 1 0 null 1
849 193115 1 0 null 1
806 193076 1 0 null 1
820 193091 1 0 null 1
8 11 1 0 null 1
845 193113 1 0 null 1
66 118 1 0 null 1
614 192966 1 0 null 1
610 192967 1 0 null 1
612 192966 1 0 null 1
855 193120 1 0 null 1
982 193236 1 0 null 1
884 193148 1 0 null 1
70 121 1 0 null 1
831 193102 1 0 null 1
856 193121 1 0 null 1
881 193145 1 0 null 1
838 193108 1 0 null 1
873 193138 1 0 null 1
851 193116 1 0 null 1
623 192976 1 0 null 1
68 120 1 0 null 1
605 192960 1 0 null 1
983 193236 1 0 null 1
993 193258 1 0 null 1
979 193234 1 0 null 1
816 193088 1 0 null 1
7 12 1 0 null 1
603 192960 1 0 null 1
981 193237 1 0 null 1
431 170435 1 0 null 1
846 193114 1 0 null 1
550 170486 1 0 null 1
649 192977 1 0 null 1
919 193171 1 0 null 1
864 193131 1 0 null 1
20 32 1 0 null 1
865 193130 1 0 null 1
724 193040 1 0 null 1
1021 19328 0 null 19 1
829 193101 1 0 null 1
868 193127 1 0 null 1
805 193077 1 0 null 1
613 192966 1 0 null 1
883 193147 1 0 null 1
647 193000 1 0 null 1
726 193039 1 0 null 1
862 193134 1 0 null 1
994 193258 1 0 null 1
549 170485 1 0 null 1
837 193107 1 0 null 1
863 193132 1 0 null 1
727 193039 1 0 null 1
409 441 1 0 null 1
833 193103 1 0 null 1
608 192965 1 0 null 1
650 193002 1 0 null 1
844 193112 1 0 null 1
625 192978 1 0 null 1
432 170436 1 0 null 1
67 116 1 0 null 1
891 193155 1 0 null 1
598 192959 1 0 null 1
850 193116 1 0 null 1
609 192967 1 0 null 1
835 193106 1 0 null 1
885 193149 1 0 null 1
725 193040 1 0 null 1
615 192968 1 0 null 1
886 193150 1 0 null 1
880 193144 1 0 null 1
606 192965 1 0 null 1
876 193140 1 0 null 1
1034 19329 0 null 14 1
840 193110 1 0 null 1
839 193108 1 0 null 1
832 193103 1 0 null 1
1020 19328 0 null 19 1
866 193129 1 0 null 1
604 192960 1 0 null 1
830 193102 1 0 null 1
875 193139 1 0 null 1
848 193115 1 0 null 1
0 null 1

5
test/ntcrule/full/2018-10-09/NTC_HTTP_REQ_BODY.0000050997 Normal file
View File

@@ -0,0 +1,5 @@
0000000004
1009 994 处女座从学习寻找自我 0 0 0 1
856 839 亦庄 0 0 0 1
1036 1021 金牛座&стейк&Taurus 1 0 0 1
908 891 王守仁 0 0 0 1

15
test/ntcrule/full/2018-10-09/NTC_HTTP_RES_BODY.0000050997 Normal file
View File

@@ -0,0 +1,15 @@
0000000014
845 828 girls 0 0 0 1
851 834 冰毒 0 0 0 1
857 840 冰糖 0 0 0 1
872 855 钓鱼 0 0 0 1
873 856 zmtests 0 0 0 1
878 861 斩首 0 0 0 1
907 890 2018-10-05 0 0 0 1
1006 991 李白 0 0 0 1
897 880 zmtests 0 0 0 1
890 873 zmtests 0 0 0 1
898 881 功能测试 0 0 0 1
871 854 春眠 0 0 0 1
892 875 girl 0 0 0 1
844 827 girl&is&can&a 1 0 0 1

10
test/ntcrule/full/2018-10-09/NTC_HTTP_URL.0000050997 Normal file
View File

@@ -0,0 +1,10 @@
0000000009
636 620 www.chinaso.com 0 0 0 1
855 838 www.chinaso.com 0 0 0 1
862 845 192.168.17.7:8080/website1/index.html 0 0 0 1
936 919 www.v6test.com 0 0 0 1
994 979 www.chinaso.com/search/pagesearch.htm?q 0 0 0 1
854 837 www.bing.com 0 0 0 1
893 876 www.arocmag.com 0 0 0 1
1008 993 astro.sina.com.cn/l/2013-05-24/101093841.shtml 0 0 0 1
1035 1020 www.chinaso.com 0 0 0 1

3
test/ntcrule/full/2018-10-09/NTC_MAIL_BODY.0000050997 Normal file
View File

@@ -0,0 +1,3 @@
0000000002
663 647 Content shell 0 0 0 1
640 624 Content shell 0 0 0 1

15
test/ntcrule/full/2018-10-09/NTC_MAIL_HDR.0000050997 Normal file
View File

@@ -0,0 +1,15 @@
0000000014
641 625 From @126.com 0 0 0 1
565 549 Subject sports 0 0 0 1
666 650 From whale 0 0 0 1
823 806 From gov.com 0 0 0 1
833 816 From hu_kwei@zmtests.com 0 0 0 1
639 623 From whale 0 0 0 1
566 550 Subject blogger 0 0 0 1
822 805 From ungov.com 0 0 0 1
567 551 Subject music 0 0 0 1
664 648 To hasake 0 0 0 1
837 820 To hu_kwei@zmtests.com 0 0 0 1
1049 1034 From ntc_test123@163.com 0 0 0 1
665 649 From whale 0 0 0 1
846 829 From @126.com 0 0 0 1

24
test/ntcrule/full/2018-10-09/NTC_UNIVERSAL_IP.0000050997 Normal file
View File

@@ -0,0 +1,24 @@
0000000023
977 962 4 10.11.36.21 255.255.255.255 0 65535 0.0.0.0 255.255.255.255 0 65535 0 0 1
1000 985 4 10.11.36.21 255.255.255.255 0 65535 0.0.0.0 255.255.255.255 0 65535 0 0 1
967 950 4 10.11.36.21 255.255.255.255 0 65535 0.0.0.0 255.255.255.255 0 65535 0 0 1
852 835 6 :: FFFF:FFFF:FFFF:FFFF:FFFF:FFFF:FFFF:FFFF 0 65535 fc00::1:1f FFFF:FFFF:FFFF:FFFF:FFFF:FFFF:FFFF:FFFF 0 65535 0 0 1
992 977 4 10.11.36.21 255.255.255.255 0 65535 0.0.0.0 255.255.255.255 0 65535 0 0 1
863 846 4 0.0.0.0 255.255.255.255 0 65535 192.168.17.3 255.255.255.255 0 65535 0 0 1
849 832 4 0.0.0.0 255.255.255.255 0 65535 192.168.17.3 255.255.255.255 0 65535 0 0 1
995 980 4 10.11.36.59 255.255.255.255 0 65535 0.0.0.0 255.255.255.255 0 65535 0 0 1
931 914 4 10.11.36.21 255.255.255.255 0 65535 0.0.0.0 255.255.255.255 0 65535 0 0 1
739 723 4 10.11.36.26 255.255.255.255 0 65535 0.0.0.0 255.255.255.255 0 65535 0 0 1
619 603 4 10.11.36.26 255.255.255.255 0 65535 0.0.0.0 255.255.255.255 0 65535 0 0 1
742 726 4 10.3.57.1 255.255.255.255 0 65535 0.0.0.0 255.255.255.255 0 65535 0 0 1
938 921 4 10.11.36.5 255.255.255.255 0 65535 0.0.0.0 255.255.255.255 0 65535 0 0 1
997 982 4 10.11.36.59 255.255.255.255 0 65535 0.0.0.0 255.255.255.255 0 65535 0 0 1
625 609 4 10.3.57.1 255.255.255.255 2345 65535 0.0.0.0 255.255.255.255 0 65535 0 0 1
847 830 6 :: FFFF:FFFF:FFFF:FFFF:FFFF:FFFF:FFFF:FFFF 0 65535 fc00::1:1f FFFF:FFFF:FFFF:FFFF:FFFF:FFFF:FFFF:FFFF 0 65535 0 0 1
865 848 4 0.0.0.0 255.255.255.255 0 65535 192.168.17.3 255.255.255.255 0 65535 0 0 1
986 971 4 10.11.36.21 255.255.255.255 0 65535 0.0.0.0 255.255.255.255 0 65535 0 0 1
628 612 4 10.3.57.1 255.255.255.255 56345 65535 10.3.57.2 255.255.255.255 179 65535 0 0 1
974 959 4 10.11.36.21 255.255.255.255 0 65535 0.0.0.0 255.255.255.255 0 65535 0 0 1
622 606 4 10.3.57.1 255.255.255.255 0 65535 0.0.0.0 255.255.255.255 0 65535 0 0 1
867 850 6 :: FFFF:FFFF:FFFF:FFFF:FFFF:FFFF:FFFF:FFFF 0 65535 fc00::1:1f FFFF:FFFF:FFFF:FFFF:FFFF:FFFF:FFFF:FFFF 0 65535 0 0 1
860 843 6 :: FFFF:FFFF:FFFF:FFFF:FFFF:FFFF:FFFF:FFFF 0 65535 fc00::1:1f FFFF:FFFF:FFFF:FFFF:FFFF:FFFF:FFFF:FFFF 0 65535 0 0 1

24
test/ntcrule/full/2018-10-09/NTC_UNIVERSAL_PROTO_TYPE.0000050997 Normal file
View File

@@ -0,0 +1,24 @@
0000000023
939 922 21 21 1
978 963 21 21 1
743 727 20 20 1
968 951 21 21 1
996 981 5 5 1
853 836 10 10 1
629 613 20 20 1
620 604 20 20 1
626 610 20 20 1
998 983 4 4 1
864 847 7 7 1
932 915 21 21 1
987 972 21 21 1
866 849 5 5 1
861 844 7 7 1
740 724 20 20 1
993 978 21 21 1
848 831 6 6 1
1001 986 21 21 1
850 833 6 6 1
868 851 5 5 1
623 607 20 20 1
975 960 21 21 1

2
test/ntcrule/full/2018-10-09/WHITE_LIST_COMPILE.0000050997 Normal file
View File

@@ -0,0 +1,2 @@
0000000001
128 1 128 1 0 0 0 1 1 0

2
test/ntcrule/full/2018-10-09/WHITE_LIST_GROUP.0000050997 Normal file
View File

@@ -0,0 +1,2 @@
0000000001
81 128 1

2
test/ntcrule/full/2018-10-09/WHITE_LIST_IP.0000050997 Normal file
View File

@@ -0,0 +1,2 @@
0000000001
81 81 4 10.11.36.7 255.255.255.255 22222 65535 192.168.17.4 255.255.255.255 80 65535 0 0 1

26
test/ntcrule/full/index/full_config_index.0000050997 Normal file
View File

@@ -0,0 +1,26 @@
APP_COMPILE 21 ./ntcrule/full/2018-10-09/APP_COMPILE.0000050997
APP_GROUP 37 ./ntcrule/full/2018-10-09/APP_GROUP.0000050997
APP_PAYLOAD 1 ./ntcrule/full/2018-10-09/APP_PAYLOAD.0000050997
APP_POLICY 16 ./ntcrule/full/2018-10-09/APP_POLICY.0000050997
DDOS_PROTECT_TARGET_IP_CB 1 ./ntcrule/full/2018-10-09/DDOS_PROTECT_TARGET_IP_CB.0000050997
MM_AV_URL 1 ./ntcrule/full/2018-10-09/MM_AV_URL.0000050997
MM_COMPILE 1 ./ntcrule/full/2018-10-09/MM_COMPILE.0000050997
MM_GROUP 1 ./ntcrule/full/2018-10-09/MM_GROUP.0000050997
NTC_ASN_IP 1 ./ntcrule/full/2018-10-09/NTC_ASN_IP.0000050997
NTC_BGP_AS 10 ./ntcrule/full/2018-10-09/NTC_BGP_AS.0000050997
NTC_COMPILE 78 ./ntcrule/full/2018-10-09/NTC_COMPILE.0000050997
NTC_DNS_FAKE_IP_CB 3 ./ntcrule/full/2018-10-09/NTC_DNS_FAKE_IP_CB.0000050997
NTC_DNS_REGION 15 ./ntcrule/full/2018-10-09/NTC_DNS_REGION.0000050997
NTC_DNS_RES_STRATEGY 2 ./ntcrule/full/2018-10-09/NTC_DNS_RES_STRATEGY.0000050997
NTC_FTP_URL 5 ./ntcrule/full/2018-10-09/NTC_FTP_URL.0000050997
NTC_GROUP2COMPILE 103 ./ntcrule/full/2018-10-09/NTC_GROUP2COMPILE.0000050997
NTC_HTTP_REQ_BODY 4 ./ntcrule/full/2018-10-09/NTC_HTTP_REQ_BODY.0000050997
NTC_HTTP_RES_BODY 14 ./ntcrule/full/2018-10-09/NTC_HTTP_RES_BODY.0000050997
NTC_HTTP_URL 9 ./ntcrule/full/2018-10-09/NTC_HTTP_URL.0000050997
NTC_MAIL_BODY 2 ./ntcrule/full/2018-10-09/NTC_MAIL_BODY.0000050997
NTC_MAIL_HDR 14 ./ntcrule/full/2018-10-09/NTC_MAIL_HDR.0000050997
NTC_UNIVERSAL_IP 23 ./ntcrule/full/2018-10-09/NTC_UNIVERSAL_IP.0000050997
NTC_UNIVERSAL_PROTO_TYPE 23 ./ntcrule/full/2018-10-09/NTC_UNIVERSAL_PROTO_TYPE.0000050997
WHITE_LIST_COMPILE 1 ./ntcrule/full/2018-10-09/WHITE_LIST_COMPILE.0000050997
WHITE_LIST_GROUP 1 ./ntcrule/full/2018-10-09/WHITE_LIST_GROUP.0000050997
WHITE_LIST_IP 1 ./ntcrule/full/2018-10-09/WHITE_LIST_IP.0000050997

2
test/table_info.conf
View File

@@ -89,7 +89,7 @@
"table_type":"expr",
"valid_column":7,
"custom": {
"scan_mode":"block",
"scan_mode":"stream",
"pattern_type":"regex",
"item_id":1,
"group_id":2,