diff --git a/CMakeLists.txt b/CMakeLists.txt index a449375..53adfcf 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -3,9 +3,9 @@ project(maatframe) include_directories(${PROJECT_SOURCE_DIR}/include) -set(CMAKE_C_FLAGS "-std=gnu99 -Wall") -set(CMAKE_CXX_FLAGS ${CMAKE_CXX_FLAGS} -Wall) - +set(CMAKE_C_FLAGS "-std=gnu99 -fgnu89-inline -fPIC -Wall") +set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fPIC -Wall") +set(MAAT_DEPEND_DYN_LIB pthread m crypto z) include_directories(include) enable_testing() diff --git a/scanner/bool_matcher.h b/include/bool_matcher.h similarity index 100% rename from scanner/bool_matcher.h rename to include/bool_matcher.h diff --git a/include/maat/maat.h b/include/maat.h similarity index 96% rename from include/maat/maat.h rename to include/maat.h index c6bc624..87ff79b 100644 --- a/include/maat/maat.h +++ b/include/maat.h @@ -16,7 +16,7 @@ #ifndef _MAAT_H_ #define _MAAT_H_ -#ifdef __cpluscplus +#ifdef __cplusplus extern "C" { #endif @@ -48,6 +48,18 @@ enum maat_scan_status { MAAT_SCAN_HIT //scan hit compile }; +#define MAX_SERVICE_DEFINE_LEN 128 +struct maat_rule { + int config_id; + int service_id; + uint8_t do_log; + uint8_t do_blacklist; + uint8_t action; + uint8_t reserved; + int serv_def_len; + char service_defined[MAX_SERVICE_DEFINE_LEN]; +}; + typedef void maat_start_callback_t(int update_type, void *u_param); typedef void maat_update_callback_t(int table_id, const char *table_line, void *u_para); typedef void maat_finish_callback_t(void *u_para); @@ -176,7 +188,7 @@ void maat_state_free(struct maat_state **state); /* return hit object compile_id */ int maat_hit_object_compile_id(struct maat *instance, struct maat_hit_object *obj); -#ifdef __cpluscplus +#ifdef __cplusplus } #endif diff --git a/scanner/CMakeLists.txt b/scanner/CMakeLists.txt index d07b230..5369c2c 100644 --- a/scanner/CMakeLists.txt +++ b/scanner/CMakeLists.txt @@ -6,4 +6,4 @@ include_directories(${PROJECT_SOURCE_DIR}/include) include_directories(${PROJECT_SOURCE_DIR}/src/inc_internal) add_library(adapter-static adapter_hs.cpp bool_matcher.cpp fqdn_engine.cpp) -target_link_libraries(adapter-static hyperscan_static hyperscan_runtime_static ipmatcher-static) +target_link_libraries(adapter-static hyperscan_static hyperscan_runtime_static ipmatcher-static) \ No newline at end of file diff --git a/scanner/adapter_hs.cpp b/scanner/adapter_hs.cpp index d669896..6d22301 100644 --- a/scanner/adapter_hs.cpp +++ b/scanner/adapter_hs.cpp @@ -75,40 +75,40 @@ struct adapter_hs_stream { }; struct matched_pattern { - unsigned int pattern_id; + unsigned long long pattern_id; unsigned long matched_l_offset; unsigned long matched_r_offset; UT_hash_handle hh; }; -struct matched_pattern_set { +struct matched_pattern_container { UT_array *pat_ids; - unsigned int pattern_id; + unsigned long long pattern_id; unsigned long long l_matched; unsigned long long r_matched; struct matched_pattern *pat_hash; }; -struct pattern_offset { - unsigned int pattern_id; - unsigned long l_offset; - unsigned long r_offset; +struct pattern_attribute { + unsigned long long pattern_id; + enum hs_match_mode match_mode; + int l_offset; + int r_offset; }; struct hs_tag { - size_t n_pat_offset; - struct pattern_offset *pat_offset; + size_t n_pat_attr; + struct pattern_attribute *pat_attr; void *user_tag; }; -static int adpt_hs_alloc_scratch(struct adapter_hs_runtime *hs_rt, - size_t n_worker_thread, int pattern_type, - struct log_handle *logger) +static int adpt_hs_alloc_scratch(struct adapter_hs_runtime *hs_rt, size_t n_worker_thread, + enum hs_pattern_type pattern_type, struct log_handle *logger) { hs_database_t *database = NULL; hs_rt->scratchs = ALLOC(hs_scratch_t *, n_worker_thread); - if (pattern_type == PATTERN_TYPE_STR) { + if (pattern_type == HS_PATTERN_TYPE_STR) { database = hs_rt->literal_db; } else { database = hs_rt->regex_db; @@ -146,21 +146,22 @@ static int adpt_hs_alloc_scratch(struct adapter_hs_runtime *hs_rt, * @retval 0(success) -1(failed) */ static int adpt_hs_build_database(struct adapter_hs_runtime *hs_rt, - struct adpt_hs_compile_data *literal_cd, - struct adpt_hs_compile_data *regex_cd, - int scan_mode, struct log_handle *logger) + struct adpt_hs_compile_data *compile_data, + enum hs_pattern_type pattern_type, + enum hs_scan_mode scan_mode, + struct log_handle *logger) { hs_error_t err; hs_compile_error_t *compile_err = NULL; - if (NULL == hs_rt) { + if (NULL == hs_rt || NULL == compile_data) { return -1; } - if (literal_cd != NULL) { - err = hs_compile_lit_multi((const char *const *)literal_cd->patterns, literal_cd->flags, - literal_cd->ids, literal_cd->pattern_lens, literal_cd->n_patterns, - scan_mode, NULL, &hs_rt->literal_db, &compile_err); + if (pattern_type == HS_PATTERN_TYPE_STR) { + err = hs_compile_lit_multi((const char *const *)compile_data->patterns, compile_data->flags, + compile_data->ids, compile_data->pattern_lens, compile_data->n_patterns, + scan_mode, NULL, &hs_rt->literal_db, &compile_err); if (err != HS_SUCCESS) { if (compile_err) { log_error(logger, MODULE_ADAPTER_HS, @@ -168,37 +169,23 @@ static int adpt_hs_build_database(struct adapter_hs_runtime *hs_rt, } hs_free_compile_error(compile_err); - goto error; + return -1; } - } - - if (regex_cd != NULL) { - err = hs_compile_ext_multi((const char *const *)regex_cd->patterns, regex_cd->flags, - regex_cd->ids, NULL, regex_cd->n_patterns, - scan_mode, NULL, &hs_rt->regex_db, &compile_err); + } else { + err = hs_compile_ext_multi((const char *const *)compile_data->patterns, compile_data->flags, + compile_data->ids, NULL, compile_data->n_patterns, + scan_mode, NULL, &hs_rt->regex_db, &compile_err); if (err != HS_SUCCESS) { if (compile_err) { log_error(logger, MODULE_ADAPTER_HS, "%s compile error: %s", __func__, compile_err->message); } hs_free_compile_error(compile_err); - goto error; + return -1; } } - - return 0; -error: - if (hs_rt->literal_db != NULL) { - hs_free_database(hs_rt->literal_db); - hs_rt->literal_db = NULL; - } - - if (hs_rt->regex_db != NULL) { - hs_free_database(hs_rt->regex_db); - hs_rt->regex_db = NULL; - } - return -1; + return 0; } struct adpt_hs_compile_data *adpt_hs_compile_data_new(size_t n_patterns) @@ -232,172 +219,130 @@ void adpt_hs_compile_data_free(struct adpt_hs_compile_data *hs_cd, size_t n_patt FREE(hs_cd); } -struct adapter_hs * -adapter_hs_initialize(int scan_mode, size_t n_worker_thread, - and_expr_t *expr_array, size_t n_expr_array, - struct log_handle *logger) +struct adapter_hs *adapter_hs_initialize(enum hs_scan_mode scan_mode, + enum hs_pattern_type pattern_type, + size_t n_worker_thread, + and_expr_t *exprs, size_t n_expr, + struct log_handle *logger) { if ((scan_mode != HS_SCAN_MODE_BLOCK && scan_mode != HS_SCAN_MODE_STREAM) || - 0 == n_worker_thread || NULL == expr_array || 0 == n_expr_array) { + (pattern_type != HS_PATTERN_TYPE_STR && pattern_type != HS_PATTERN_TYPE_REG) || + 0 == n_worker_thread || NULL == exprs || 0 == n_expr) { log_error(logger, MODULE_ADAPTER_HS, "%s input parameters illegal!", __func__); return NULL; } /* get the sum of pattern */ - size_t literal_pattern_num = 0; - size_t regex_pattern_num = 0; + size_t pattern_num = 0; - for (size_t i = 0; i < n_expr_array; i++) { - if (expr_array[i].n_patterns > MAX_EXPR_PATTERN_NUM) { + for (size_t i = 0; i < n_expr; i++) { + if (exprs[i].n_patterns > MAX_EXPR_PATTERN_NUM) { log_error(logger, MODULE_ADAPTER_HS, "the number of patterns in one expression should less than %d", MAX_EXPR_PATTERN_NUM); return NULL; } - for (size_t j = 0; j < expr_array[i].n_patterns; j++) { - /* pat_len should not 0 */ - if (0 == expr_array[i].patterns[j].pat_len) { + for (size_t j = 0; j < exprs[i].n_patterns; j++) { + if (0 == exprs[i].patterns[j].pat_len) { + log_error(logger, MODULE_ADAPTER_HS, "expr pattern length should not 0"); return NULL; } - if (expr_array[i].patterns[j].type == PATTERN_TYPE_STR) { - literal_pattern_num++; - } else if (expr_array[i].patterns[j].type == PATTERN_TYPE_REG) { - regex_pattern_num++; - } else { - log_error(logger, MODULE_ADAPTER_HS, "unknown pattern type: %d", - expr_array[i].patterns[j].type); - return NULL; - } + pattern_num++; } } - struct adpt_hs_compile_data *literal_cd = NULL; - struct adpt_hs_compile_data *regex_cd = NULL; - if (literal_pattern_num > 0) { - literal_cd = adpt_hs_compile_data_new(literal_pattern_num); + if (0 == pattern_num) { + log_error(logger, MODULE_ADAPTER_HS, "expr array has no valid pattern"); + return NULL; } - - if (regex_pattern_num > 0) { - regex_cd = adpt_hs_compile_data_new(regex_pattern_num); - } - - uint32_t literal_index = 0; - uint32_t regex_index = 0; - uint32_t pattern_id = 0; - /* alloc exprs for bool matcher*/ - struct bool_expr *exprs = ALLOC(struct bool_expr, n_expr_array); + struct adpt_hs_compile_data *compile_data = NULL; + compile_data = adpt_hs_compile_data_new(pattern_num); + + uint32_t pattern_index = 0; + struct bool_expr *bool_exprs = ALLOC(struct bool_expr, n_expr); /* populate adpt_hs_compile_data and bool_expr */ - for (size_t i = 0; i < n_expr_array; i++) { + for (size_t i = 0; i < n_expr; i++) { struct hs_tag *hs_tag = ALLOC(struct hs_tag, 1); - hs_tag->pat_offset = ALLOC(struct pattern_offset, expr_array[i].n_patterns); - hs_tag->n_pat_offset = expr_array[i].n_patterns; - hs_tag->user_tag = expr_array[i].user_tag; + hs_tag->pat_attr = ALLOC(struct pattern_attribute, exprs[i].n_patterns); + hs_tag->n_pat_attr = exprs[i].n_patterns; + hs_tag->user_tag = exprs[i].user_tag; - for (size_t j = 0; j < expr_array[i].n_patterns; j++) { + for (size_t j = 0; j < exprs[i].n_patterns; j++) { size_t pat_len = 0; - hs_tag->pat_offset[j].pattern_id = pattern_id; - hs_tag->pat_offset[j].l_offset = expr_array[i].patterns[j].l_offset; - hs_tag->pat_offset[j].r_offset = expr_array[i].patterns[j].r_offset; - - if (expr_array[i].patterns[j].type == PATTERN_TYPE_STR) { - literal_cd->ids[literal_index] = pattern_id; - - /* set flags */ - literal_cd->flags[literal_index] = HS_FLAG_SOM_LEFTMOST; - if (expr_array[i].patterns[j].case_sensitive == 1) { - literal_cd->flags[literal_index] |= HS_FLAG_CASELESS; - } - - pat_len = expr_array[i].patterns[j].pat_len; - literal_cd->pattern_lens[literal_index] = pat_len; - literal_cd->patterns[literal_index] = ALLOC(char, pat_len); - memcpy(literal_cd->patterns[literal_index], - expr_array[i].patterns[j].pat, - expr_array[i].patterns[j].pat_len); - literal_index++; - } else { - regex_cd->ids[regex_index] = pattern_id; - - /* set flags */ - regex_cd->flags[regex_index] = HS_FLAG_SOM_LEFTMOST; - if (expr_array[i].patterns[j].case_sensitive == 1) { - regex_cd->flags[literal_index] |= HS_FLAG_CASELESS; - } - - pat_len = expr_array[i].patterns[j].pat_len; - regex_cd->pattern_lens[regex_index] = pat_len; - regex_cd->patterns[regex_index] = ALLOC(char, pat_len); - memcpy(regex_cd->patterns[regex_index], - expr_array[i].patterns[j].pat, - expr_array[i].patterns[j].pat_len); - regex_index++; + hs_tag->pat_attr[j].pattern_id = pattern_index; + hs_tag->pat_attr[j].match_mode = exprs[i].patterns[j].match_mode; + if (exprs[i].patterns[j].match_mode == HS_MATCH_MODE_SUB) { + hs_tag->pat_attr[j].l_offset = exprs[i].patterns[j].l_offset; + hs_tag->pat_attr[j].r_offset = exprs[i].patterns[j].r_offset; } - exprs[i].items[j].item_id = pattern_id; - pattern_id++; - } - exprs[i].expr_id = expr_array[i].expr_id; - exprs[i].item_num = expr_array[i].n_patterns; - exprs[i].user_tag = hs_tag; - } - if (literal_cd != NULL) { - literal_cd->n_patterns = literal_index; - } - - if (regex_cd != NULL) { - regex_cd->n_patterns = regex_index; + compile_data->ids[pattern_index] = pattern_index; + compile_data->flags[pattern_index] = HS_FLAG_SOM_LEFTMOST; + if (exprs[i].patterns[j].case_sensitive == HS_CASE_INSESITIVE) { + compile_data->flags[pattern_index] |= HS_FLAG_CASELESS; + } + + pat_len = exprs[i].patterns[j].pat_len; + compile_data->pattern_lens[pattern_index] = pat_len; + compile_data->patterns[pattern_index] = ALLOC(char, pat_len); + memcpy(compile_data->patterns[pattern_index], exprs[i].patterns[j].pat, + exprs[i].patterns[j].pat_len); + + bool_exprs[i].items[j].item_id = pattern_index; + pattern_index++; + } + + bool_exprs[i].expr_id = exprs[i].expr_id; + bool_exprs[i].item_num = exprs[i].n_patterns; + bool_exprs[i].user_tag = hs_tag; } + compile_data->n_patterns = pattern_index; int ret = -1; - int max_patterns_type = 0; size_t mem_size = 0; struct adapter_hs *hs_instance = ALLOC(struct adapter_hs, 1); hs_instance->n_worker_thread = n_worker_thread; - hs_instance->n_patterns = pattern_id; - hs_instance->n_expr = n_expr_array; + hs_instance->n_patterns = pattern_index; + hs_instance->n_expr = n_expr; hs_instance->hs_rt = ALLOC(struct adapter_hs_runtime, 1); + //mytest + // for (size_t i = 0; i < n_expr_array; i++) { + // printf("exprs[%zu] expr_id:%llu, item_num:%zu\n", i, exprs[i].expr_id, exprs[i].item_num); + // for (size_t j = 0; j < exprs[i].item_num; j++) { + // printf("item[%zu] item_id: %llu\n", j, exprs[i].items[j].item_id); + // } + // } /* create bool matcher */ - hs_instance->hs_rt->bm = bool_matcher_new(exprs, n_expr_array, &mem_size); + hs_instance->hs_rt->bm = bool_matcher_new(bool_exprs, n_expr, &mem_size); if (hs_instance->hs_rt->bm != NULL) { log_info(logger, MODULE_ADAPTER_HS, "Adapter_hs module: build bool matcher of %zu expressions with %zu bytes memory", - n_expr_array, mem_size); + n_expr, mem_size); } else { log_error(logger, MODULE_ADAPTER_HS, "Adapter_hs module: build bool matcher failed"); goto error; } - FREE(exprs); + FREE(bool_exprs); /* build hs database */ - ret = adpt_hs_build_database(hs_instance->hs_rt, literal_cd, regex_cd, scan_mode, logger); + ret = adpt_hs_build_database(hs_instance->hs_rt, compile_data, pattern_type, scan_mode, logger); if (ret < 0) { goto error; } - if (literal_cd != NULL) { - adpt_hs_compile_data_free(literal_cd, literal_index); + if (compile_data != NULL) { + adpt_hs_compile_data_free(compile_data, pattern_index); } - if (regex_cd != NULL) { - adpt_hs_compile_data_free(regex_cd, regex_index); - } - - /* which pattern type has more patterns, use it as hs_alloc_scratch's input parameter */ - if (literal_pattern_num > regex_pattern_num) { - max_patterns_type = PATTERN_TYPE_STR; - } else { - max_patterns_type = PATTERN_TYPE_REG; - } - - ret = adpt_hs_alloc_scratch(hs_instance->hs_rt, n_worker_thread, max_patterns_type, logger); + ret = adpt_hs_alloc_scratch(hs_instance->hs_rt, n_worker_thread, pattern_type, logger); if (ret < 0) { goto error; } @@ -456,7 +401,7 @@ static inline int compare_pattern_id(const void *a, const void *b) } } -UT_icd ut_pattern_id_icd = {sizeof(unsigned int), NULL, NULL, NULL}; +UT_icd ut_pattern_id_icd = {sizeof(unsigned long long), NULL, NULL, NULL}; /** * @param id: pattern id */ @@ -464,24 +409,62 @@ int matched_event_cb(unsigned int id, unsigned long long from, unsigned long long to, unsigned int flags, void *ctx) { // put id in set - printf("matched_event_cb, expr_id:%u, from:%llu to:%llu\n", id, from, to); - - struct matched_pattern_set *matched_pattern_set = (struct matched_pattern_set *)ctx; - unsigned int pattern_id = id; + struct matched_pattern_container *matched_pat_container = (struct matched_pattern_container *)ctx; + unsigned long long pattern_id = id; - if (utarray_find(matched_pattern_set->pat_ids, &pattern_id, compare_pattern_id)) { + if (utarray_find(matched_pat_container->pat_ids, &pattern_id, compare_pattern_id)) { return -1; } - matched_pattern_set->pattern_id = pattern_id; - matched_pattern_set->l_matched = from; - matched_pattern_set->r_matched = to; - utarray_push_back(matched_pattern_set->pat_ids, &pattern_id); - utarray_sort(matched_pattern_set->pat_ids, compare_pattern_id); + utarray_push_back(matched_pat_container->pat_ids, &pattern_id); + utarray_sort(matched_pat_container->pat_ids, compare_pattern_id); + + struct matched_pattern *matched_pat = ALLOC(struct matched_pattern, 1); + matched_pat->pattern_id = pattern_id; + matched_pat->matched_l_offset = from; + matched_pat->matched_r_offset = to; + + HASH_ADD_INT(matched_pat_container->pat_hash, pattern_id, matched_pat); return 0; } +int is_real_matched_pattern(struct matched_pattern *matched_pat, enum hs_match_mode match_mode, + size_t data_len, int attr_l_offset, int attr_r_offset) +{ + if (match_mode == HS_MATCH_MODE_EXACTLY) { + if (matched_pat->matched_l_offset == 0 && + matched_pat->matched_r_offset == data_len) { + return 0; + } + } else if (match_mode == HS_MATCH_MODE_PREFIX) { + if (matched_pat->matched_l_offset == 0) { + return 0; + } + } else if (match_mode == HS_MATCH_MODE_SUFFIX) { + if (matched_pat->matched_r_offset == data_len) { + return 0; + } + } else if (match_mode == HS_MATCH_MODE_SUB) { + if (attr_l_offset == -1) { + attr_l_offset = 0; + } + + if (attr_r_offset == -1) { + attr_r_offset = (int)data_len; + } + + if (matched_pat->matched_l_offset >= (unsigned long)attr_l_offset && + matched_pat->matched_r_offset <= (unsigned long)attr_r_offset) { + return 0; + } + } else { + assert(0); + } + + return -1; +} + int adapter_hs_scan(struct adapter_hs *hs_instance, int thread_id, const char *data, size_t data_len, struct hs_scan_result *results, @@ -496,16 +479,16 @@ int adapter_hs_scan(struct adapter_hs *hs_instance, int thread_id, hs_scratch_t *scratch = hs_rt->scratchs[thread_id]; hs_error_t err; - struct matched_pattern_set matched_pat_set; + struct matched_pattern_container matched_pat_container; - matched_pat_set.pat_hash = NULL; - utarray_new(matched_pat_set.pat_ids, &ut_pattern_id_icd); - utarray_reserve(matched_pat_set.pat_ids, hs_instance->n_patterns); + matched_pat_container.pat_hash = NULL; + utarray_new(matched_pat_container.pat_ids, &ut_pattern_id_icd); + utarray_reserve(matched_pat_container.pat_ids, hs_instance->n_patterns); int err_count = 0; if (hs_rt->literal_db != NULL) { err = hs_scan(hs_rt->literal_db, data, data_len, 0, scratch, - matched_event_cb, &matched_pat_set); + matched_event_cb, &matched_pat_container); if (err != HS_SUCCESS) { //log_error() err_count++; @@ -514,7 +497,7 @@ int adapter_hs_scan(struct adapter_hs *hs_instance, int thread_id, if (hs_rt->regex_db != NULL) { err = hs_scan(hs_rt->regex_db, data, data_len, 0, scratch, - matched_event_cb, &matched_pat_set); + matched_event_cb, &matched_pat_container); if (err != HS_SUCCESS) { //log_error() err_count++; @@ -525,27 +508,20 @@ int adapter_hs_scan(struct adapter_hs *hs_instance, int thread_id, return -1; } - struct matched_pattern *matched_pat = ALLOC(struct matched_pattern, 1); - unsigned int pattern_id = matched_pat_set.pattern_id; - matched_pat->pattern_id = pattern_id; - matched_pat->matched_l_offset = matched_pat_set.l_matched; - matched_pat->matched_r_offset = matched_pat_set.r_matched; - - HASH_ADD_INT(matched_pat_set.pat_hash, pattern_id, matched_pat); - - size_t matched_pattern_ids_cnt = utarray_len(matched_pat_set.pat_ids); + size_t matched_pattern_ids_cnt = utarray_len(matched_pat_container.pat_ids); + size_t i = 0; unsigned long long items[matched_pattern_ids_cnt]; memset(items, 0, sizeof(unsigned long long) * matched_pattern_ids_cnt); - for (size_t i = 0; i < matched_pattern_ids_cnt; i++) { - items[i] = *(unsigned long long *)utarray_eltptr(matched_pat_set.pat_ids, i); + for (i = 0; i < matched_pattern_ids_cnt; i++) { + items[i] = *(unsigned long long *)utarray_eltptr(matched_pat_container.pat_ids, i); } int ret = 0; int matched_index = 0; int real_matched_index = 0; - struct bool_expr_match *bool_matcher_results = NULL; - bool_matcher_results = ALLOC(struct bool_expr_match, hs_instance->n_expr); + struct hs_tag *hs_tag = NULL; + struct bool_expr_match *bool_matcher_results = ALLOC(struct bool_expr_match, hs_instance->n_expr); int bool_matcher_ret = bool_matcher_match(hs_rt->bm, items, matched_pattern_ids_cnt, bool_matcher_results, hs_instance->n_expr); if (bool_matcher_ret < 0) { @@ -558,16 +534,20 @@ int adapter_hs_scan(struct adapter_hs *hs_instance, int thread_id, } for (matched_index = 0; matched_index < bool_matcher_ret; matched_index++) { - //results[matched_index].item_id = bool_matcher_results[matched_index].expr_id; - struct hs_tag *hs_tag = (struct hs_tag *)bool_matcher_results[matched_index].user_tag; - for (size_t i = 0; i < hs_tag->n_pat_offset; i++) { - //命中的 item = pat1 & pat2 - int pattern_id = hs_tag->pat_offset[i].pattern_id; - struct matched_pattern *matched_pat = NULL; - HASH_FIND_INT(matched_pat_set.pat_hash, &pattern_id, matched_pat); - if (matched_pat) { - if (matched_pat->matched_l_offset >= hs_tag->pat_offset[i].l_offset && - matched_pat->matched_r_offset <= hs_tag->pat_offset[i].r_offset) { + hs_tag = (struct hs_tag *)bool_matcher_results[matched_index].user_tag; + + /* check if real matched pattern, because pattern match_mode is different */ + for (i = 0; i < hs_tag->n_pat_attr; i++) { + struct matched_pattern *tmp_matched_pat = NULL; + int pattern_id = hs_tag->pat_attr[i].pattern_id; + HASH_FIND_INT(matched_pat_container.pat_hash, &pattern_id, tmp_matched_pat); + if (tmp_matched_pat) { + int matched_ret = is_real_matched_pattern(tmp_matched_pat, + hs_tag->pat_attr[i].match_mode, + data_len, + hs_tag->pat_attr[i].l_offset, + hs_tag->pat_attr[i].r_offset); + if (0 == matched_ret) { results[real_matched_index].item_id = bool_matcher_results[matched_index].expr_id; results[real_matched_index].user_tag = hs_tag->user_tag; real_matched_index++; @@ -576,15 +556,15 @@ int adapter_hs_scan(struct adapter_hs *hs_instance, int thread_id, } } } - *n_hit_result = bool_matcher_ret; + *n_hit_result = real_matched_index; next: FREE(bool_matcher_results); struct matched_pattern *pattern = NULL, *tmp_pattern = NULL; - HASH_ITER(hh, matched_pat_set.pat_hash, pattern, tmp_pattern) { - HASH_DELETE(hh, matched_pat_set.pat_hash, pattern); + HASH_ITER(hh, matched_pat_container.pat_hash, pattern, tmp_pattern) { + HASH_DELETE(hh, matched_pat_container.pat_hash, pattern); FREE(pattern); } - utarray_free(matched_pat_set.pat_ids); + utarray_free(matched_pat_container.pat_ids); return ret; } diff --git a/scanner/adapter_hs.h b/scanner/adapter_hs.h index 4c42e03..b2b81a5 100644 --- a/scanner/adapter_hs.h +++ b/scanner/adapter_hs.h @@ -11,7 +11,7 @@ #ifndef _ADAPTER_HS_H_ #define _ADAPTER_HS_H_ -#ifdef __cpluscplus +#ifdef __cplusplus extern "C" { #endif @@ -32,10 +32,24 @@ enum hs_scan_mode { HS_SCAN_MODE_MAX }; -/* pattern type: PATTERN_TYPE_STR(pure literal string) or PATTERN_TYPE_REG(regex expression) */ -enum pattern_type { - PATTERN_TYPE_STR = 0, - PATTERN_TYPE_REG, +/* match method */ +enum hs_match_mode { + HS_MATCH_MODE_EXACTLY = 1, /* scan data must match pattern exactly */ + HS_MATCH_MODE_PREFIX, /* pattern must in the head of scan_data */ + HS_MATCH_MODE_SUFFIX, /* pattern must in the end of scan_data */ + HS_MATCH_MODE_SUB, /* pattern must in the range[l_offset, r_offset] of scan_data */ + HS_MATCH_MODE_MAX +}; + +enum hs_pattern_type { + HS_PATTERN_TYPE_STR = 0, /* pure literal string */ + HS_PATTERN_TYPE_REG, /* regex expression */ + HS_PATTERN_TYPE_MAX +}; + +enum hs_case_sensitive { + HS_CASE_SENSITIVE = 0, + HS_CASE_INSESITIVE }; struct hs_scan_result { @@ -44,16 +58,15 @@ struct hs_scan_result { }; typedef struct { - /* pattern type */ - int type; + enum hs_case_sensitive case_sensitive; + enum hs_match_mode match_mode; - /* default 0(sensitive) 1(insensitive) */ - int case_sensitive; - - /* (0)sub string match (1)complete match */ - int match_mode; - - /* just match in scan_data's range of [l_offset, r_offset] */ + /* + * just match in scan_data's range of [l_offset, r_offset], -1 means no limits + * for example: + * [-1, r_offset] means the pattern must in scan_data's [0 ~ r_offset] + * [l_offset, -1] means the pattern must in scan_data's [l_offset ~ end] + */ int l_offset; int r_offset; @@ -81,10 +94,11 @@ typedef struct { * * @retval the pointer to adapter_hs instance */ -struct adapter_hs * -adapter_hs_initialize(int scan_mode, size_t n_worker_thread, - and_expr_t *expr_array, size_t n_expr_array, - struct log_handle *logger); +struct adapter_hs *adapter_hs_initialize(enum hs_scan_mode scan_mode, + enum hs_pattern_type pattern_type, + size_t n_worker_thread, + and_expr_t *exprs, size_t n_expr, + struct log_handle *logger); /** * @brief scan input data to match logic AND expression, return all matched expr_id @@ -120,7 +134,7 @@ int adapter_hs_scan_stream(struct adapter_hs_stream *stream, const char *data, s void adapter_hs_stream_close(struct adapter_hs_stream *stream); -#ifdef __cpluscplus +#ifdef __cplusplus } #endif diff --git a/scanner/fqdn_engine.cpp b/scanner/fqdn_engine.cpp index 99ef716..d601ae1 100644 --- a/scanner/fqdn_engine.cpp +++ b/scanner/fqdn_engine.cpp @@ -44,7 +44,7 @@ struct packedRT_t unsigned char B[4]; }; -void * aligned_malloc(size_t size, size_t align) +static void * aligned_malloc(size_t size, size_t align) { void * malloc_ptr; void * aligned_ptr; @@ -73,7 +73,7 @@ void * aligned_malloc(size_t size, size_t align) return aligned_ptr; } -void aligned_free(void * aligned_ptr) +static void aligned_free(void * aligned_ptr) { if (aligned_ptr) { diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 475ebf7..8bf591c 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -7,12 +7,13 @@ set(MAAT_FRAME_VERSION ${MAAT_FRAME_MAJOR_VERSION}.${MAAT_FRAME_MINOR_VERSION}.$ message(STATUS "Maat Frame, Version: ${MAAT_FRAME_VERSION}") +add_definitions(-D_GNU_SOURCE) add_definitions(-fPIC) -set(MAAT_SRC json2iris.cpp maat_api.cpp rcu_hash.cpp maat_garbage_collection.cpp maat_config_monitor.cpp - maat_rule.cpp maat_kv.cpp maat_ex_data.cpp maat_utils.cpp maat_command.cpp maat_redis_monitor.cpp - maat_table.cpp maat_compile.cpp maat_group.cpp maat_ip.cpp maat_flag.cpp maat_interval.cpp - maat_expr.cpp maat_plugin.cpp maat_ip_plugin.cpp maat_bool_plugin.cpp maat_fqdn_plugin.cpp - maat_virtual.cpp) +set(MAAT_SRC alignment.c json2iris.c maat_api.c rcu_hash.c maat_garbage_collection.c maat_config_monitor.c + maat_rule.c maat_kv.c maat_ex_data.c maat_utils.c maat_command.c maat_redis_monitor.c + maat_table.c maat_compile.c maat_group.c maat_ip.c maat_flag.c maat_interval.c + maat_expr.c maat_fqdn.c maat_port_proto.c maat_plugin.c maat_ip_plugin.c maat_bool_plugin.c + maat_fqdn_plugin.c maat_virtual.c) set(LIB_SOURCE_FILES ${PROJECT_SOURCE_DIR}/deps/cJSON/cJSON.c ${PROJECT_SOURCE_DIR}/deps/log/log.c) @@ -28,18 +29,31 @@ set_target_properties(maat_frame_static PROPERTIES LINKER_LANGUAGE CXX) set_target_properties(maat_frame_static PROPERTIES OUTPUT_NAME maatframe) set_target_properties(maat_frame_static PROPERTIES CLEAN_DIRECT_OUTPUT 1) -target_link_libraries(maat_frame_static adapter-static hiredis-static igraph-static flagmatcher-static intervalmatcher-static pthread crypto z) +target_link_libraries(maat_frame_static hiredis-static ${MAAT_DEPEND_DYN_LIB}) +target_link_libraries(maat_frame_static igraph-static) +target_link_libraries(maat_frame_static ipmatcher-static) +target_link_libraries(maat_frame_static adapter-static) +target_link_libraries(maat_frame_static flagmatcher-static) +target_link_libraries(maat_frame_static intervalmatcher-static) # Shared Library Output -#add_library(maat_frame_shared SHARED ${MAAT_SRC} ${LIB_SOURCE_FILES}) -#set_target_properties(maat_frame_shared PROPERTIES LINKER_LANGUAGE CXX) -#set_target_properties(maat_frame_shared PROPERTIES OUTPUT_NAME maatframe) -#set_target_properties(maat_frame_shared PROPERTIES CLEAN_DIRECT_OUTPUT 1) -#set_target_properties(maat_frame_shared PROPERTIES VERSION ${MAAT_FRAME_MAJOR_VERSION}.${MAAT_FRAME_MINOR_VERSION}) -#set_target_properties(maat_frame_shared PROPERTIES SOVERSION ${MAAT_FRAME_MAJOR_VERSION}) +add_library(maat_frame_shared SHARED ${MAAT_SRC} ${LIB_SOURCE_FILES}) +set_target_properties(maat_frame_shared PROPERTIES LINKER_LANGUAGE CXX) +set_target_properties(maat_frame_shared PROPERTIES OUTPUT_NAME maatframe) +set_target_properties(maat_frame_shared PROPERTIES CLEAN_DIRECT_OUTPUT 1) +set_target_properties(maat_frame_shared PROPERTIES VERSION ${MAAT_FRAME_MAJOR_VERSION}.${MAAT_FRAME_MINOR_VERSION}) +set_target_properties(maat_frame_shared PROPERTIES SOVERSION ${MAAT_FRAME_MAJOR_VERSION}) +set_target_properties(maat_frame_shared PROPERTIES LINK_FLAGS "-Wl,--version-script=${CMAKE_CURRENT_LIST_DIR}/version.map") -#target_link_libraries(maat_frame_shared hyperscan_static) +target_link_libraries(maat_frame_shared hiredis-static ${MAAT_DEPEND_DYN_LIB}) +target_link_libraries(maat_frame_shared igraph-static ${MAAT_DEPEND_DYN_LIB}) +target_link_libraries(maat_frame_shared ipmatcher-static) +target_link_libraries(maat_frame_shared adapter-static) +target_link_libraries(maat_frame_shared flagmatcher-static) +target_link_libraries(maat_frame_shared intervalmatcher-static) # install set(CMAKE_INSTALL_PREFIX /opt/MESA/) -#install(FILES ${PROJECT_SOURCE_DIR}/include/maat DESTINATION ${CMAKE_INSTALL_PREFIX}/include/MESA/ COMPONENT HEADER) \ No newline at end of file +install(FILES ${PROJECT_SOURCE_DIR}/include/maat.h DESTINATION ${CMAKE_INSTALL_PREFIX}/include/MESA/ COMPONENT HEADER) +install(FILES ${PROJECT_SOURCE_DIR}/include/bool_matcher.h DESTINATION ${CMAKE_INSTALL_PREFIX}/include/MESA/ COMPONENT HEADER) +install(TARGETS maat_frame_shared LIBRARY DESTINATION ${CMAKE_INSTALL_PREFIX}/lib/ COMPONENT LIBRARIES) \ No newline at end of file diff --git a/src/alignment.c b/src/alignment.c new file mode 100644 index 0000000..c6106ba --- /dev/null +++ b/src/alignment.c @@ -0,0 +1,67 @@ +/* +********************************************************************************************** +* File: alignment.c +* Description: +* Authors: Zheng Chao +* Date: 2022-10-31 +* Copyright: (c) 2018-2022 Geedge Networks, Inc. All rights reserved. +*********************************************************************************************** +*/ + +#include +#include + +#include "alignment.h" + +#define CPU_CACHE_ALIGMENT 64 + +inline long long *alignment_int64_array_alloc(int size) +{ + return (long long *)calloc(CPU_CACHE_ALIGMENT, size); +} + +inline long long alignment_int64_array_sum(long long *array, int size) +{ + int offset = 0; + long long sum = 0; + + for (int i = 0; i < size; i++) { + offset = (CPU_CACHE_ALIGMENT / sizeof(long long)) * i; + sum += array[offset]; + } + + return sum; +} + +inline long long alignment_int64_array_add(long long *array, int offset, long long op_val) +{ + int idx = (CPU_CACHE_ALIGMENT / sizeof(long long)) * offset; + array[idx] += op_val; + + return array[idx]; +} + +inline long long alignment_int64_array_cnt(long long *array, int size) +{ + int offset = 0; + int cnt = 0; + + for (int i = 0; i < size; i++) { + offset = (CPU_CACHE_ALIGMENT / sizeof(long long)) * i; + if (array[offset] > 0) { + cnt++; + } + } + + return cnt; +} + +inline void alignment_int64_array_reset(long long *array, int size) +{ + memset(array, 0, CPU_CACHE_ALIGMENT * size); +} + +inline void alignment_int64_array_free(long long *array) +{ + free(array); +} \ No newline at end of file diff --git a/src/inc_internal/alignment.h b/src/inc_internal/alignment.h index 61ad4fb..fb8083e 100644 --- a/src/inc_internal/alignment.h +++ b/src/inc_internal/alignment.h @@ -11,68 +11,24 @@ #ifndef _ALIGNMENT_H_ #define _ALIGNMENT_H_ -#ifdef __cpluscplus +#ifdef __cplusplus extern "C" { #endif -#include -#include +long long *alignment_int64_array_alloc(int size); -#define CPU_CACHE_ALIGMENT 64 +long long alignment_int64_array_sum(long long *array, int size); -inline long long *alignment_int64_array_alloc(int size) -{ - return (long long *)calloc(CPU_CACHE_ALIGMENT, size); -} +long long alignment_int64_array_add(long long *array, int offset, long long op_val); -inline long long alignment_int64_array_sum(long long *array, int size) -{ - int offset = 0; - long long sum = 0; - - for (int i = 0; i < size; i++) { - offset = (CPU_CACHE_ALIGMENT / sizeof(long long)) * i; - sum += array[offset]; - } +long long alignment_int64_array_cnt(long long *array, int size); - return sum; -} +void alignment_int64_array_reset(long long *array, int size); -inline long long alignment_int64_array_add(long long *array, int offset, long long op_val) -{ - int idx = (CPU_CACHE_ALIGMENT / sizeof(long long)) * offset; - array[idx] += op_val; +void alignment_int64_array_free(long long *array); - return array[idx]; -} - -inline long long alignment_int64_array_cnt(long long *array, int size) -{ - int offset = 0; - int cnt = 0; - - for (int i = 0; i < size; i++) { - offset = (CPU_CACHE_ALIGMENT / sizeof(long long)) * i; - if (array[offset] > 0) { - cnt++; - } - } - - return cnt; -} - -inline void alignment_int64_array_reset(long long *array, int size) -{ - memset(array, 0, CPU_CACHE_ALIGMENT * size); -} - -inline void alignment_int64_array_free(long long *array) -{ - free(array); -} - -#ifdef __cpluscplus +#ifdef __cplusplus } #endif diff --git a/src/inc_internal/json2iris.h b/src/inc_internal/json2iris.h index f5d566b..0bfb530 100644 --- a/src/inc_internal/json2iris.h +++ b/src/inc_internal/json2iris.h @@ -11,19 +11,20 @@ #ifndef _JSON2IRIS_H_ #define _JSON2IRIS_H_ -#ifdef __cpluscplus +#ifdef __cplusplus extern "C" { #endif #include "hiredis/hiredis.h" +#include "log/log.h" int json2iris(const char *json_buff, const char *json_filename, redisContext *redis_write_ctx, char *iris_dir_buf, int buf_len, char *encrypt_key, char *encrypt_algo, struct log_handle *logger); -#ifdef __cpluscplus +#ifdef __cplusplus } #endif diff --git a/src/inc_internal/maat_bool_plugin.h b/src/inc_internal/maat_bool_plugin.h index 5ec7d0d..3865eaf 100644 --- a/src/inc_internal/maat_bool_plugin.h +++ b/src/inc_internal/maat_bool_plugin.h @@ -11,16 +11,18 @@ #ifndef _MAAT_BOOL_PLUGIN_H_ #define _MAAT_BOOL_PLUGIN_H_ -#ifdef __cpluscplus +#ifdef __cplusplus extern "C" { #endif -#include "log/log.h" -#include "maat/maat.h" +#include "maat.h" #include "cJSON/cJSON.h" struct bool_plugin_runtime; +struct table_manager; +struct log_handle; +struct maat_garbage_bin; /* bool plugin schema API */ void *bool_plugin_schema_new(cJSON *json, struct table_manager *tbl_mgr, @@ -49,7 +51,7 @@ int bool_plugin_runtime_commit(void *bool_plugin_runtime, const char *table_name struct ex_data_runtime *bool_plugin_runtime_get_ex_data_rt(void *bool_plugin_runtime); -#ifdef __cpluscplus +#ifdef __cplusplus } #endif diff --git a/src/inc_internal/maat_command.h b/src/inc_internal/maat_command.h index 8bbfa41..4539d35 100644 --- a/src/inc_internal/maat_command.h +++ b/src/inc_internal/maat_command.h @@ -11,11 +11,13 @@ #ifndef _MAAT_COMMAND_H_ #define _MAAT_COMMAND_H_ -#ifdef __cpluscplus +#ifdef __cplusplus extern "C" { #endif +#include "maat.h" + enum maat_operation { MAAT_OP_DEL = 0, MAAT_OP_ADD, @@ -39,7 +41,7 @@ struct maat_cmd_line */ int maat_cmd_set_line(struct maat *maat_instance, const struct maat_cmd_line *line_rule); -#ifdef __cpluscplus +#ifdef __cplusplus } #endif diff --git a/src/inc_internal/maat_common.h b/src/inc_internal/maat_common.h index 572bc95..992c41b 100644 --- a/src/inc_internal/maat_common.h +++ b/src/inc_internal/maat_common.h @@ -11,7 +11,7 @@ #ifndef _MAAT_COMMON_H_ #define _MAAT_COMMON_H_ -#ifdef __cpluscplus +#ifdef __cplusplus extern "C" { #endif @@ -41,7 +41,7 @@ struct maat_options { }; }; -#ifdef __cpluscplus +#ifdef __cplusplus } #endif diff --git a/src/inc_internal/maat_compile.h b/src/inc_internal/maat_compile.h index 5cb11b7..b9bf316 100644 --- a/src/inc_internal/maat_compile.h +++ b/src/inc_internal/maat_compile.h @@ -11,14 +11,13 @@ #ifndef _MAAT_COMPILE_H_ #define _MAAT_COMPILE_H_ -#ifdef __cpluscplus +#ifdef __cplusplus extern "C" { #endif -#include "log/log.h" #include "cJSON/cJSON.h" -#include "maat/maat.h" +#include "maat.h" #include "maat_kv.h" #include "maat_rule.h" @@ -32,6 +31,11 @@ struct compile_ex_data_schema { int table_id; }; +struct compile_schema; +struct compile_runtime; +struct maat_compile_state; +struct group2group_runtime; + /* compile schema API */ void *compile_schema_new(cJSON *json, struct table_manager *tbl_mgr, const char *table_name, struct log_handle *logger); @@ -93,7 +97,7 @@ int maat_compile_state_update(struct maat_item *item_hash, int vtable_id, int maat_compile_state_has_NOT_clause(struct maat_compile_state *compile_state); -#ifdef __cpluscplus +#ifdef __cplusplus } #endif diff --git a/src/inc_internal/maat_config_monitor.h b/src/inc_internal/maat_config_monitor.h index b42ea5e..77344b3 100644 --- a/src/inc_internal/maat_config_monitor.h +++ b/src/inc_internal/maat_config_monitor.h @@ -11,7 +11,7 @@ #ifndef _MAAT_CONFIG_MONITOR_H_ #define _MAAT_CONFIG_MONITOR_H_ -#ifdef __cpluscplus +#ifdef __cplusplus extern "C" { #endif @@ -27,7 +27,7 @@ void config_monitor_traverse(long long version, const char *idx_dir, int load_maat_json_file(struct maat *maat_instance, const char *json_filename, char *err_str, size_t err_str_sz); -#ifdef __cpluscplus +#ifdef __cplusplus } #endif diff --git a/src/inc_internal/maat_ex_data.h b/src/inc_internal/maat_ex_data.h index 82e8ddd..cc8feac 100644 --- a/src/inc_internal/maat_ex_data.h +++ b/src/inc_internal/maat_ex_data.h @@ -11,12 +11,12 @@ #ifndef _MAAT_EX_DATA_H_ #define _MAAT_EX_DATA_H_ -#ifdef __cpluscplus +#ifdef __cplusplus extern "C" { #endif -#include "maat/maat.h" +#include "maat.h" #include "rcu_hash.h" struct ex_data_container { @@ -90,7 +90,7 @@ void *ex_data_runtime_get_custom_data(struct ex_data_runtime *ex_data_rt, size_t ex_data_runtime_ex_container_count(struct ex_data_runtime *ex_data_rt); -#ifdef __cpluscplus +#ifdef __cplusplus } #endif diff --git a/src/inc_internal/maat_expr.h b/src/inc_internal/maat_expr.h index 3bf7432..b1c512e 100644 --- a/src/inc_internal/maat_expr.h +++ b/src/inc_internal/maat_expr.h @@ -11,15 +11,18 @@ #ifndef _MAAT_EXPR_H_ #define _MAAT_EXPR_H_ -#ifdef __cpluscplus +#ifdef __cplusplus extern "C" { #endif -#include "log/log.h" +#include "maat_rule.h" #include "cJSON/cJSON.h" struct expr_runtime; +struct table_manager; +struct log_handle; +struct maat_garbage_bin; void *expr_schema_new(cJSON *json, struct table_manager *tbl_mgr, const char *table_name, struct log_handle *logger); @@ -41,13 +44,13 @@ int expr_runtime_commit(void *expr_runtime, const char *table_name); * * @retval the num of hit group_id */ -int expr_runtime_scan_string(struct expr_runtime *expr_rt, int thread_id, - const char *data, size_t data_len, - int *group_ids, size_t group_ids_size, - int vtable_ids, struct maat_state *state); +int expr_runtime_scan(struct expr_runtime *expr_rt, int thread_id, + const char *data, size_t data_len, + int *group_ids, size_t group_ids_size, + int vtable_ids, struct maat_state *state); void expr_runtime_stream_open(struct expr_runtime *expr_rt, int thread_id); -int expr_runtime_scan_stream(struct expr_runtime *expr_rt, const char *data, +int expr_runtime_stream_scan(struct expr_runtime *expr_rt, const char *data, size_t data_len, int *group_ids, size_t group_ids_size, int vtable_id, struct maat_state *state); void expr_runtime_stream_close(struct expr_runtime *expr_rt); @@ -55,7 +58,7 @@ void expr_runtime_stream_close(struct expr_runtime *expr_rt); void expr_runtime_scan_hit_inc(struct expr_runtime *expr_rt, int thread_id); long long expr_runtime_scan_hit_sum(struct expr_runtime *expr_rt, int n_thread); -#ifdef __cpluscplus +#ifdef __cplusplus } #endif diff --git a/src/inc_internal/maat_flag.h b/src/inc_internal/maat_flag.h index ca642e5..8deedba 100644 --- a/src/inc_internal/maat_flag.h +++ b/src/inc_internal/maat_flag.h @@ -11,17 +11,20 @@ #ifndef _MAAT_FLAG_H_ #define _MAAT_FLAG_H_ -#ifdef __cpluscplus +#ifdef __cplusplus extern "C" { #endif #include -#include "log/log.h" +#include "maat_rule.h" #include "cJSON/cJSON.h" struct flag_runtime; +struct table_manager; +struct log_handle; +struct maat_garbage_bin; void *flag_schema_new(cJSON *json, struct table_manager *tbl_mgr, const char *table_name, struct log_handle *logger); @@ -43,14 +46,14 @@ int flag_runtime_commit(void *flag_runtime, const char *table_name); * * @retval the num of hit group_id */ -int flag_runtime_scan_flag(struct flag_runtime *flag_rt, int thread_id, - uint64_t flag, int *group_ids, size_t group_ids_size, - int vtable_id, struct maat_state *state); +int flag_runtime_scan(struct flag_runtime *flag_rt, int thread_id, + uint64_t flag, int *group_ids, size_t group_ids_size, + int vtable_id, struct maat_state *state); void flag_runtime_scan_hit_inc(struct flag_runtime *flag_rt, int thread_id); long long flag_runtime_scan_hit_sum(struct flag_runtime *flag_rt, int n_thread); -#ifdef __cpluscplus +#ifdef __cplusplus } #endif diff --git a/src/inc_internal/maat_fqdn.h b/src/inc_internal/maat_fqdn.h new file mode 100644 index 0000000..aae9179 --- /dev/null +++ b/src/inc_internal/maat_fqdn.h @@ -0,0 +1,55 @@ +/* +********************************************************************************************** +* File: maat_fqdn.h +* Description: +* Authors: Liu WenTan +* Date: 2022-10-31 +* Copyright: (c) 2018-2022 Geedge Networks, Inc. All rights reserved. +*********************************************************************************************** +*/ + +#ifndef _MAAT_FQDN_H_ +#define _MAAT_FQDN_H_ + +#ifdef __cplusplus +extern "C" +{ +#endif + +#include "maat_rule.h" +#include "cJSON/cJSON.h" + +struct fqdn_runtime; +struct table_manager; +struct log_handle; +struct maat_garbage_bin; + +void *fqdn_schema_new(cJSON *json, struct table_manager *tbl_mgr, + const char *table_name, struct log_handle *logger); +void fqdn_schema_free(void *fqdn_schema); + +/* fqdn runtime API */ +void *fqdn_runtime_new(void *fqdn_schema, int max_thread_num, + struct maat_garbage_bin *garbage_bin, + struct log_handle *logger); +void fqdn_runtime_free(void *fqdn_runtime); + +int fqdn_runtime_update(void *fqdn_runtime, void *fqdn_schema, + const char *line, int valid_column); +int fqdn_runtime_commit(void *fqdn_runtime, const char *table_name); + +struct ex_data_runtime *fqdn_runtime_get_ex_data_rt(struct fqdn_runtime *fqdn_rt); + +/* fqdn runtime scan API */ +int fqdn_runtime_scan(struct fqdn_runtime *fqdn_rt, int thread_id, int port, + int proto, int *group_ids, size_t group_id_size, + int vtable_id, struct maat_state *state); + +void fqdn_runtime_scan_hit_inc(struct fqdn_runtime *fqdn_rt, int thread_id); +long long fqdn_runtime_scan_hit_sum(struct fqdn_runtime *fqdn_rt, int n_thread); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/src/inc_internal/maat_fqdn_plugin.h b/src/inc_internal/maat_fqdn_plugin.h index 5d6fb21..922d0fc 100644 --- a/src/inc_internal/maat_fqdn_plugin.h +++ b/src/inc_internal/maat_fqdn_plugin.h @@ -11,16 +11,18 @@ #ifndef _MAAT_FQDN_PLUGIN_H_ #define _MAAT_FQDN_PLUGIN_H_ -#ifdef __cpluscplus +#ifdef __cplusplus extern "C" { #endif -#include "log/log.h" -#include "maat/maat.h" +#include "maat.h" #include "cJSON/cJSON.h" struct fqdn_plugin_runtime; +struct table_manager; +struct log_handle; +struct maat_garbage_bin; /* fqdn plugin schema API */ void *fqdn_plugin_schema_new(cJSON *json, struct table_manager *tbl_mgr, @@ -49,7 +51,7 @@ int fqdn_plugin_runtime_commit(void *fqdn_plugin_runtime, const char *table_name struct ex_data_runtime *fqdn_plugin_runtime_get_ex_data_rt(void *fqdn_plugin_runtime); -#ifdef __cpluscplus +#ifdef __cplusplus } #endif diff --git a/src/inc_internal/maat_garbage_collection.h b/src/inc_internal/maat_garbage_collection.h index 3ce3224..49e6bfc 100644 --- a/src/inc_internal/maat_garbage_collection.h +++ b/src/inc_internal/maat_garbage_collection.h @@ -11,7 +11,7 @@ #ifndef _MAAT_GARBAGE_COLLECTION_H_ #define _MAAT_GARBAGE_COLLECTION_H_ -#ifdef __cpluscplus +#ifdef __cplusplus extern "C" { #endif @@ -33,7 +33,7 @@ size_t maat_garbage_bin_get_size(struct maat_garbage_bin *bin); void maat_garbage_collect_by_force(struct maat_garbage_bin *bin); -#ifdef __cpluscplus +#ifdef __cplusplus } #endif diff --git a/src/inc_internal/maat_group.h b/src/inc_internal/maat_group.h index 5bdbdea..787cbc2 100644 --- a/src/inc_internal/maat_group.h +++ b/src/inc_internal/maat_group.h @@ -11,17 +11,18 @@ #ifndef _MAAT_GROUP_H_ #define _MAAT_GROUP_H_ -#ifdef __cpluscplus +#ifdef __cplusplus extern "C" { #endif #include "cJSON/cJSON.h" -#include "uthash/uthash.h" -#include "igraph/igraph.h" #include "maat_kv.h" struct maat_group; +struct table_manager; +struct log_handle; +struct maat_garbage_bin; /* group2group schema API */ void *group2group_schema_new(cJSON *json, struct table_manager *tbl_mgr, @@ -49,7 +50,7 @@ int group2group_runtime_update(void *g2g_runtime, void *g2g_schema, const char *line, int valid_column); int group2group_runtime_commit(void *g2g_runtime, const char *table_name); -#ifdef __cpluscplus +#ifdef __cplusplus } #endif diff --git a/src/inc_internal/maat_interval.h b/src/inc_internal/maat_interval.h index ab40f9f..344f5b7 100644 --- a/src/inc_internal/maat_interval.h +++ b/src/inc_internal/maat_interval.h @@ -11,17 +11,20 @@ #ifndef _MAAT_INTERVAL_H_ #define _MAAT_INTERVAL_H_ -#ifdef __cpluscplus +#ifdef __cplusplus extern "C" { #endif #include -#include "log/log.h" +#include "maat_rule.h" #include "cJSON/cJSON.h" struct interval_runtime; +struct table_manager; +struct log_handle; +struct maat_garbage_bin; void *interval_schema_new(cJSON *json, struct table_manager *tbl_mgr, const char *table_name, struct log_handle *logger); @@ -43,14 +46,14 @@ int interval_runtime_commit(void *interval_runtime, const char *table_name); * * @retval the num of hit group_id */ -int interval_runtime_scan_intval(struct interval_runtime *interval_rt, int thread_id, - uint64_t integer, int *group_ids, size_t group_ids_size, - int vtable_id, struct maat_state *state); +int interval_runtime_scan(struct interval_runtime *interval_rt, int thread_id, + uint64_t integer, int *group_ids, size_t group_ids_size, + int vtable_id, struct maat_state *state); void interval_runtime_scan_hit_inc(struct interval_runtime *interval_rt, int thread_id); long long interval_runtime_scan_hit_sum(struct interval_runtime *interval_rt, int n_thread); -#ifdef __cpluscplus +#ifdef __cplusplus } #endif diff --git a/src/inc_internal/maat_ip.h b/src/inc_internal/maat_ip.h index 3352612..0d6c548 100644 --- a/src/inc_internal/maat_ip.h +++ b/src/inc_internal/maat_ip.h @@ -11,12 +11,17 @@ #ifndef _MAAT_IP_H_ #define _MAAT_IP_H_ -#ifdef __cpluscplus +#ifdef __cplusplus extern "C" { #endif +#include "cJSON/cJSON.h" + struct ip_runtime; +struct table_manager; +struct log_handle; +struct maat_garbage_bin; void *ip_schema_new(cJSON *json, struct table_manager *tbl_mgr, const char *table_name, struct log_handle *logger); @@ -35,14 +40,14 @@ int ip_runtime_commit(void *ip_runtime, const char *table_name); struct ex_data_runtime *ip_runtime_get_ex_data_rt(struct ip_runtime *ip_rt); /* ip runtime scan API */ -int ip_runtime_scan_ip(struct ip_runtime *ip_rt, int thread_id, int ip_type, +int ip_runtime_scan(struct ip_runtime *ip_rt, int thread_id, int ip_type, uint8_t *ip_addr, int *group_ids, size_t group_id_size, int vtable_id, struct maat_state *state); void ip_runtime_scan_hit_inc(struct ip_runtime *ip_rt, int thread_id); long long ip_runtime_scan_hit_sum(struct ip_runtime *ip_rt, int n_thread); -#ifdef __cpluscplus +#ifdef __cplusplus } #endif diff --git a/src/inc_internal/maat_ip_plugin.h b/src/inc_internal/maat_ip_plugin.h index 2b8332f..eec3c13 100644 --- a/src/inc_internal/maat_ip_plugin.h +++ b/src/inc_internal/maat_ip_plugin.h @@ -11,16 +11,18 @@ #ifndef _MAAT_IP_PLUGIN_H_ #define _MAAT_IP_PLUGIN_H_ -#ifdef __cpluscplus +#ifdef __cplusplus extern "C" { #endif -#include "log/log.h" -#include "maat/maat.h" +#include "maat.h" #include "cJSON/cJSON.h" struct ip_plugin_runtime; +struct table_manager; +struct log_handle; +struct maat_garbage_bin; /* ip plugin schema API */ void *ip_plugin_schema_new(cJSON *json, struct table_manager *tbl_mgr, @@ -49,7 +51,7 @@ int ip_plugin_runtime_commit(void *ip_plugin_runtime, const char *table_name); struct ex_data_runtime *ip_plugin_runtime_get_ex_data_rt(void *ip_plugin_runtime); -#ifdef __cpluscplus +#ifdef __cplusplus } #endif diff --git a/src/inc_internal/maat_kv.h b/src/inc_internal/maat_kv.h index c8108d0..37217ee 100644 --- a/src/inc_internal/maat_kv.h +++ b/src/inc_internal/maat_kv.h @@ -11,11 +11,13 @@ #ifndef _MAAT_KV_H_ #define _MAAT_KV_H_ -#ifdef __cpluscplus +#ifdef __cplusplus extern "C" { #endif +#include + struct maat_kv_store; struct maat_kv_store *maat_kv_store_new(void); @@ -30,7 +32,7 @@ int maat_kv_read_unNull(struct maat_kv_store *store, const char *key, size_t key struct maat_kv_store *maat_kv_store_duplicate(struct maat_kv_store *store); -#ifdef __cpluscplus +#ifdef __cplusplus } #endif diff --git a/src/inc_internal/maat_limits.h b/src/inc_internal/maat_limits.h index 517c900..d09165a 100644 --- a/src/inc_internal/maat_limits.h +++ b/src/inc_internal/maat_limits.h @@ -11,14 +11,14 @@ #ifndef _MAAT_LIMITS_H_ #define _MAAT_LIMITS_H_ -#ifdef __cpluscplus +#ifdef __cplusplus extern "C" { #endif #define MAX_KEYWORDS_STR 1024 -#ifdef __cpluscplus +#ifdef __cplusplus } #endif diff --git a/src/inc_internal/maat_plugin.h b/src/inc_internal/maat_plugin.h index 34567cf..61f9807 100644 --- a/src/inc_internal/maat_plugin.h +++ b/src/inc_internal/maat_plugin.h @@ -11,12 +11,13 @@ #ifndef _MAAT_PLUGIN_H_ #define _MAAT_PLUGIN_H_ -#ifdef __cpluscplus +#ifdef __cplusplus extern "C" { #endif -#include "maat/maat.h" +#include "cJSON/cJSON.h" +#include "maat.h" #define MAX_FOREIGN_CLMN_NUM 8 @@ -59,7 +60,7 @@ int plugin_runtime_commit(void *plugin_runtime, const char *table_name); struct ex_data_runtime *plugin_runtime_get_ex_data_rt(void *plugin_runtime); -#ifdef __cpluscplus +#ifdef __cplusplus } #endif diff --git a/src/inc_internal/maat_port_proto.h b/src/inc_internal/maat_port_proto.h new file mode 100644 index 0000000..94a8f75 --- /dev/null +++ b/src/inc_internal/maat_port_proto.h @@ -0,0 +1,56 @@ +/* +********************************************************************************************** +* File: maat_port_proto.h +* Description: +* Authors: Liu WenTan +* Date: 2022-10-31 +* Copyright: (c) 2018-2022 Geedge Networks, Inc. All rights reserved. +*********************************************************************************************** +*/ + +#ifndef _MAAT_PORT_PROTO_H_ +#define _MAAT_PORT_PROTO_H_ + +#ifdef __cplusplus +extern "C" +{ +#endif + +#include "maat_rule.h" +#include "cJSON/cJSON.h" + +/* pp is short for port proto */ +struct pp_runtime; +struct table_manager; +struct log_handle; +struct maat_garbage_bin; + +void *pp_schema_new(cJSON *json, struct table_manager *tbl_mgr, + const char *table_name, struct log_handle *logger); +void pp_schema_free(void *pp_schema); + +/* ip runtime API */ +void *pp_runtime_new(void *pp_schema, int max_thread_num, + struct maat_garbage_bin *garbage_bin, + struct log_handle *logger); +void pp_runtime_free(void *pp_runtime); + +int pp_runtime_update(void *pp_runtime, void *pp_schema, + const char *line, int valid_column); +int pp_runtime_commit(void *pp_runtime, const char *table_name); + +struct ex_data_runtime *pp_runtime_get_ex_data_rt(struct pp_runtime *pp_rt); + +/* ip runtime scan API */ +int pp_runtime_scan(struct pp_runtime *pp_rt, int thread_id, int port, + int proto, int *group_ids, size_t group_id_size, + int vtable_id, struct maat_state *state); + +void pp_runtime_scan_hit_inc(struct pp_runtime *pp_rt, int thread_id); +long long pp_runtime_scan_hit_sum(struct pp_runtime *pp_rt, int n_thread); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/src/inc_internal/maat_redis_monitor.h b/src/inc_internal/maat_redis_monitor.h index b4e7b01..d25e760 100644 --- a/src/inc_internal/maat_redis_monitor.h +++ b/src/inc_internal/maat_redis_monitor.h @@ -11,7 +11,7 @@ #ifndef _MAAT_REDIS_MONITOR_H_ #define _MAAT_REDIS_MONITOR_H_ -#ifdef __cpluscplus +#ifdef __cplusplus extern "C" { #endif @@ -25,7 +25,7 @@ void redis_monitor_traverse(long long version, struct source_redis_ctx *mr_ctx, int (*update_fn)(const char *, const char *, void *), void (*finish_fn)(void *), void *u_param); -#ifdef __cpluscplus +#ifdef __cplusplus } #endif diff --git a/src/inc_internal/maat_rule.h b/src/inc_internal/maat_rule.h index 55107a9..b61dc1b 100644 --- a/src/inc_internal/maat_rule.h +++ b/src/inc_internal/maat_rule.h @@ -11,7 +11,7 @@ #ifndef _MAAT_RULE_H_ #define _MAAT_RULE_H_ -#ifdef __cpluscplus +#ifdef __cplusplus extern "C" { #endif @@ -29,8 +29,10 @@ extern "C" #include "uthash/uthash.h" #include "maat_command.h" #include "IPMatcher.h" +#include "maat.h" #include "maat_kv.h" #include "maat_table.h" +#include "maat_rule.h" #include "maat_virtual.h" #define MAX_TABLE_NUM 256 @@ -66,18 +68,6 @@ struct maat_rule_head { int serv_def_len; }; -#define MAX_SERVICE_DEFINE_LEN 128 -struct maat_rule { - int config_id; - int service_id; - uint8_t do_log; - uint8_t do_blacklist; - uint8_t action; - uint8_t reserved; - int serv_def_len; - char service_defined[MAX_SERVICE_DEFINE_LEN]; -}; - #define ITEM_RULE_MAGIC 0x4d3c2b1a struct maat_item_inner { long long magic_num; @@ -329,7 +319,7 @@ void maat_cmd_set_serial_rule(struct serial_rule *rule, enum maat_operation op, void fill_maat_rule(struct maat_rule *rule, const struct maat_rule_head *rule_head, const char *srv_def, int srv_def_len); -#ifdef __cpluscplus +#ifdef __cplusplus } #endif diff --git a/src/inc_internal/maat_table.h b/src/inc_internal/maat_table.h index e5038ed..f2adad7 100644 --- a/src/inc_internal/maat_table.h +++ b/src/inc_internal/maat_table.h @@ -11,7 +11,7 @@ #ifndef _MAAT_TABLE_H_ #define _MAAT_TABLE_H_ -#ifdef __cpluscplus +#ifdef __cplusplus extern "C" { #endif @@ -19,6 +19,8 @@ extern "C" #include #include +#include "maat_garbage_collection.h" + enum table_type { TABLE_TYPE_INVALID = -1, TABLE_TYPE_FLAG = 0, @@ -74,7 +76,7 @@ void *table_manager_get_runtime(struct table_manager *tbl_mgr, int table_id); int table_manager_update_runtime(struct table_manager *tbl_mgr, int table_id, const char *line); void table_manager_commit_runtime(struct table_manager *tbl_mgr, int table_id); -#ifdef __cpluscplus +#ifdef __cplusplus } #endif diff --git a/src/inc_internal/maat_utils.h b/src/inc_internal/maat_utils.h index 53d167d..50d3ec5 100644 --- a/src/inc_internal/maat_utils.h +++ b/src/inc_internal/maat_utils.h @@ -11,7 +11,7 @@ #ifndef _MAAT_UTILS_H_ #define _MAAT_UTILS_H_ -#ifdef __cpluscplus +#ifdef __cplusplus extern "C" { #endif @@ -67,12 +67,7 @@ enum maat_ip_format ip_format_str2int(const char *format); int ip_format2range(int ip_type, enum maat_ip_format format, const char *ip1, const char *ip2, uint32_t range_begin[], uint32_t range_end[]); -inline void ipv6_ntoh(unsigned int *v6_addr) -{ - for (unsigned int i = 0; i < 4; i++) { - v6_addr[i] = ntohl(v6_addr[i]); - } -} + #define UNUSED __attribute__((unused)) @@ -80,6 +75,8 @@ const char *module_name_str(const char *name); char *maat_strdup(const char *s); +void ipv6_ntoh(unsigned int *v6_addr); + int get_column_pos(const char *line, int column_seq, size_t *offset, size_t *len); /* the column value must be integer */ @@ -111,7 +108,7 @@ int system_cmd_mkdir(const char* path); int system_cmd_rmdir(const char *dir); -#ifdef __cpluscplus +#ifdef __cplusplus } #endif diff --git a/src/inc_internal/maat_virtual.h b/src/inc_internal/maat_virtual.h index e73c53d..57956ab 100644 --- a/src/inc_internal/maat_virtual.h +++ b/src/inc_internal/maat_virtual.h @@ -11,11 +11,13 @@ #ifndef _MAAT_VIRTUAL_H_ #define _MAAT_VIRTUAL_H_ -#ifdef __cpluscplus +#ifdef __cplusplus extern "C" { #endif +#include "cJSON/cJSON.h" + enum scan_type { SCAN_TYPE_INVALID = -1, SCAN_TYPE_NONE = 0, @@ -30,6 +32,9 @@ enum scan_type { SCAN_TYPE_MAX }; +struct table_manager; +struct log_handle; + void *virtual_schema_new(cJSON *json, struct table_manager *tbl_mgr, const char *table_name, struct log_handle *logger); @@ -37,7 +42,7 @@ void virtual_schema_free(void *virtual_schema); int virtual_table_get_physical_table_id(void *virtual_schema, enum scan_type type); -#ifdef __cpluscplus +#ifdef __cplusplus } #endif diff --git a/src/inc_internal/rcu_hash.h b/src/inc_internal/rcu_hash.h index 44537d1..8267a19 100644 --- a/src/inc_internal/rcu_hash.h +++ b/src/inc_internal/rcu_hash.h @@ -11,7 +11,7 @@ #ifndef _RCU_HASH_H_ #define _RCU_HASH_H_ -#ifdef __cpluscplus +#ifdef __cplusplus extern "C" { #endif @@ -62,7 +62,7 @@ size_t rcu_hash_garbage_queue_len(struct rcu_hash_table *htable); size_t rcu_hash_list_updating_data(struct rcu_hash_table *htable, void ***data_array); -#ifdef __cpluscplus +#ifdef __cplusplus } #endif diff --git a/src/json2iris.cpp b/src/json2iris.c similarity index 100% rename from src/json2iris.cpp rename to src/json2iris.c diff --git a/src/maat_api.cpp b/src/maat_api.c similarity index 90% rename from src/maat_api.cpp rename to src/maat_api.c index 1781528..dac9729 100644 --- a/src/maat_api.cpp +++ b/src/maat_api.c @@ -14,7 +14,7 @@ #include "maat_utils.h" #include "json2iris.h" -#include "maat/maat.h" +#include "maat.h" #include "maat_rule.h" #include "maat_common.h" #include "maat_kv.h" @@ -29,6 +29,7 @@ #include "maat_group.h" #include "maat_expr.h" #include "maat_flag.h" +#include "maat_interval.h" #include "maat_ip.h" #include "maat_plugin.h" #include "maat_ip_plugin.h" @@ -41,6 +42,12 @@ enum district_set_flag { DISTRICT_FLAG_SET }; +struct maat_stream { + struct maat *maat_instance; + struct maat_runtime *ref_maat_rt; + int vtable_id; +}; + enum scan_type maat_table_get_scan_type(enum table_type table_type) { enum scan_type ret = SCAN_TYPE_INVALID; @@ -381,12 +388,12 @@ int maat_table_get_id(struct maat *maat_instance, const char *table_name) return table_id; } -inline void maat_runtime_ref_inc(struct maat_runtime *maat_rt, int thread_id) +static inline void maat_runtime_ref_inc(struct maat_runtime *maat_rt, int thread_id) { alignment_int64_array_add(maat_rt->ref_cnt, thread_id, 1); } -inline void maat_runtime_ref_dec(struct maat_runtime *maat_rt, int thread_id) +static inline void maat_runtime_ref_dec(struct maat_runtime *maat_rt, int thread_id) { alignment_int64_array_add(maat_rt->ref_cnt, thread_id, -1); } @@ -701,7 +708,7 @@ struct maat_state *grab_state(struct maat_state **state, struct maat *maat_insta return mid; } -inline int scan_status_should_compile_NOT(struct maat_state *state) +static inline int scan_status_should_compile_NOT(struct maat_state *state) { if (state && (LAST_SCAN_SET == state->is_last_scan) && state->compile_state && maat_compile_state_has_NOT_clause(state->compile_state)) { @@ -723,7 +730,7 @@ int maat_scan_flag(struct maat *maat_instance, int table_id, int thread_id, uint64_t flag, int *results, size_t n_result, size_t *n_hit_result, struct maat_state **state) { - if ((NULL == maat_instance) || (table_id < 0) || (table_id >= MAX_TABLE_NUM) + if ((NULL == maat_instance) || (table_id < 0) || (table_id >= MAX_TABLE_NUM) || (thread_id < 0) || (NULL == results) || (0 == n_result) || (NULL == state)) { return MAAT_SCAN_ERR; } @@ -762,7 +769,7 @@ int maat_scan_flag(struct maat *maat_instance, int table_id, int thread_id, return MAAT_SCAN_ERR; } - int group_hit_cnt = flag_runtime_scan_flag((struct flag_runtime *)flag_rt, + int group_hit_cnt = flag_runtime_scan((struct flag_runtime *)flag_rt, thread_id, flag, group_ids, MAX_SCANNER_HIT_GROUP_NUM, @@ -814,11 +821,99 @@ int maat_scan_flag(struct maat *maat_instance, int table_id, int thread_id, return MAAT_SCAN_OK; } -int maat_scan_integer(struct maat *instance, int table_id, int thread_id, +int maat_scan_integer(struct maat *maat_instance, int table_id, int thread_id, unsigned int intval, int *results, size_t n_result, size_t *n_hit_result, struct maat_state **state) { - return 0; + if ((NULL == maat_instance) || (table_id < 0) || (table_id >= MAX_TABLE_NUM) + || (thread_id < 0) || (NULL == results) || (0 == n_result) || (NULL == state)) { + return MAAT_SCAN_ERR; + } + + struct maat_state *mid = NULL; + mid = grab_state(state, maat_instance, thread_id); + mid->scan_cnt++; + + int physical_table_id = 0; + int vtable_id = 0; + + enum table_type table_type = table_manager_get_table_type(maat_instance->tbl_mgr, table_id); + if (table_type == TABLE_TYPE_VIRTUAL) { + //find physical table id + void *virtual_schema = table_manager_get_schema(maat_instance->tbl_mgr, table_id); + physical_table_id = virtual_table_get_physical_table_id(virtual_schema, SCAN_TYPE_FLAG); + if (physical_table_id < 0) { + return MAAT_SCAN_ERR; + } + } else { + physical_table_id = table_id; + } + + if (NULL == maat_instance->maat_rt) { + log_error(maat_instance->logger, MODULE_MAAT_API, + "maat_scan_string error because of maat_runtime is NULL"); + return MAAT_SCAN_OK; + } + + maat_runtime_ref_inc(maat_instance->maat_rt, thread_id); + alignment_int64_array_add(maat_instance->thread_call_cnt, thread_id, 1); + + int group_ids[MAX_SCANNER_HIT_GROUP_NUM] = {-1}; + void *interval_rt = table_manager_get_runtime(maat_instance->tbl_mgr, table_id); + if (NULL == interval_rt) { + return MAAT_SCAN_ERR; + } + + int group_hit_cnt = interval_runtime_scan((struct interval_runtime *)interval_rt, + thread_id, + intval, group_ids, + MAX_SCANNER_HIT_GROUP_NUM, + vtable_id, mid); + if (group_hit_cnt < 0) { + return MAAT_SCAN_ERR; + } + + size_t n_hit_compile = 0; + if (group_hit_cnt > 0 || scan_status_should_compile_NOT(mid)) { + // come here means group_hit_cnt > 0, at least MAAT_SCAN_HALF_HIT, or MAAT_SCAN_HIT + if (group_hit_cnt > 0) { + interval_runtime_scan_hit_inc((struct interval_runtime *)interval_rt, thread_id); + } + + int compile_table_id = -1; + if (mid->compile_table_id == -1) { + compile_table_id = maat_instance->default_compile_table_id; + } else { + compile_table_id = mid->compile_table_id; + } + + void *compile_rt = table_manager_get_runtime(maat_instance->tbl_mgr, compile_table_id); + n_hit_compile = hit_group_to_compile(compile_rt, results, n_result, mid); + *n_hit_result = n_hit_compile; + + assert(mid->is_last_scan < LAST_SCAN_FINISHED); + if (LAST_SCAN_SET == mid->is_last_scan) { + mid->is_last_scan = LAST_SCAN_FINISHED; + } + } + + if (n_hit_compile > 0) { + alignment_int64_array_add(maat_instance->hit_cnt, thread_id, 1); + if (0 == group_hit_cnt) { + //hit NOT group + alignment_int64_array_add(maat_instance->not_grp_hit_cnt, thread_id, 1); + } + return MAAT_SCAN_HIT; + } else { + // n_hit_compile == 0 + if (group_hit_cnt > 0) { + return MAAT_SCAN_HALF_HIT; + } + } + + maat_runtime_ref_dec(maat_instance->maat_rt, thread_id); + + return MAAT_SCAN_OK; } int maat_scan_ipv4(struct maat *maat_instance, int table_id, int thread_id, @@ -857,7 +952,7 @@ int maat_scan_ipv4(struct maat *maat_instance, int table_id, int thread_id, // return 0; // } - int group_hit_cnt = ip_runtime_scan_ip((struct ip_runtime *)ip_rt, thread_id, IPv4, + int group_hit_cnt = ip_runtime_scan((struct ip_runtime *)ip_rt, thread_id, IPv4, (uint8_t *)&ip_addr, group_ids, sizeof(group_ids), vtable_id, mid); if (group_hit_cnt < 0) { @@ -944,7 +1039,7 @@ int maat_scan_ipv6(struct maat *maat_instance, int table_id, int thread_id, // return 0; // } - int group_hit_cnt = ip_runtime_scan_ip((struct ip_runtime *)ip_rt, thread_id, IPv6, ip_addr, + int group_hit_cnt = ip_runtime_scan((struct ip_runtime *)ip_rt, thread_id, IPv6, ip_addr, group_ids, sizeof(group_ids), vtable_id, mid); if (group_hit_cnt < 0) { return MAAT_SCAN_ERR; @@ -1045,7 +1140,7 @@ int maat_scan_string(struct maat *maat_instance, int table_id, int thread_id, return MAAT_SCAN_ERR; } - int group_hit_cnt = expr_runtime_scan_string((struct expr_runtime *)expr_rt, + int group_hit_cnt = expr_runtime_scan((struct expr_runtime *)expr_rt, thread_id, data, data_len, group_ids, MAX_SCANNER_HIT_GROUP_NUM, diff --git a/src/maat_bool_plugin.cpp b/src/maat_bool_plugin.c similarity index 100% rename from src/maat_bool_plugin.cpp rename to src/maat_bool_plugin.c diff --git a/src/maat_command.cpp b/src/maat_command.c similarity index 100% rename from src/maat_command.cpp rename to src/maat_command.c diff --git a/src/maat_compile.cpp b/src/maat_compile.c similarity index 99% rename from src/maat_compile.cpp rename to src/maat_compile.c index f151557..88ce8c3 100644 --- a/src/maat_compile.cpp +++ b/src/maat_compile.c @@ -20,7 +20,6 @@ #include "maat_compile.h" #include "maat_garbage_collection.h" #include "maat_group.h" -#include "maat/maat.h" #include "rcu_hash.h" #include "maat_table.h" diff --git a/src/maat_config_monitor.cpp b/src/maat_config_monitor.c similarity index 100% rename from src/maat_config_monitor.cpp rename to src/maat_config_monitor.c diff --git a/src/maat_ex_data.cpp b/src/maat_ex_data.c similarity index 99% rename from src/maat_ex_data.cpp rename to src/maat_ex_data.c index 0663c57..f2a2e60 100644 --- a/src/maat_ex_data.cpp +++ b/src/maat_ex_data.c @@ -14,7 +14,6 @@ #include "uthash/uthash.h" #include "uthash/utarray.h" #include "log/log.h" -#include "rcu_hash.h" #include "maat_utils.h" #include "maat_ex_data.h" @@ -69,8 +68,8 @@ void ex_data_runtime_free(struct ex_data_runtime *ex_data_rt) if (ex_data_rt->cache_rows != NULL) { utarray_free(ex_data_rt->cache_rows); - ex_data_rt->cache_rows=NULL; - ex_data_rt->cache_row_num=0; + ex_data_rt->cache_rows = NULL; + ex_data_rt->cache_row_num = 0; } if (ex_data_rt->htable != NULL) { diff --git a/src/maat_expr.cpp b/src/maat_expr.c similarity index 88% rename from src/maat_expr.cpp rename to src/maat_expr.c index 9b76f67..1957950 100644 --- a/src/maat_expr.cpp +++ b/src/maat_expr.c @@ -35,6 +35,7 @@ struct expr_schema { int match_method_column; int is_hexbin_column; enum hs_scan_mode scan_mode; /* adapter_hs scan mode */ + enum hs_pattern_type pattern_type; /* literal or regex */ int table_id; //ugly struct table_manager *ref_tbl_mgr; }; @@ -61,13 +62,14 @@ struct expr_item { int district_id; char keywords[MAX_KEYWORDS_STR]; enum expr_type expr_type; - enum match_method match_method; + enum hs_match_mode match_mode; int is_hexbin; int is_case_sensitive; }; struct expr_runtime { enum hs_scan_mode scan_mode; + enum hs_pattern_type pattern_type; struct adapter_hs *hs; struct adapter_hs_stream *hs_stream; struct rcu_hash_table *htable; @@ -111,28 +113,28 @@ enum expr_type int_to_expr_type(int expr_type) return type; } -enum match_method int_to_match_method_type(int match_method_type) +enum hs_match_mode int_to_match_mode(int match_method) { - enum match_method type = MATCH_METHOD_MAX; + enum hs_match_mode mode = HS_MATCH_MODE_MAX; - switch (match_method_type) { + switch (match_method) { case 0: - type = MATCH_METHOD_SUB; + mode = HS_MATCH_MODE_SUB; break; case 1: - type = MATCH_METHOD_RIGHT; + mode = HS_MATCH_MODE_SUFFIX; break; case 2: - type = MATCH_METHOD_LEFT; + mode = HS_MATCH_MODE_PREFIX; break; case 3: - type = MATCH_METHOD_COMPLETE; + mode = HS_MATCH_MODE_EXACTLY; break; default: break; } - return type; + return mode; } struct expr_item *expr_item_new(const char *line, struct expr_schema *expr_schema, @@ -205,7 +207,7 @@ struct expr_item *expr_item_new(const char *line, struct expr_schema *expr_schem } match_method_type = atoi(line + column_offset); - expr_item->match_method = int_to_match_method_type(match_method_type); + expr_item->match_mode = int_to_match_mode(match_method_type); ret = get_column_pos(line, expr_schema->is_hexbin_column, &column_offset, &column_len); if (ret < 0) { @@ -252,7 +254,6 @@ struct expr_item *expr_item_new(const char *line, struct expr_schema *expr_schem } memcpy(expr_item->keywords, (line + column_offset), column_len); - return expr_item; error: FREE(expr_item); @@ -264,17 +265,42 @@ void expr_item_free(struct expr_item *expr_item) FREE(expr_item); } +enum hs_scan_mode scan_mode_str_to_enum(const char *mode_str) +{ + enum hs_scan_mode scan_mode = HS_SCAN_MODE_MAX; + + if (strcmp(mode_str, "block") == 0) { + scan_mode = HS_SCAN_MODE_BLOCK; + } else if (strcmp(mode_str, "stream") == 0) { + scan_mode = HS_SCAN_MODE_STREAM; + } else { + assert(0); + } + + return scan_mode; +} + +enum hs_pattern_type pattern_type_str_to_enum(const char *type_str) +{ + enum hs_pattern_type pattern_type = HS_PATTERN_TYPE_MAX; + + if (strcmp(type_str, "literal") == 0) { + pattern_type = HS_PATTERN_TYPE_STR; + } else if (strcmp(type_str, "regex") == 0) { + pattern_type = HS_PATTERN_TYPE_REG; + } else { + assert(0); + } + + return pattern_type; +} + void *expr_schema_new(cJSON *json, struct table_manager *tbl_mgr, const char *table_name, struct log_handle *logger) { int read_cnt = 0; struct expr_schema *expr_schema = ALLOC(struct expr_schema, 1); - struct maat_kv_store *scan_mode_map = maat_kv_store_new(); - maat_kv_register(scan_mode_map, "block", HS_SCAN_MODE_BLOCK); - maat_kv_register(scan_mode_map, "stream", HS_SCAN_MODE_STREAM); - - int ret = -1; cJSON *custom_item = NULL; cJSON *item = cJSON_GetObjectItem(json, "table_id"); if (item != NULL && item->type == cJSON_Number) { @@ -291,13 +317,13 @@ void *expr_schema_new(cJSON *json, struct table_manager *tbl_mgr, custom_item = cJSON_GetObjectItem(item, "scan_mode"); if (custom_item != NULL && custom_item->type == cJSON_String) { - ret = maat_kv_read(scan_mode_map, custom_item->valuestring, - (int*)&(expr_schema->scan_mode)); - if (ret < 0) { - log_error(logger, MODULE_EXPR, "scan_mode %s illegal", - custom_item->valuestring); - goto error; - } + expr_schema->scan_mode = scan_mode_str_to_enum(custom_item->valuestring); + read_cnt++; + } + + custom_item = cJSON_GetObjectItem(item, "pattern_type"); + if (custom_item != NULL && custom_item->type == cJSON_String) { + expr_schema->pattern_type = pattern_type_str_to_enum(custom_item->valuestring); read_cnt++; } @@ -319,10 +345,10 @@ void *expr_schema_new(cJSON *json, struct table_manager *tbl_mgr, read_cnt++; } + /* expr_plus has district */ custom_item = cJSON_GetObjectItem(item, "district"); if (custom_item != NULL && custom_item->type == cJSON_Number) { expr_schema->district_column = custom_item->valueint; - read_cnt++; } custom_item = cJSON_GetObjectItem(item, "expr_type"); @@ -345,14 +371,12 @@ void *expr_schema_new(cJSON *json, struct table_manager *tbl_mgr, expr_schema->ref_tbl_mgr = tbl_mgr; - if (read_cnt < 8) { + if (read_cnt < 9) { goto error; } - maat_kv_store_free(scan_mode_map); return expr_schema; error: - maat_kv_store_free(scan_mode_map); FREE(expr_schema); return NULL; } @@ -394,6 +418,7 @@ void *expr_runtime_new(void *expr_schema, int max_thread_num, expr_rt->htable = rcu_hash_new(expr_ex_data_free); expr_rt->scan_mode = schema->scan_mode; + expr_rt->pattern_type = schema->pattern_type; expr_rt->item_user_data_free = maat_item_inner_free; expr_rt->n_worker_thread = max_thread_num; expr_rt->ref_garbage_bin = garbage_bin; @@ -477,17 +502,18 @@ int expr_runtime_update_row(struct expr_runtime *expr_rt, char *key, return 0; } -enum pattern_type expr_type2pattern_type(enum expr_type expr_type) +enum hs_pattern_type expr_type2pattern_type(enum expr_type expr_type) { - enum pattern_type pattern_type = PATTERN_TYPE_STR; + enum hs_pattern_type pattern_type = HS_PATTERN_TYPE_MAX; switch (expr_type) { case EXPR_TYPE_STRING: case EXPR_TYPE_AND: case EXPR_TYPE_OFFSET: + pattern_type = HS_PATTERN_TYPE_STR; break; case EXPR_TYPE_REGEX: - pattern_type = PATTERN_TYPE_REG; + pattern_type = HS_PATTERN_TYPE_REG; break; default: break; @@ -496,7 +522,7 @@ enum pattern_type expr_type2pattern_type(enum expr_type expr_type) return pattern_type; } -int converHextoint(char srctmp) +int convertHextoint(char srctmp) { if (isdigit(srctmp)) { return srctmp - '0'; @@ -512,8 +538,8 @@ size_t hex2bin(char *hex, int hex_len, char *binary, size_t size) size_t resultlen = 0; int high,low; for (int i = 0; i < hex_len && size > resultlen; i += 2, resultlen++) { - high = converHextoint(hex[i]); - low = converHextoint(hex[i+1]); + high = convertHextoint(hex[i]); + low = convertHextoint(hex[i+1]); binary[resultlen] = high * 16 + low; } @@ -612,15 +638,15 @@ and_expr_t *expr_item_to_expr_rule(struct expr_item *expr_item, void *user_data, size_t sub_key_len = 0; for (i = 0; i < sub_expr_cnt; i++) { - expr_rule->expr_id = expr_item->item_id; + if (FALSE == expr_item->is_case_sensitive) { // insensitive - expr_rule->patterns[i].case_sensitive = 1; + expr_rule->patterns[i].case_sensitive = HS_CASE_INSESITIVE; } - expr_rule->patterns[i].type = expr_type2pattern_type(expr_item->expr_type); + enum hs_pattern_type pattern_type = expr_type2pattern_type(expr_item->expr_type); - if (TRUE == expr_item->is_hexbin && expr_rule->patterns[i].type != EXPR_TYPE_REGEX) { + if (TRUE == expr_item->is_hexbin && pattern_type != HS_PATTERN_TYPE_REG) { region_str_len = strlen(sub_key_array[i]) + 1; region_string = ALLOC(char, region_str_len); region_str_len = hex2bin(sub_key_array[i], strlen(sub_key_array[i]), region_string, region_str_len); @@ -637,11 +663,26 @@ and_expr_t *expr_item_to_expr_rule(struct expr_item *expr_item, void *user_data, memcpy(expr_rule->patterns[i].pat, sub_key_array[i], sub_key_len); expr_rule->patterns[i].pat_len = sub_key_len; } + + expr_rule->patterns[i].match_mode = expr_item->match_mode; + if (expr_rule->patterns[i].match_mode == HS_MATCH_MODE_SUB) { + expr_rule->patterns[i].l_offset = key_left_offset[i]; + expr_rule->patterns[i].r_offset = key_right_offset[i]; + } } - + expr_rule->expr_id = expr_item->item_id; expr_rule->user_tag = user_data; expr_rule->n_patterns = sub_expr_cnt; + //mytest + // printf("expr_rule->expr_id:%u\n", expr_rule->expr_id); + // printf("expr_rule->n_patterns:%zu\n", expr_rule->n_patterns); + // for (size_t i = 0; i < expr_rule->n_patterns; i++) { + // printf("expr_rule->patterns[%zu].pat:%s\n", i, expr_rule->patterns[i].pat); + // printf("expr_rule->patterns[%zu].pat_len:%zu\n", i, expr_rule->patterns[i].pat_len); + // printf("expr_rule->patterns[%zu].l_offset:%d\n", i, expr_rule->patterns[i].l_offset); + // printf("expr_rule->patterns[%zu].r_offset:%d\n", i, expr_rule->patterns[i].r_offset); + // } return expr_rule; } @@ -755,7 +796,7 @@ int expr_runtime_commit(void *expr_runtime, const char *table_name) log_info(expr_rt->logger, MODULE_EXPR, "table[%s] committing %zu expr rules for rebuilding adapter_hs engine", table_name, rule_cnt); - new_adapter_hs = adapter_hs_initialize(expr_rt->scan_mode, + new_adapter_hs = adapter_hs_initialize(expr_rt->scan_mode, expr_rt->pattern_type, expr_rt->n_worker_thread, rules, rule_cnt, expr_rt->logger); @@ -781,7 +822,7 @@ int expr_runtime_commit(void *expr_runtime, const char *table_name) return ret; } -int expr_runtime_scan_string(struct expr_runtime *expr_rt, int thread_id, +int expr_runtime_scan(struct expr_runtime *expr_rt, int thread_id, const char *data, size_t data_len, int *group_ids, size_t group_ids_size, int vtable_id, struct maat_state *state) @@ -839,7 +880,7 @@ void expr_runtime_stream_open(struct expr_runtime *expr_rt, int thread_id) expr_rt->hs_stream = hs_stream; } -int expr_runtime_scan_stream(struct expr_runtime *expr_rt, const char *data, +int expr_runtime_stream_scan(struct expr_runtime *expr_rt, const char *data, size_t data_len, int *group_ids, size_t group_ids_size, int vtable_id, struct maat_state *state) { diff --git a/src/maat_flag.cpp b/src/maat_flag.c similarity index 99% rename from src/maat_flag.cpp rename to src/maat_flag.c index 3e50dcd..17b57bc 100644 --- a/src/maat_flag.cpp +++ b/src/maat_flag.c @@ -410,7 +410,7 @@ int flag_runtime_commit(void *flag_runtime, const char *table_name) return ret; } -int flag_runtime_scan_flag(struct flag_runtime *flag_rt, int thread_id, +int flag_runtime_scan(struct flag_runtime *flag_rt, int thread_id, uint64_t flag, int *group_ids, size_t group_ids_size, int vtable_id, struct maat_state *state) { diff --git a/src/maat_fqdn.c b/src/maat_fqdn.c new file mode 100644 index 0000000..b23788a --- /dev/null +++ b/src/maat_fqdn.c @@ -0,0 +1,72 @@ +/* +********************************************************************************************** +* File: maat_fqdn.cpp +* Description: +* Authors: Liu WenTan +* Date: 2022-10-31 +* Copyright: (c) 2018-2022 Geedge Networks, Inc. All rights reserved. +*********************************************************************************************** +*/ + +#include "maat_fqdn.h" +#include "log/log.h" + +struct fqdn_runtime { + +}; + +void *fqdn_schema_new(cJSON *json, struct table_manager *tbl_mgr, + const char *table_name, struct log_handle *logger) +{ + return NULL; +} + +void fqdn_schema_free(void *fqdn_schema) +{ + +} + +void *fqdn_runtime_new(void *fqdn_schema, int max_thread_num, + struct maat_garbage_bin *garbage_bin, + struct log_handle *logger) +{ + return NULL; +} + +void fqdn_runtime_free(void *fqdn_runtime) +{ + +} + +int fqdn_runtime_update(void *fqdn_runtime, void *fqdn_schema, + const char *line, int valid_column) +{ + return 0; +} + +int fqdn_runtime_commit(void *fqdn_runtime, const char *table_name) +{ + return 0; +} + +struct ex_data_runtime *fqdn_runtime_get_ex_data_rt(struct fqdn_runtime *fqdn_rt) +{ + return NULL; +} + +int fqdn_runtime_scan(struct fqdn_runtime *fqdn_rt, int thread_id, int port, + int proto, int *group_ids, size_t group_id_size, + int vtable_id, struct maat_state *state) +{ + return 0; +} + +void fqdn_runtime_scan_hit_inc(struct fqdn_runtime *fqdn_rt, int thread_id) +{ + +} + +long long fqdn_runtime_scan_hit_sum(struct fqdn_runtime *fqdn_rt, int n_thread) +{ + return 0; +} \ No newline at end of file diff --git a/src/maat_fqdn_plugin.cpp b/src/maat_fqdn_plugin.c similarity index 100% rename from src/maat_fqdn_plugin.cpp rename to src/maat_fqdn_plugin.c diff --git a/src/maat_garbage_collection.cpp b/src/maat_garbage_collection.c similarity index 100% rename from src/maat_garbage_collection.cpp rename to src/maat_garbage_collection.c diff --git a/src/maat_group.cpp b/src/maat_group.c similarity index 99% rename from src/maat_group.cpp rename to src/maat_group.c index 386a2f0..0c374b6 100644 --- a/src/maat_group.cpp +++ b/src/maat_group.c @@ -11,11 +11,12 @@ #include #include +#include "log/log.h" #include "maat_group.h" #include "maat_utils.h" #include "uthash/uthash.h" #include "igraph/igraph.h" -#include "log/log.h" +#include "maat_kv.h" #define MODULE_GROUP module_name_str("maat.group") diff --git a/src/maat_interval.cpp b/src/maat_interval.c similarity index 99% rename from src/maat_interval.cpp rename to src/maat_interval.c index 89ed40a..5beb66d 100644 --- a/src/maat_interval.cpp +++ b/src/maat_interval.c @@ -439,7 +439,7 @@ int interval_runtime_commit(void *interval_runtime, const char *table_name) return ret; } -int interval_runtime_scan_intval(struct interval_runtime *interval_rt, int thread_id, +int interval_runtime_scan(struct interval_runtime *interval_rt, int thread_id, uint64_t integer, int *group_ids, size_t group_ids_size, int vtable_id, struct maat_state *state) diff --git a/src/maat_ip.cpp b/src/maat_ip.c similarity index 99% rename from src/maat_ip.cpp rename to src/maat_ip.c index c4be7f3..6f821f8 100644 --- a/src/maat_ip.cpp +++ b/src/maat_ip.c @@ -12,7 +12,6 @@ #include #include "log/log.h" -#include "cJSON/cJSON.h" #include "maat_utils.h" #include "maat_ex_data.h" #include "IPMatcher.h" @@ -479,7 +478,7 @@ int ip_runtime_commit(void *ip_runtime, const char *table_name) return ret; } -int ip_runtime_scan_ip(struct ip_runtime *ip_rt, int thread_id, int ip_type, +int ip_runtime_scan(struct ip_runtime *ip_rt, int thread_id, int ip_type, uint8_t *ip_addr, int *group_ids, size_t group_id_size, int vtable_id, struct maat_state *state) { diff --git a/src/maat_ip_plugin.cpp b/src/maat_ip_plugin.c similarity index 99% rename from src/maat_ip_plugin.cpp rename to src/maat_ip_plugin.c index e555641..7cd5741 100644 --- a/src/maat_ip_plugin.cpp +++ b/src/maat_ip_plugin.c @@ -10,6 +10,7 @@ #include +#include "log/log.h" #include "maat_utils.h" #include "maat_ip_plugin.h" #include "maat_ex_data.h" diff --git a/src/maat_kv.cpp b/src/maat_kv.c similarity index 99% rename from src/maat_kv.cpp rename to src/maat_kv.c index 03ab275..59c8443 100644 --- a/src/maat_kv.cpp +++ b/src/maat_kv.c @@ -8,7 +8,6 @@ *********************************************************************************************** */ -#include #include #include "uthash/uthash.h" diff --git a/src/maat_plugin.cpp b/src/maat_plugin.c similarity index 99% rename from src/maat_plugin.cpp rename to src/maat_plugin.c index 78ace61..7c28c14 100644 --- a/src/maat_plugin.cpp +++ b/src/maat_plugin.c @@ -11,7 +11,6 @@ #include #include "log/log.h" -#include "cJSON/cJSON.h" #include "maat_utils.h" #include "maat_rule.h" #include "maat_plugin.h" diff --git a/src/maat_port_proto.c b/src/maat_port_proto.c new file mode 100644 index 0000000..1152bca --- /dev/null +++ b/src/maat_port_proto.c @@ -0,0 +1,69 @@ +/* +********************************************************************************************** +* File: maat_ip.cpp +* Description: +* Authors: Liu WenTan +* Date: 2022-10-31 +* Copyright: (c) 2018-2022 Geedge Networks, Inc. All rights reserved. +*********************************************************************************************** +*/ + +#include "maat_port_proto.h" +#include "log/log.h" + +void *pp_schema_new(cJSON *json, struct table_manager *tbl_mgr, + const char *table_name, struct log_handle *logger) +{ + +} + +void pp_schema_free(void *pp_schema) +{ + +} + +/* ip runtime API */ +void *pp_runtime_new(void *pp_schema, int max_thread_num, + struct maat_garbage_bin *garbage_bin, + struct log_handle *logger) +{ + +} + +void pp_runtime_free(void *pp_runtime) +{ + +} + +int pp_runtime_update(void *pp_runtime, void *pp_schema, + const char *line, int valid_column) +{ + +} + +int pp_runtime_commit(void *pp_runtime, const char *table_name) +{ + +} + +struct ex_data_runtime *pp_runtime_get_ex_data_rt(struct pp_runtime *pp_rt) +{ + +} + +int pp_runtime_scan(struct pp_runtime *pp_rt, int thread_id, int port, + int proto, int *group_ids, size_t group_id_size, + int vtable_id, struct maat_state *state) +{ + +} + +void pp_runtime_scan_hit_inc(struct pp_runtime *pp_rt, int thread_id) +{ + +} + +long long pp_runtime_scan_hit_sum(struct pp_runtime *pp_rt, int n_thread) +{ + +} \ No newline at end of file diff --git a/src/maat_redis_monitor.cpp b/src/maat_redis_monitor.c similarity index 99% rename from src/maat_redis_monitor.cpp rename to src/maat_redis_monitor.c index 3f57f01..0240356 100644 --- a/src/maat_redis_monitor.cpp +++ b/src/maat_redis_monitor.c @@ -13,7 +13,7 @@ #include #include -#include "maat/maat.h" +#include "maat.h" #include "maat_utils.h" #include "maat_command.h" #include "maat_config_monitor.h" diff --git a/src/maat_rule.cpp b/src/maat_rule.c similarity index 100% rename from src/maat_rule.cpp rename to src/maat_rule.c diff --git a/src/maat_table.cpp b/src/maat_table.c similarity index 99% rename from src/maat_table.cpp rename to src/maat_table.c index 6ef6d8b..3a418b2 100644 --- a/src/maat_table.cpp +++ b/src/maat_table.c @@ -15,7 +15,6 @@ #include "maat_utils.h" #include "maat_table.h" #include "maat_rule.h" -#include "maat_garbage_collection.h" #include "maat_kv.h" #include "maat_expr.h" #include "maat_ip.h" diff --git a/src/maat_utils.cpp b/src/maat_utils.c similarity index 97% rename from src/maat_utils.cpp rename to src/maat_utils.c index d23e7ff..fd4b229 100644 --- a/src/maat_utils.cpp +++ b/src/maat_utils.c @@ -32,6 +32,13 @@ const char *module_name_str(const char *name) return module; } +inline void ipv6_ntoh(unsigned int *v6_addr) +{ + for (unsigned int i = 0; i < 4; i++) { + v6_addr[i] = ntohl(v6_addr[i]); + } +} + char *maat_strdup(const char *s) { if (NULL == s) { @@ -440,7 +447,7 @@ int ip_format2range(int ip_type, enum maat_ip_format format, const char *ip1, co return -1; } - ipv4_addr = ntohl(ipv4_addr); + //ipv4_addr = ntohl(ipv4_addr); uint32_t ipv4_range_end = 0; uint32_t ipv4_mask = 0; switch (format) { @@ -450,7 +457,7 @@ int ip_format2range(int ip_type, enum maat_ip_format format, const char *ip1, co if (ret <= 0) { return -1; } - ipv4_range_end = ntohl(ipv4_range_end); + //ipv4_range_end = ntohl(ipv4_range_end); range_end[0] = ipv4_range_end; break; case IP_FORMAT_MASK: @@ -458,7 +465,7 @@ int ip_format2range(int ip_type, enum maat_ip_format format, const char *ip1, co if (ret <= 0) { return -1; } - ipv4_mask = ntohl(ipv4_mask); + //ipv4_mask = ntohl(ipv4_mask); range_begin[0] = ipv4_addr & ipv4_mask; range_end[0] = ipv4_addr | ~ipv4_mask; break; @@ -485,7 +492,7 @@ int ip_format2range(int ip_type, enum maat_ip_format format, const char *ip1, co if (ret <= 0) { return -1; } - ipv6_ntoh(ipv6_addr); + //ipv6_ntoh(ipv6_addr); switch (format) { case IP_FORMAT_RANGE: @@ -493,7 +500,7 @@ int ip_format2range(int ip_type, enum maat_ip_format format, const char *ip1, co if (ret <= 0) { return -1; } - ipv6_ntoh(ipv6_range_end); + //ipv6_ntoh(ipv6_range_end); memcpy(range_begin, ipv6_addr, sizeof(ipv6_addr)); memcpy(range_end, ipv6_range_end, sizeof(ipv6_range_end)); @@ -503,7 +510,7 @@ int ip_format2range(int ip_type, enum maat_ip_format format, const char *ip1, co if (ret <= 0) { return -1; } - ipv6_ntoh(ipv6_mask); + //ipv6_ntoh(ipv6_mask); for (i = 0; i < 4; i++) { range_begin[i]=ipv6_addr[i] & ipv6_mask[i]; diff --git a/src/maat_virtual.cpp b/src/maat_virtual.c similarity index 99% rename from src/maat_virtual.cpp rename to src/maat_virtual.c index 8785f08..097ed6a 100644 --- a/src/maat_virtual.cpp +++ b/src/maat_virtual.c @@ -10,7 +10,6 @@ #include -#include "cJSON/cJSON.h" #include "maat_kv.h" #include "maat_utils.h" #include "log/log.h" diff --git a/src/rcu_hash.cpp b/src/rcu_hash.c similarity index 100% rename from src/rcu_hash.cpp rename to src/rcu_hash.c diff --git a/src/version.map b/src/version.map new file mode 100644 index 0000000..683189d --- /dev/null +++ b/src/version.map @@ -0,0 +1,9 @@ +VERS_3.0{ +global: + extern "C" { + *MAAT_FRAME_VERSION_*; + *maat_*; + *bool_matcher_*; + }; +local: *; +}; \ No newline at end of file diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 79788f0..f5596b6 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -1,9 +1,10 @@ +include_directories(${PROJECT_SOURCE_DIR}/include) include_directories(${PROJECT_SOURCE_DIR}/src/inc_internal) include_directories(${PROJECT_SOURCE_DIR}/deps) include_directories(${PROJECT_SOURCE_DIR}/scanner) add_executable(maat_api_gtest maat_api_gtest.cpp) -target_link_libraries(maat_api_gtest maat_frame_static gtest_static) +target_link_libraries(maat_api_gtest maat_frame_shared gtest_static) add_executable(rcu_hash_gtest rcu_hash_gtest.cpp) target_link_libraries(rcu_hash_gtest maat_frame_static gtest_static) @@ -19,5 +20,6 @@ target_link_libraries(adapter_hs_gtest maat_frame_static gtest_static) file(COPY rule DESTINATION ./) file(COPY table_info.conf DESTINATION ./) -file(COPY and_expr.conf DESTINATION ./) +file(COPY literal_expr.conf DESTINATION ./) +file(COPY regex_expr.conf DESTINATION ./) file(COPY maat_json.json DESTINATION ./) \ No newline at end of file diff --git a/test/adapter_hs_gtest.cpp b/test/adapter_hs_gtest.cpp index 2b0145f..5d67760 100644 --- a/test/adapter_hs_gtest.cpp +++ b/test/adapter_hs_gtest.cpp @@ -3,61 +3,146 @@ #include "log/log.h" #include "adapter_hs.h" #include "maat_utils.h" +#include "cJSON/cJSON.h" struct log_handle *g_logger = NULL; -int parse_and_expr_file(const char *filename, and_expr_t expr[], size_t *n_expr) +enum hs_match_mode match_method_to_match_mode(const char *method) { - FILE *fp = fopen(filename, "r"); + enum hs_match_mode mode = HS_MATCH_MODE_MAX; - char line[4096] = {0}; - size_t i = 0; - while (NULL != fgets(line, sizeof(line), fp)) { - if (line[0] == '#' || line[0] == ' ' || line[0] == '\t') { - memset(line, 0, sizeof(line)); - continue; + if (strcmp(method, "sub") == 0) { + mode = HS_MATCH_MODE_SUB; + } else if (strcmp(method, "exactly") == 0) { + mode = HS_MATCH_MODE_EXACTLY; + } else if (strcmp(method, "prefix") == 0) { + mode = HS_MATCH_MODE_PREFIX; + } else if (strcmp(method, "suffix") == 0) { + mode = HS_MATCH_MODE_SUFFIX; + } else { + assert(0); + } + + return mode; +} + +enum hs_case_sensitive case_sensitive_str_to_enum(const char *str) +{ + enum hs_case_sensitive case_sensitive = HS_CASE_SENSITIVE; + + if (strcmp(str, "yes") == 0) { + case_sensitive = HS_CASE_SENSITIVE; + } else if (strcmp(str, "no") == 0) { + case_sensitive = HS_CASE_INSESITIVE; + } else { + assert(0); + } + + return case_sensitive; +} + +int parse_config_file(const char *filename, and_expr_t exprs[], size_t *n_expr) +{ + unsigned char *json_buff = NULL; + size_t json_buff_size = 0; + + int ret = load_file_to_memory(filename, &json_buff, &json_buff_size); + if (ret < 0) { + printf("load file:%s to memory failed.\n", filename); + return -1; + } + + size_t rule_cnt = 0; + cJSON *rules_obj = NULL; + cJSON *root = cJSON_Parse((const char *)json_buff); + if (NULL == root) { + printf("Error before: %-200.200s\n", cJSON_GetErrorPtr()); + ret = -1; + goto next; + } + + rules_obj = cJSON_GetObjectItem(root, "expr_rules"); + if (NULL == rules_obj) { + printf("Error before: %-200.200s\n", cJSON_GetErrorPtr()); + ret = -1; + goto next; + } + + rule_cnt = cJSON_GetArraySize(rules_obj); + for (size_t i = 0; i < rule_cnt; i++) { + cJSON *expr_obj = cJSON_GetArrayItem(rules_obj, i); + cJSON *tmp_item = cJSON_GetObjectItem(expr_obj, "expr_id"); + if (tmp_item != NULL && tmp_item->type == cJSON_Number) { + exprs[i].expr_id = tmp_item->valueint; } - char pattern_buf[1024] = {0}; - int ret = sscanf(line, "%u\t%lu\t%s", &(expr[i].expr_id), &(expr[i].n_patterns), pattern_buf); - EXPECT_NE(ret, 0); + tmp_item = cJSON_GetObjectItem(expr_obj, "pattern_num"); + if (tmp_item != NULL && tmp_item->type == cJSON_Number) { + exprs[i].n_patterns = tmp_item->valueint; + } - str_unescape(pattern_buf); - char *expr_token = NULL; - char *sub_expr_token = NULL; - char *save_expr_ptr = NULL; - size_t j = 0; - for (expr_token = pattern_buf; ; expr_token = NULL) { - sub_expr_token = strtok_r(expr_token, "&", &save_expr_ptr); - if (sub_expr_token == NULL) { - break; + tmp_item = cJSON_GetObjectItem(expr_obj, "patterns"); + if (NULL == tmp_item || tmp_item->type != cJSON_Array) { + printf("json has no patterns array.\n"); + ret = -1; + goto next; + } + + size_t pattern_cnt = cJSON_GetArraySize(tmp_item); + for (size_t j = 0; j < pattern_cnt; j++) { + cJSON *pat_item = cJSON_GetArrayItem(tmp_item, j); + cJSON *item = cJSON_GetObjectItem(pat_item, "match_method"); + if (item != NULL && item->type == cJSON_String) { + exprs[i].patterns[j].match_mode = match_method_to_match_mode(item->valuestring); } - char *pattern_token = sub_expr_token; - char *save_pattern_ptr = NULL; - char *sub_pattern_token = strtok_r(pattern_token, ":", &save_pattern_ptr); - expr[i].patterns[j].type = atoi(sub_pattern_token); - size_t str_len = strlen(save_pattern_ptr); - expr[i].patterns[j].pat = (char *)calloc(sizeof(char), str_len); - memcpy(expr[i].patterns[j].pat, save_pattern_ptr, str_len); - expr[i].patterns[j].pat_len = str_len; - j++; - } - i++; - memset(line, 0, sizeof(line)); - } - *n_expr = i; - fclose(fp); + item = cJSON_GetObjectItem(pat_item, "case_sensitive"); + if (item != NULL && item->type == cJSON_String) { + exprs[i].patterns[j].case_sensitive = case_sensitive_str_to_enum(item->valuestring); + } - return 0; + item = cJSON_GetObjectItem(pat_item, "pattern"); + if (item != NULL && item->type == cJSON_String) { + exprs[i].patterns[j].pat = ALLOC(char, strlen(item->valuestring)); + memcpy(exprs[i].patterns[j].pat, item->valuestring, strlen(item->valuestring)); + exprs[i].patterns[j].pat_len = strlen(item->valuestring); + } + + if (exprs[i].patterns->match_mode == HS_MATCH_MODE_SUB) { + item = cJSON_GetObjectItem(pat_item, "offset"); + if (item != NULL && item->type == cJSON_String) { + int key_left_offset = -1; + int key_right_offset = -1; + sscanf(item->valuestring, "%d~%d", &key_left_offset, &key_right_offset); + if (key_left_offset < -1 || key_right_offset < -1) { + printf("Error: offset should not less than -1, left_offset:%d, right_offset:%d\n", + key_left_offset, key_right_offset); + } + exprs[i].patterns[j].l_offset = key_left_offset; + exprs[i].patterns[j].r_offset = key_right_offset; + } else { + exprs[i].patterns[j].l_offset = -1; + exprs[i].patterns[j].r_offset = -1; + } + } + } + exprs[i].n_patterns = pattern_cnt; + } + + *n_expr = rule_cnt; +next: + cJSON_Delete(root); + return ret; } void expr_array_free(and_expr_t expr_array[], size_t n_expr_array) { for (size_t i = 0; i < n_expr_array; i++) { for (size_t j = 0; j < expr_array[i].n_patterns; j++) { - free(expr_array[i].patterns[j].pat); - expr_array[i].patterns[j].pat = NULL; + if (expr_array[i].patterns[j].pat != NULL) { + free(expr_array[i].patterns[j].pat); + expr_array[i].patterns[j].pat = NULL; + } } } } @@ -68,15 +153,15 @@ TEST(block_mode_initialize, invalid_input_parameter) and_expr_t exprs[64]; /* case1: invalid scan_mode parameter */ - hs_instance = adapter_hs_initialize(0, 1, exprs, 1, g_logger); + hs_instance = adapter_hs_initialize(HS_SCAN_MODE_MAX, HS_PATTERN_TYPE_REG, 1, exprs, 1, g_logger); EXPECT_EQ(hs_instance, nullptr); /* case2: invalid expr parameter */ - hs_instance = adapter_hs_initialize(HS_SCAN_MODE_BLOCK, 1, nullptr, 1, g_logger); + hs_instance = adapter_hs_initialize(HS_SCAN_MODE_BLOCK, HS_PATTERN_TYPE_REG, 1, nullptr, 1, g_logger); EXPECT_EQ(hs_instance, nullptr); /* case3: invalid expr num */ - hs_instance = adapter_hs_initialize(HS_SCAN_MODE_BLOCK, 1, exprs, 0, g_logger); + hs_instance = adapter_hs_initialize(HS_SCAN_MODE_BLOCK, HS_PATTERN_TYPE_REG, 1, exprs, 0, g_logger); EXPECT_EQ(hs_instance, nullptr); } @@ -85,312 +170,398 @@ TEST(block_mode_scan, invalid_input_parameter) and_expr_t expr_array[64]; size_t n_expr_array = 0; - struct adapter_hs *hs_instance = adapter_hs_initialize(HS_SCAN_MODE_BLOCK, 1, nullptr, 0, g_logger); + struct adapter_hs *hs_instance = adapter_hs_initialize(HS_SCAN_MODE_BLOCK, HS_PATTERN_TYPE_REG, 1, nullptr, 0, g_logger); EXPECT_EQ(hs_instance, nullptr); - hs_instance = adapter_hs_initialize(0, 1, expr_array, n_expr_array, g_logger); + hs_instance = adapter_hs_initialize(HS_SCAN_MODE_MAX, HS_PATTERN_TYPE_REG, 1, expr_array, n_expr_array, g_logger); EXPECT_EQ(hs_instance, nullptr); n_expr_array = 1; expr_array[0].expr_id = 101; expr_array[0].n_patterns = 10; - hs_instance = adapter_hs_initialize(HS_SCAN_MODE_BLOCK, 1, expr_array, n_expr_array, g_logger); + hs_instance = adapter_hs_initialize(HS_SCAN_MODE_BLOCK, HS_PATTERN_TYPE_REG, 1, expr_array, n_expr_array, g_logger); EXPECT_EQ(hs_instance, nullptr); memset(expr_array, 0, sizeof(expr_array)); n_expr_array = 1; expr_array[0].expr_id = 101; expr_array[0].n_patterns = 1; - expr_array[0].patterns[0].type = 0; - hs_instance = adapter_hs_initialize(HS_SCAN_MODE_BLOCK, 1, expr_array, n_expr_array, g_logger); + hs_instance = adapter_hs_initialize(HS_SCAN_MODE_BLOCK, HS_PATTERN_TYPE_REG, 1, expr_array, n_expr_array, g_logger); EXPECT_EQ(hs_instance, nullptr); } -TEST(block_mode_scan, literal_hit1) +TEST(block_mode_scan, literal_sub_has_normal_offset) { and_expr_t expr_array[64] = {0}; size_t n_expr_array = 0; - - int ret = parse_and_expr_file("./and_expr.conf", expr_array, &n_expr_array); - EXPECT_EQ(ret, 0); - EXPECT_EQ(n_expr_array, 10); - struct adapter_hs *hs_instance = adapter_hs_initialize(HS_SCAN_MODE_BLOCK, 1, expr_array, n_expr_array, g_logger); + int ret = parse_config_file("./literal_expr.conf", expr_array, &n_expr_array); + EXPECT_EQ(ret, 0); + EXPECT_EQ(n_expr_array, 15); + + struct adapter_hs *hs_instance = adapter_hs_initialize(HS_SCAN_MODE_BLOCK, HS_PATTERN_TYPE_STR, 1, expr_array, n_expr_array, g_logger); EXPECT_NE(hs_instance, nullptr); expr_array_free(expr_array, n_expr_array); - char data0[64] = "luis"; - struct hs_scan_result result0[64] = {0}; - size_t n_result0 = 0; - ret = adapter_hs_scan(hs_instance, 0, data0, strlen(data0), result0, 64, &n_result0); - EXPECT_EQ(ret, 0); - EXPECT_EQ(n_result0, 0); - - char data1[64] = "hello"; - struct hs_scan_result result1[64] = {0}; - size_t n_result1 = 0; - ret = adapter_hs_scan(hs_instance, 0, data1, strlen(data1), result1, 64, &n_result1); - EXPECT_EQ(ret, 0); - EXPECT_EQ(n_result1, 1); - EXPECT_EQ(result1[0].item_id, 101); - - char data2[64] = "world"; - struct hs_scan_result result2[64] = {0}; - size_t n_result2 = 0; - ret = adapter_hs_scan(hs_instance, 0, data2, strlen(data2), result2, 64, &n_result2); - EXPECT_EQ(ret, 0); - EXPECT_EQ(n_result2, 1); - EXPECT_EQ(result2[0].item_id, 102); - - adapter_hs_destroy(hs_instance); - hs_instance = nullptr; -} - -TEST(block_mode_scan, literal_hit2) -{ - and_expr_t expr_array[64] = {0}; - size_t n_expr_array = 0; - - int ret = parse_and_expr_file("./and_expr.conf", expr_array, &n_expr_array); - EXPECT_EQ(ret, 0); - - struct adapter_hs *hs_instance = adapter_hs_initialize(HS_SCAN_MODE_BLOCK, 1, - expr_array, n_expr_array, g_logger); - EXPECT_NE(hs_instance, nullptr); - expr_array_free(expr_array, n_expr_array); - - char data0[64] = "hello maat"; - struct hs_scan_result result0[64] = {0}; - size_t n_result0 = 0; - ret = adapter_hs_scan(hs_instance, 0, data0, strlen(data0), result0, 64, &n_result0); - EXPECT_EQ(ret, 0); - EXPECT_EQ(n_result0, 2); - EXPECT_EQ(result0[0].item_id, 103); - EXPECT_EQ(result0[1].item_id, 101); - - char data1[64] = "maat World"; - struct hs_scan_result result1[64] = {0}; - size_t n_result1 = 0; - ret = adapter_hs_scan(hs_instance, 0, data1, strlen(data1), result1, 64, &n_result1); - EXPECT_EQ(ret, 0); - EXPECT_EQ(n_result1, 2); - EXPECT_EQ(result1[0].item_id, 103); - EXPECT_EQ(result1[1].item_id, 102); - - adapter_hs_destroy(hs_instance); - hs_instance = nullptr; -} - -TEST(block_mode_scan, literal_hit3) -{ - and_expr_t expr_array[64] = {0}; - size_t n_expr_array = 0; - - int ret = parse_and_expr_file("./and_expr.conf", expr_array, &n_expr_array); - EXPECT_EQ(ret, 0); - - struct adapter_hs *hs_instance = adapter_hs_initialize(HS_SCAN_MODE_BLOCK, 1, expr_array, n_expr_array, g_logger); - EXPECT_NE(hs_instance, nullptr); - expr_array_free(expr_array, n_expr_array); - - char data0[64] = "hello world"; - struct hs_scan_result result0[64] = {0}; - size_t n_result0 = 0; - ret = adapter_hs_scan(hs_instance, 0, data0, strlen(data0), result0, 64, &n_result0); - EXPECT_EQ(ret, 0); - EXPECT_EQ(n_result0, 3); - EXPECT_EQ(result0[0].item_id, 104); - EXPECT_EQ(result0[1].item_id, 102); - EXPECT_EQ(result0[2].item_id, 101); - - char data1[64] = "hello World"; - struct hs_scan_result result1[64] = {0}; - size_t n_result1 = 0; - ret = adapter_hs_scan(hs_instance, 0, data1, strlen(data1), result1, 64, &n_result1); - EXPECT_EQ(ret, 0); - EXPECT_EQ(n_result1, 2); - EXPECT_EQ(result1[0].item_id, 102); - EXPECT_EQ(result1[1].item_id, 101); - - adapter_hs_destroy(hs_instance); - hs_instance = nullptr; -} - -TEST(block_mode_scan, literal_russian_hit3) -{ - and_expr_t expr_array[64] = {0}; - size_t n_expr_array = 0; - - int ret = parse_and_expr_file("./and_expr.conf", expr_array, &n_expr_array); - EXPECT_EQ(ret, 0); - - struct adapter_hs *hs_instance = adapter_hs_initialize(HS_SCAN_MODE_BLOCK, 1, expr_array, n_expr_array, g_logger); - EXPECT_NE(hs_instance, nullptr); - expr_array_free(expr_array, n_expr_array); - - char data0[64] = "يىلىدىكى"; - struct hs_scan_result result0[64] = {0}; - size_t n_result0 = 0; - ret = adapter_hs_scan(hs_instance, 0, data0, strlen(data0), result0, 64, &n_result0); - EXPECT_EQ(ret, 0); - EXPECT_EQ(n_result0, 1); - EXPECT_EQ(result0[0].item_id, 109); - - adapter_hs_destroy(hs_instance); - hs_instance = nullptr; -} - -TEST(block_mode_scan, literal_case_insensitive_hit4) -{ - and_expr_t expr_array[64] = {0}; - size_t n_expr_array = 0; - - int ret = parse_and_expr_file("./and_expr.conf", expr_array, &n_expr_array); - EXPECT_EQ(ret, 0); - - struct adapter_hs *hs_instance = adapter_hs_initialize(HS_SCAN_MODE_BLOCK, 1, expr_array, n_expr_array, g_logger); - EXPECT_NE(hs_instance, nullptr); - expr_array_free(expr_array, n_expr_array); - - char data0[64] = "today"; + char scan_data1[64] = "hello aaa"; struct hs_scan_result result[64] = {0}; size_t n_result = 0; - ret = adapter_hs_scan(hs_instance, 0, data0, strlen(data0), result, 64, &n_result); + + ret = adapter_hs_scan(hs_instance, 0, scan_data1, strlen(scan_data1), result, 64, &n_result); EXPECT_EQ(ret, 0); EXPECT_EQ(n_result, 0); - char data1[64] = "Today"; - n_result = 0; + char scan_data2[64] = "Ahello aaa"; memset(result, 0, sizeof(result)); - ret = adapter_hs_scan(hs_instance, 0, data1, strlen(data1), result, 64, &n_result); + n_result = 0; + ret = adapter_hs_scan(hs_instance, 0, scan_data2, strlen(scan_data2), result, 64, &n_result); EXPECT_EQ(ret, 0); EXPECT_EQ(n_result, 1); - EXPECT_EQ(result[0].item_id, 110); + EXPECT_EQ(result[0].item_id, 101); + + char scan_data3[64] = "Aahello aaa"; + memset(result, 0, sizeof(result)); + n_result = 0; + ret = adapter_hs_scan(hs_instance, 0, scan_data3, strlen(scan_data3), result, 64, &n_result); + EXPECT_EQ(ret, 0); + EXPECT_EQ(n_result, 0); adapter_hs_destroy(hs_instance); hs_instance = nullptr; } -TEST(block_mode_scan, literal_and_regex1) +TEST(block_mode_scan, literal_sub_has_left_unlimit_offset) +{ + and_expr_t expr_array[64] = {0}; + size_t n_expr_array = 0; + + int ret = parse_config_file("./literal_expr.conf", expr_array, &n_expr_array); + EXPECT_EQ(ret, 0); + EXPECT_EQ(n_expr_array, 15); + + struct adapter_hs *hs_instance = adapter_hs_initialize(HS_SCAN_MODE_BLOCK, HS_PATTERN_TYPE_STR, 1, expr_array, n_expr_array, g_logger); + EXPECT_NE(hs_instance, nullptr); + expr_array_free(expr_array, n_expr_array); + + char scan_data1[64] = "hello bbb"; + struct hs_scan_result result[64] = {0}; + size_t n_result = 0; + + ret = adapter_hs_scan(hs_instance, 0, scan_data1, strlen(scan_data1), result, 64, &n_result); + EXPECT_EQ(ret, 0); + EXPECT_EQ(n_result, 1); + EXPECT_EQ(result[0].item_id, 102); + + char scan_data2[64] = "Ahello bbb"; + memset(result, 0, sizeof(result)); + n_result = 0; + ret = adapter_hs_scan(hs_instance, 0, scan_data2, strlen(scan_data2), result, 64, &n_result); + EXPECT_EQ(ret, 0); + EXPECT_EQ(n_result, 1); + EXPECT_EQ(result[0].item_id, 102); + + char scan_data3[64] = "Aahello bbb"; + memset(result, 0, sizeof(result)); + n_result = 0; + ret = adapter_hs_scan(hs_instance, 0, scan_data3, strlen(scan_data3), result, 64, &n_result); + EXPECT_EQ(ret, 0); + EXPECT_EQ(n_result, 0); + + adapter_hs_destroy(hs_instance); + hs_instance = nullptr; +} + +TEST(block_mode_scan, literal_sub_has_right_unlimit_offset) +{ + and_expr_t expr_array[64] = {0}; + size_t n_expr_array = 0; + + int ret = parse_config_file("./literal_expr.conf", expr_array, &n_expr_array); + EXPECT_EQ(ret, 0); + EXPECT_EQ(n_expr_array, 15); + + struct adapter_hs *hs_instance = adapter_hs_initialize(HS_SCAN_MODE_BLOCK, HS_PATTERN_TYPE_STR, 1, expr_array, n_expr_array, g_logger); + EXPECT_NE(hs_instance, nullptr); + expr_array_free(expr_array, n_expr_array); + + char scan_data1[64] = "hello ccc"; + struct hs_scan_result result[64] = {0}; + size_t n_result = 0; + + ret = adapter_hs_scan(hs_instance, 0, scan_data1, strlen(scan_data1), result, 64, &n_result); + EXPECT_EQ(ret, 0); + EXPECT_EQ(n_result, 0); + + char scan_data2[64] = "1234hello ccc"; + memset(result, 0, sizeof(result)); + n_result = 0; + ret = adapter_hs_scan(hs_instance, 0, scan_data2, strlen(scan_data2), result, 64, &n_result); + EXPECT_EQ(ret, 0); + EXPECT_EQ(n_result, 0); + + char scan_data3[64] = "12345hello ccc"; + memset(result, 0, sizeof(result)); + n_result = 0; + ret = adapter_hs_scan(hs_instance, 0, scan_data3, strlen(scan_data3), result, 64, &n_result); + EXPECT_EQ(ret, 0); + EXPECT_EQ(n_result, 1); + EXPECT_EQ(result[0].item_id, 103); + + char scan_data4[64] = "12345hello cccAaBb"; + memset(result, 0, sizeof(result)); + n_result = 0; + ret = adapter_hs_scan(hs_instance, 0, scan_data4, strlen(scan_data4), result, 64, &n_result); + EXPECT_EQ(ret, 0); + EXPECT_EQ(n_result, 1); + EXPECT_EQ(result[0].item_id, 103); + + char scan_data5[64] = "123456hello cccAaBb"; + memset(result, 0, sizeof(result)); + n_result = 0; + ret = adapter_hs_scan(hs_instance, 0, scan_data5, strlen(scan_data5), result, 64, &n_result); + EXPECT_EQ(ret, 0); + EXPECT_EQ(n_result, 1); + EXPECT_EQ(result[0].item_id, 103); + + adapter_hs_destroy(hs_instance); + hs_instance = nullptr; +} + +TEST(block_mode_scan, literal_sub_with_no_offset) { and_expr_t expr_array[64] = {0}; size_t n_expr_array = 0; - int ret = parse_and_expr_file("./and_expr.conf", expr_array, &n_expr_array); + int ret = parse_config_file("./literal_expr.conf", expr_array, &n_expr_array); EXPECT_EQ(ret, 0); - struct adapter_hs *hs_instance = adapter_hs_initialize(HS_SCAN_MODE_BLOCK, 1, + struct adapter_hs *hs_instance = adapter_hs_initialize(HS_SCAN_MODE_BLOCK, HS_PATTERN_TYPE_STR, 1, expr_array, n_expr_array, g_logger); + EXPECT_NE(hs_instance, nullptr); + expr_array_free(expr_array, n_expr_array); + + char scan_data1[64] = "hello ddd"; + struct hs_scan_result result[64] = {0}; + size_t n_result = 0; + ret = adapter_hs_scan(hs_instance, 0, scan_data1, strlen(scan_data1), result, 64, &n_result); + EXPECT_EQ(ret, 0); + EXPECT_EQ(n_result, 1); + EXPECT_EQ(result[0].item_id, 104); + + char scan_data2[64] = "123hello ddd"; + memset(result, 0, sizeof(result)); + n_result = 0; + ret = adapter_hs_scan(hs_instance, 0, scan_data2, strlen(scan_data2), result, 64, &n_result); + EXPECT_EQ(ret, 0); + EXPECT_EQ(n_result, 1); + EXPECT_EQ(result[0].item_id, 104); + + char scan_data3[64] = "123hello ddd456"; + memset(result, 0, sizeof(result)); + n_result = 0; + ret = adapter_hs_scan(hs_instance, 0, scan_data3, strlen(scan_data3), result, 64, &n_result); + EXPECT_EQ(ret, 0); + EXPECT_EQ(n_result, 1); + EXPECT_EQ(result[0].item_id, 104); + + char scan_data4[64] = "helloddd"; + memset(result, 0, sizeof(result)); + n_result = 0; + ret = adapter_hs_scan(hs_instance, 0, scan_data4, strlen(scan_data4), result, 64, &n_result); + EXPECT_EQ(ret, 0); + EXPECT_EQ(n_result, 0); + + adapter_hs_destroy(hs_instance); + hs_instance = nullptr; +} + +TEST(block_mode_scan, literal_exactly) +{ + and_expr_t expr_array[64] = {0}; + size_t n_expr_array = 0; + + int ret = parse_config_file("./literal_expr.conf", expr_array, &n_expr_array); + EXPECT_EQ(ret, 0); + + struct adapter_hs *hs_instance = adapter_hs_initialize(HS_SCAN_MODE_BLOCK, HS_PATTERN_TYPE_STR, 1, expr_array, n_expr_array, g_logger); + EXPECT_NE(hs_instance, nullptr); + expr_array_free(expr_array, n_expr_array); + + char scan_data1[64] = "hello eee"; + struct hs_scan_result result[64] = {0}; + size_t n_result = 0; + + ret = adapter_hs_scan(hs_instance, 0, scan_data1, strlen(scan_data1), result, 64, &n_result); + EXPECT_EQ(ret, 0); + EXPECT_EQ(n_result, 1); + EXPECT_EQ(result[0].item_id, 105); + + char scan_data2[64] = "Ahello eee"; + memset(result, 0, sizeof(result)); + n_result = 0; + + ret = adapter_hs_scan(hs_instance, 0, scan_data2, strlen(scan_data2), result, 64, &n_result); + EXPECT_EQ(ret, 0); + EXPECT_EQ(n_result, 0); + + char scan_data3[64] = "hello eeeB"; + memset(result, 0, sizeof(result)); + n_result = 0; + + ret = adapter_hs_scan(hs_instance, 0, scan_data3, strlen(scan_data3), result, 64, &n_result); + EXPECT_EQ(ret, 0); + EXPECT_EQ(n_result, 0); + + adapter_hs_destroy(hs_instance); + hs_instance = nullptr; +} + +TEST(block_mode_scan, literal_prefix) +{ + and_expr_t expr_array[64] = {0}; + size_t n_expr_array = 0; + + int ret = parse_config_file("./literal_expr.conf", expr_array, &n_expr_array); + EXPECT_EQ(ret, 0); + + struct adapter_hs *hs_instance = adapter_hs_initialize(HS_SCAN_MODE_BLOCK, HS_PATTERN_TYPE_STR, 1, expr_array, n_expr_array, g_logger); EXPECT_NE(hs_instance, nullptr); expr_array_free(expr_array, n_expr_array); - char data0[64] = "1hello world"; - struct hs_scan_result result0[64] = {0}; - size_t n_result0 = 0; - ret = adapter_hs_scan(hs_instance, 0, data0, strlen(data0), result0, 64, &n_result0); - EXPECT_EQ(ret, 0); - EXPECT_EQ(n_result0, 4); - EXPECT_EQ(result0[0].item_id, 105); - EXPECT_EQ(result0[1].item_id, 104); - EXPECT_EQ(result0[2].item_id, 102); - EXPECT_EQ(result0[3].item_id, 101); + char scan_data1[64] = "hello fff"; + struct hs_scan_result result[64] = {0}; + size_t n_result = 0; - char data1[64] = "8hello World"; - struct hs_scan_result result1[64] = {0}; - size_t n_result1 = 0; - ret = adapter_hs_scan(hs_instance, 0, data1, strlen(data1), result1, 64, &n_result1); + ret = adapter_hs_scan(hs_instance, 0, scan_data1, strlen(scan_data1), result, 64, &n_result); EXPECT_EQ(ret, 0); - EXPECT_EQ(n_result1, 2); - EXPECT_EQ(result1[0].item_id, 102); - EXPECT_EQ(result1[1].item_id, 101); + EXPECT_EQ(n_result, 1); + EXPECT_EQ(result[0].item_id, 106); + + char scan_data2[64] = "Ahello fff"; + memset(result, 0, sizeof(result)); + n_result = 0; + + ret = adapter_hs_scan(hs_instance, 0, scan_data2, strlen(scan_data2), result, 64, &n_result); + EXPECT_EQ(ret, 0); + EXPECT_EQ(n_result, 0); + + char scan_data3[64] = "Ahello fffBCD"; + memset(result, 0, sizeof(result)); + n_result = 0; + + ret = adapter_hs_scan(hs_instance, 0, scan_data3, strlen(scan_data3), result, 64, &n_result); + EXPECT_EQ(ret, 0); + EXPECT_EQ(n_result, 0); + + char scan_data4[64] = "hello fffBCD"; + memset(result, 0, sizeof(result)); + n_result = 0; + + ret = adapter_hs_scan(hs_instance, 0, scan_data4, strlen(scan_data4), result, 64, &n_result); + EXPECT_EQ(ret, 0); + EXPECT_EQ(n_result, 1); + EXPECT_EQ(result[0].item_id, 106); adapter_hs_destroy(hs_instance); hs_instance = nullptr; } -TEST(block_mode_scan, literal_and_regex2) +TEST(block_mode_scan, literal_suffix) { and_expr_t expr_array[64] = {0}; size_t n_expr_array = 0; - int ret = parse_and_expr_file("./and_expr.conf", expr_array, &n_expr_array); + int ret = parse_config_file("./literal_expr.conf", expr_array, &n_expr_array); EXPECT_EQ(ret, 0); - struct adapter_hs *hs_instance = adapter_hs_initialize(HS_SCAN_MODE_BLOCK, 1, + struct adapter_hs *hs_instance = adapter_hs_initialize(HS_SCAN_MODE_BLOCK, HS_PATTERN_TYPE_STR, 1, expr_array, n_expr_array, g_logger); EXPECT_NE(hs_instance, nullptr); expr_array_free(expr_array, n_expr_array); - char data0[64] = "1hello 2world"; - struct hs_scan_result result0[64] = {0}; - size_t n_result0 = 0; - ret = adapter_hs_scan(hs_instance, 0, data0, strlen(data0), result0, 64, &n_result0); - EXPECT_EQ(ret, 0); - EXPECT_EQ(n_result0, 5); - EXPECT_EQ(result0[0].item_id, 106); - EXPECT_EQ(result0[1].item_id, 105); - EXPECT_EQ(result0[2].item_id, 104); - EXPECT_EQ(result0[3].item_id, 102); - EXPECT_EQ(result0[4].item_id, 101); + char scan_data1[64] = "hello ggg"; + struct hs_scan_result result[64] = {0}; + size_t n_result = 0; - char data1[64] = "8hello 9World"; - struct hs_scan_result result1[64] = {0}; - size_t n_result1 = 0; - ret = adapter_hs_scan(hs_instance, 0, data1, strlen(data1), result1, 64, &n_result1); + ret = adapter_hs_scan(hs_instance, 0, scan_data1, strlen(scan_data1), result, 64, &n_result); EXPECT_EQ(ret, 0); - EXPECT_EQ(n_result1, 2); - EXPECT_EQ(result1[0].item_id, 102); - EXPECT_EQ(result1[1].item_id, 101); + EXPECT_EQ(n_result, 1); + EXPECT_EQ(result[0].item_id, 107); + + char scan_data2[64] = "ABChello ggg"; + memset(result, 0, sizeof(result)); + n_result = 0; + + ret = adapter_hs_scan(hs_instance, 0, scan_data2, strlen(scan_data2), result, 64, &n_result); + EXPECT_EQ(ret, 0); + EXPECT_EQ(n_result, 1); + EXPECT_EQ(result[0].item_id, 107); + + char scan_data3[64] = "ABChello gggDEF"; + memset(result, 0, sizeof(result)); + n_result = 0; + + ret = adapter_hs_scan(hs_instance, 0, scan_data3, strlen(scan_data3), result, 64, &n_result); + EXPECT_EQ(ret, 0); + EXPECT_EQ(n_result, 0); + + char scan_data4[64] = "hello gggDEF"; + memset(result, 0, sizeof(result)); + n_result = 0; + + ret = adapter_hs_scan(hs_instance, 0, scan_data4, strlen(scan_data4), result, 64, &n_result); + EXPECT_EQ(ret, 0); + EXPECT_EQ(n_result, 0); adapter_hs_destroy(hs_instance); hs_instance = nullptr; } -TEST(block_mode_scan, regex_hit1) +// TEST(block_mode_scan, literal_sub_with_hexbin) +// { +// and_expr_t expr_array[64] = {0}; +// size_t n_expr_array = 0; + +// int ret = parse_config_file("./and_expr.conf", expr_array, &n_expr_array); +// EXPECT_EQ(ret, 0); + +// struct adapter_hs *hs_instance = adapter_hs_initialize(HS_SCAN_MODE_BLOCK, 1, +// expr_array, n_expr_array, g_logger); +// EXPECT_NE(hs_instance, nullptr); +// expr_array_free(expr_array, n_expr_array); + +// char data0[64] = "Cookie: Txa123aheadBCAxd"; +// struct hs_scan_result result0[64] = {0}; +// size_t n_result0 = 0; +// ret = adapter_hs_scan(hs_instance, 0, data0, strlen(data0), result0, 64, &n_result0); +// EXPECT_EQ(ret, 0); +// EXPECT_EQ(n_result0, 1); +// EXPECT_EQ(result0[0].item_id, 107); + +// adapter_hs_destroy(hs_instance); +// hs_instance = nullptr; +// } + +TEST(block_mode_scan, literal_with_chinese) { and_expr_t expr_array[64] = {0}; size_t n_expr_array = 0; - int ret = parse_and_expr_file("./and_expr.conf", expr_array, &n_expr_array); + int ret = parse_config_file("./literal_expr.conf", expr_array, &n_expr_array); EXPECT_EQ(ret, 0); - struct adapter_hs *hs_instance = adapter_hs_initialize(HS_SCAN_MODE_BLOCK, 1, + struct adapter_hs *hs_instance = adapter_hs_initialize(HS_SCAN_MODE_BLOCK, HS_PATTERN_TYPE_STR, 1, expr_array, n_expr_array, g_logger); EXPECT_NE(hs_instance, nullptr); expr_array_free(expr_array, n_expr_array); - char data0[64] = "Cookie: Txa123aheadBCAxd"; + char data0[64] = "#中国 你好"; struct hs_scan_result result0[64] = {0}; size_t n_result0 = 0; ret = adapter_hs_scan(hs_instance, 0, data0, strlen(data0), result0, 64, &n_result0); EXPECT_EQ(ret, 0); EXPECT_EQ(n_result0, 1); - EXPECT_EQ(result0[0].item_id, 107); - - adapter_hs_destroy(hs_instance); - hs_instance = nullptr; -} - -TEST(block_mode_scan, chinese_hit1) -{ - and_expr_t expr_array[64] = {0}; - size_t n_expr_array = 0; - - int ret = parse_and_expr_file("./and_expr.conf", expr_array, &n_expr_array); - EXPECT_EQ(ret, 0); - - struct adapter_hs *hs_instance = adapter_hs_initialize(HS_SCAN_MODE_BLOCK, 1, - expr_array, n_expr_array, g_logger); - EXPECT_NE(hs_instance, nullptr); - expr_array_free(expr_array, n_expr_array); - - char data0[64] = "中国 你好"; - struct hs_scan_result result0[64] = {0}; - size_t n_result0 = 0; - ret = adapter_hs_scan(hs_instance, 0, data0, strlen(data0), result0, 64, &n_result0); - EXPECT_EQ(ret, 0); - EXPECT_EQ(n_result0, 1); - EXPECT_EQ(result0[0].item_id, 108); + EXPECT_EQ(result0[0].item_id, 114); adapter_hs_destroy(hs_instance); hs_instance = nullptr; @@ -405,5 +576,6 @@ int main(int argc, char **argv) ret = RUN_ALL_TESTS(); log_handle_destroy(g_logger); + return ret; } diff --git a/test/and_expr.conf b/test/and_expr.conf deleted file mode 100644 index f795066..0000000 --- a/test/and_expr.conf +++ /dev/null @@ -1,14 +0,0 @@ -# logic AND expressions sample -# expr_id pattern_num expression(several patterns) -# pattern_type:pattern_string&pattern_type:pattern_string&... -# PATTERN_TYPE_STR(0) PATTERN_TYPE_REG(1) -101 1 0:hello -102 1 1:[W|w]orld -103 1 0:maat -104 2 0:hello&0:world -105 2 1:[0-9]hello&0:world -106 2 1:[0-9]hello&1:[0-9]world -107 2 1:Cookie:\\s&1:head -108 1 0:你好 -109 1 0:يىلىدىكى -110 1 0:Today \ No newline at end of file diff --git a/test/literal_expr.conf b/test/literal_expr.conf new file mode 100644 index 0000000..dfbf055 --- /dev/null +++ b/test/literal_expr.conf @@ -0,0 +1,146 @@ +{ + "expr_rules": [ + { + "expr_id": 101, + "pattern_num": 1, + "patterns": [ + { + "match_method": "sub", + "case_sensitive": "yes", + "is_hexbin": "no", + "pattern": "hello aaa", + "offset": "1~10" + } + ] + }, + { + "expr_id": 102, + "pattern_num": 1, + "patterns": [ + { + "match_method": "sub", + "case_sensitive": "yes", + "is_hexbin": "no", + "pattern": "hello bbb", + "offset": "-1~10" + } + ] + }, + { + "expr_id": 103, + "pattern_num": 1, + "patterns": [ + { + "match_method": "sub", + "case_sensitive": "yes", + "is_hexbin": "no", + "pattern": "hello ccc", + "offset": "5~-1" + } + ] + }, + { + "expr_id": 104, + "pattern_num": 1, + "patterns": [ + { + "match_method": "sub", + "case_sensitive": "yes", + "is_hexbin": "no", + "pattern": "hello ddd" + } + ] + }, + { + "expr_id": 105, + "pattern_num": 1, + "patterns": [ + { + "match_method": "exactly", + "case_sensitive": "yes", + "is_hexbin": "no", + "pattern": "hello eee" + } + ] + }, + { + "expr_id": 106, + "pattern_num": 1, + "patterns": [ + { + "match_method": "prefix", + "case_sensitive": "yes", + "is_hexbin": "no", + "pattern": "hello fff" + } + ] + }, + { + "expr_id": 107, + "pattern_num": 1, + "patterns": [ + { + "match_method": "suffix", + "case_sensitive": "yes", + "is_hexbin": "no", + "pattern": "hello ggg" + } + ] + }, + { + "expr_id": 108, + "pattern_num": 1, + "patterns": [ + { + "match_method": "sub", + "case_sensitive": "yes", + "is_hexbin": "yes", + "pattern": "2f68746d6c" + } + ] + }, + + { + "expr_id": 109, + "pattern_num": 2, + "patterns": [ + { + "match_method": "sub", + "case_sensitive": "yes", + "is_hexbin": "no", + "pattern": "multi" + }, + { + "match_method": "sub", + "case_sensitive": "yes", + "is_hexbin": "no", + "pattern": "pat" + } + ] + }, + { + "expr_id": 110, + "pattern_num": 1, + "patterns": [ + { + "match_method": "sub", + "case_sensitive": "yes", + "is_hexbin": "no", + "pattern": "#中国" + } + ] + }, + { + "expr_id": 111, + "pattern_num": 1, + "patterns": [ + { + "match_method": "sub", + "case_sensitive": "yes", + "is_hexbin": "no", + "pattern": "يىلىدىكى" + } + ] + } + ] +} diff --git a/test/maat_api_gtest.cpp b/test/maat_api_gtest.cpp index 7c8cd80..5754251 100644 --- a/test/maat_api_gtest.cpp +++ b/test/maat_api_gtest.cpp @@ -1,4 +1,4 @@ -#include "maat/maat.h" +#include "maat.h" #include diff --git a/test/maat_framework_gtest.cpp b/test/maat_framework_gtest.cpp index b84d6b1..19e7113 100644 --- a/test/maat_framework_gtest.cpp +++ b/test/maat_framework_gtest.cpp @@ -1,11 +1,12 @@ #include -#include "maat/maat.h" +#include "maat.h" #include "maat_rule.h" #include "maat_utils.h" #include "maat_command.h" #include "IPMatcher.h" #include "json2iris.h" +#include "log/log.h" #include "maat_config_monitor.h" #define MODULE_FRAMEWORK_GTEST module_name_str("maat.framework_gtest") @@ -15,6 +16,8 @@ const char *json_path="./maat_json.json"; const char *json_filename = "maat_json.json"; struct maat *g_maat_instance = NULL; +extern int system_cmd_rmdir(const char *dir); + class MaatFlagScan : public testing::Test { protected: @@ -234,7 +237,9 @@ TEST_F(MaatStringScan, ExprPlus) { EXPECT_EQ(results[0], 190); maat_state_free(&state); } + //TODO: +#if 0 TEST_F(MaatStringScan, ShouldNotHitExprPlus) { int results[5] = {0}; size_t n_hit_result = 0; @@ -259,6 +264,7 @@ TEST_F(MaatStringScan, ShouldNotHitExprPlus) { int ret = maat_state_set_scan_district(g_maat_instance, &state, region_name, strlen(region_name)); ASSERT_EQ(ret, 0); + ret = maat_scan_string(g_maat_instance, table_id, 0, (char *)udp_payload_not_hit, sizeof(udp_payload_not_hit), results, 5, &n_hit_result, &state); EXPECT_EQ(ret, MAAT_SCAN_HALF_HIT); @@ -348,41 +354,7 @@ TEST_F(MaatStringScan, ExprPlusWithOffset) maat_state_free(&state); } - -#if 0 -TEST_F(MaatStringScan, hit_two_expr) { - int table_id = MaatStringScan::table_id; - char data[128] = "should hit aaa bbb"; - int results[5] = {0}; - size_t n_hit_result = 0; - struct maat_state *state = NULL; - int ret = maat_scan_string(g_maat_instance, table_id, 0, data, strlen(data), results, - sizeof(results), &n_hit_result, &state); - EXPECT_EQ(ret, MAAT_SCAN_HIT); - EXPECT_EQ(n_hit_result, 2); - EXPECT_EQ(results[0], 139); - EXPECT_EQ(results[1], 138); - - maat_state_free(&state); -} - -TEST_F(MaatStringScan, hit_three_expr) { - int table_id = MaatStringScan::table_id; - - char data[128] = "should hit aaa bbb C#中国"; - int results[5] = {0}; - size_t n_hit_result = 0; - struct maat_state *state = NULL; - int ret = maat_scan_string(g_maat_instance, table_id, 0, data, strlen(data), results, - sizeof(results), &n_hit_result, &state); - EXPECT_EQ(ret, MAAT_SCAN_HIT); - EXPECT_EQ(n_hit_result, 3); - EXPECT_EQ(results[0], 139); - EXPECT_EQ(results[1], 138); - EXPECT_EQ(results[2], 129); - - maat_state_free(&state); -} +#endif class MaatIPScan : public testing::Test { @@ -399,9 +371,9 @@ protected: }; int MaatIPScan::table_id; -TEST_F(MaatIPScan, hit_ip) { +TEST_F(MaatIPScan, IPv4) { int table_id = MaatIPScan::table_id; - char ip_str[32] = "192.168.58.20"; + char ip_str[32] = "10.0.7.100"; uint32_t sip; int ret = inet_pton(AF_INET, ip_str, &sip); EXPECT_EQ(ret, 1); @@ -409,166 +381,36 @@ TEST_F(MaatIPScan, hit_ip) { int results[3] = {-1}; size_t n_hit_result = 0; struct maat_state *state = NULL; - ret = maat_scan_ipv4(g_maat_instance, table_id, 0, sip, results, sizeof(results), + ret = maat_scan_ipv4(g_maat_instance, table_id, 0, sip, results, 3, &n_hit_result, &state); EXPECT_EQ(ret, MAAT_SCAN_HIT); - EXPECT_EQ(n_hit_result, 1); - EXPECT_EQ(results[0], 7); - maat_state_free(&state); - - state = NULL; - ret = maat_scan_ipv4(g_maat_instance, table_id, 0, sip, results, sizeof(results), - &n_hit_result, &state); - EXPECT_EQ(ret, MAAT_SCAN_OK); - EXPECT_EQ(n_hit_result, 0); - - maat_state_free(&state); -} - -TEST_F(MaatStringScan, hit_ip_and_port_range) { - int table_id = table_manager_get_table_id(g_maat_instance->tbl_mgr, "IP_PLUS_CONFIG"); - char ip_str[32] = "192.168.50.24"; - uint32_t sip; - int ret = inet_pton(AF_INET, ip_str, &sip); - EXPECT_EQ(ret, 1); - - int results[3] = {-1}; - size_t n_hit_result = 0; - struct maat_state *state = NULL; - ret = maat_scan_ipv4(g_maat_instance, table_id, 0, sip, results, sizeof(results), &n_hit_result, &state); - EXPECT_EQ(ret, MAAT_SCAN_HIT); - EXPECT_EQ(n_hit_result, 1); - EXPECT_EQ(results[0], 4); - maat_state_free(&state); - - memset(results, 0, sizeof(results)); - n_hit_result = 0; - state = NULL; - ret = maat_scan_ipv4(g_maat_instance, table_id, 0, sip, results, sizeof(results), &n_hit_result, &state); - EXPECT_EQ(ret, MAAT_SCAN_HIT); - EXPECT_EQ(n_hit_result, 1); - EXPECT_EQ(results[0], 4); - maat_state_free(&state); - - memset(results, 0, sizeof(results)); - n_hit_result = 0; - state = NULL; - ret = maat_scan_ipv4(g_maat_instance, table_id, 0, sip, results, sizeof(results), &n_hit_result, &state); - EXPECT_EQ(ret, MAAT_SCAN_OK); - EXPECT_EQ(n_hit_result, 0); -} - -TEST(maat_scan_ipv4, hit_ip_range_and_port_range) { - int table_id = table_manager_get_table_id(g_maat_instance->tbl_mgr, "IP_PLUS_CONFIG"); - char ip_str1[32] = "10.0.1.20"; - char ip_str2[32] = "10.0.1.25"; - char ip_str3[32] = "10.0.1.26"; - uint32_t sip; - int ret = inet_pton(AF_INET, ip_str1, &sip); - EXPECT_EQ(ret, 1); - - int results[3] = {-1}; - size_t n_hit_result = 0; - struct maat_state *state = NULL; - ret = maat_scan_ipv4(g_maat_instance, table_id, 0, sip, results, sizeof(results), &n_hit_result, &state); - EXPECT_EQ(ret, MAAT_SCAN_HIT); - EXPECT_EQ(n_hit_result, 1); - EXPECT_EQ(results[0], 8); - maat_state_free(&state); - - ret = inet_pton(AF_INET, ip_str2, &sip); - EXPECT_EQ(ret, 1); - state = NULL; - ret = maat_scan_ipv4(g_maat_instance, table_id, 0, sip, results, sizeof(results), &n_hit_result, &state); - EXPECT_EQ(ret, MAAT_SCAN_HIT); - EXPECT_EQ(n_hit_result, 1); - EXPECT_EQ(results[0], 8); - maat_state_free(&state); - - ret = inet_pton(AF_INET, ip_str3, &sip); - EXPECT_EQ(ret, 1); - state = NULL; - ret = maat_scan_ipv4(g_maat_instance, table_id, 0, sip, results, sizeof(results), &n_hit_result, &state); - EXPECT_EQ(ret, MAAT_SCAN_OK); - EXPECT_EQ(n_hit_result, 0); - maat_state_free(&state); -} - -TEST(maat_scan_ipv4, hit_ip_cidr_and_port_range) { - int table_id = table_manager_get_table_id(g_maat_instance->tbl_mgr, "IP_PLUS_CONFIG"); - char ip_str1[32] = "192.168.0.1"; - char ip_str2[32] = "192.168.0.0"; - uint32_t sip; - int ret = inet_pton(AF_INET, ip_str1, &sip); - EXPECT_EQ(ret, 1); - - int results[3] = {-1}; - size_t n_hit_result = 0; - struct maat_state *state = NULL; - ret = maat_scan_ipv4(g_maat_instance, table_id, 0, sip, results, sizeof(results), &n_hit_result, &state); - EXPECT_EQ(ret, MAAT_SCAN_HIT); - EXPECT_EQ(n_hit_result, 1); - EXPECT_EQ(results[0], 50); - maat_state_free(&state); - - ret = inet_pton(AF_INET, ip_str2, &sip); - EXPECT_EQ(ret, 1); - state = NULL; - ret = maat_scan_ipv4(g_maat_instance, table_id, 0, sip, results, sizeof(results), &n_hit_result, &state); - EXPECT_EQ(ret, MAAT_SCAN_OK); - EXPECT_EQ(n_hit_result, 0); - maat_state_free(&state); -} - -TEST(maat_scan_ipv4, hit_ip_cidr_and_port_mask) { - int table_id = table_manager_get_table_id(g_maat_instance->tbl_mgr, "IP_PLUS_CONFIG"); - char ip_str[32] = "192.168.40.10"; - uint32_t sip; - int ret = inet_pton(AF_INET, ip_str, &sip); - EXPECT_EQ(ret, 1); - - int results[3] = {-1}; - size_t n_hit_result = 0; - struct maat_state *state = NULL; - ret = maat_scan_ipv4(g_maat_instance, table_id, 0, sip, results, sizeof(results), &n_hit_result, &state); - EXPECT_EQ(ret, MAAT_SCAN_HIT); EXPECT_EQ(n_hit_result, 2); - EXPECT_EQ(results[0], 63); - EXPECT_EQ(results[1], 67); - maat_state_free(&state); - - state = NULL; - ret = maat_scan_ipv4(g_maat_instance, table_id, 0, sip, results, sizeof(results), &n_hit_result, &state); - EXPECT_EQ(ret, MAAT_SCAN_OK); - EXPECT_EQ(n_hit_result, 0); + EXPECT_EQ(results[0], 169); + EXPECT_EQ(results[1], 154); maat_state_free(&state); } -TEST(maat_scan_ipv6, hit_ip_range_and_port_mask) { - int table_id = table_manager_get_table_id(g_maat_instance->tbl_mgr, "IP_PLUS_CONFIG"); +TEST_F(MaatIPScan, IPv6) { + int table_id = MaatIPScan::table_id; char ip_str[32] = "1001:da8:205:1::101"; - uint8_t sip6[128]; - int ret = inet_pton(AF_INET6, ip_str, sip6); + uint8_t sip[16]; + int ret = inet_pton(AF_INET6, ip_str, &sip); EXPECT_EQ(ret, 1); int results[3] = {-1}; size_t n_hit_result = 0; struct maat_state *state = NULL; - ret = maat_scan_ipv6(g_maat_instance, table_id, 0, sip6, results, sizeof(results), &n_hit_result, &state); + ret = maat_scan_ipv6(g_maat_instance, table_id, 0, sip, results, 3, + &n_hit_result, &state); EXPECT_EQ(ret, MAAT_SCAN_HIT); EXPECT_EQ(n_hit_result, 1); - EXPECT_EQ(results[0], 47); - maat_state_free(&state); + EXPECT_EQ(results[0], 155); - state = NULL; - ret = maat_scan_ipv6(g_maat_instance, table_id, 0, sip6, results, sizeof(results), &n_hit_result, &state); - EXPECT_EQ(ret, MAAT_SCAN_OK); - EXPECT_EQ(n_hit_result, 0); maat_state_free(&state); } -TEST(maat_scan_string, dynamic_config) { - int table_id = table_manager_get_table_id(g_maat_instance->tbl_mgr, "HTTP_URL"); +TEST_F(MaatStringScan, dynamic_config) { + int table_id = maat_table_get_id(g_maat_instance, "HTTP_URL"); char data[128] = "hello world"; int results[5] = {0}; @@ -597,8 +439,8 @@ TEST(maat_scan_string, dynamic_config) { maat_state_free(&state); } -TEST(maat_scan_ip, dynamic_config) { - int table_id = table_manager_get_table_id(g_maat_instance->tbl_mgr, "IP_PLUS_CONFIG"); +TEST_F(MaatIPScan, dynamic_config) { + int table_id = maat_table_get_id(g_maat_instance, "IP_PLUS_CONFIG"); char ip_str[32] = "10.0.6.201"; uint32_t sip; @@ -630,7 +472,6 @@ TEST(maat_scan_ip, dynamic_config) { EXPECT_EQ(results[0], 9998); maat_state_free(&state); } -#endif int count_line_num_cb(const char *table_name, const char *line, void *u_para) { diff --git a/test/maat_input_mode_gtest.cpp b/test/maat_input_mode_gtest.cpp index f178c14..b8a31ae 100644 --- a/test/maat_input_mode_gtest.cpp +++ b/test/maat_input_mode_gtest.cpp @@ -1,4 +1,4 @@ -#include "maat/maat.h" +#include "maat.h" #include "maat_utils.h" #include "maat_rule.h" #include "json2iris.h" @@ -36,7 +36,7 @@ TEST(json_mode, maat_scan_string) { struct maat *maat_instance = maat_new(opts, table_info_path); EXPECT_NE(maat_instance, nullptr); - int table_id = table_manager_get_table_id(maat_instance->tbl_mgr, "KEYWORDS_TABLE"); + int table_id = maat_table_get_id(maat_instance, "KEYWORDS_TABLE"); char scan_data[128] = "string1, string2, string3, string4, string5, string6, string7, string8"; int results[5] = {0}; @@ -81,7 +81,7 @@ TEST(iris_mode, maat_scan_string) { struct maat *maat_instance = maat_new(opts, table_info_path); EXPECT_NE(maat_instance, nullptr); - int table_id = table_manager_get_table_id(maat_instance->tbl_mgr, "KEYWORDS_TABLE"); + int table_id = maat_table_get_id(maat_instance, "KEYWORDS_TABLE"); char scan_data[128] = "string1, string2, string3, string4, string5, string6, string7, string8"; int results[5] = {0}; @@ -202,7 +202,7 @@ TEST(redis_mode, maat_scan_string) { maat_options_set_logger(opts, g_logger); struct maat *maat_instance = maat_new(opts, table_info_path); - int table_id = table_manager_get_table_id(maat_instance->tbl_mgr, "KEYWORDS_TABLE"); + int table_id = maat_table_get_id(maat_instance, "KEYWORDS_TABLE"); char scan_data[128] = "string1, string2, string3, string4, string5, string6, string7, string8"; int results[5] = {0}; diff --git a/test/regex_expr.conf b/test/regex_expr.conf new file mode 100644 index 0000000..2fcc7ef --- /dev/null +++ b/test/regex_expr.conf @@ -0,0 +1,52 @@ +{ + "expr_rules": [ + { + "expr_id": 301, + "pattern_num": 1, + "patterns": [ + { + "match_method": "sub", + "case_sensitive": "yes", + "is_hexbin": "no", + "pattern": "[W|w]orld dream" + } + ] + }, + { + "expr_id": 302, + "pattern_num": 2, + "patterns": [ + { + "match_method": "sub", + "case_sensitive": "yes", + "is_hexbin": "no", + "pattern": "[0-9]today" + }, + { + "match_method": "sub", + "case_sensitive": "yes", + "is_hexbin": "no", + "pattern": "[0-9]Lunch" + } + ] + }, + { + "expr_id": 303, + "pattern_num": 2, + "patterns": [ + { + "match_method": "sub", + "case_sensitive": "yes", + "is_hexbin": "no", + "pattern": "Cookie:\\s" + }, + { + "match_method": "sub", + "case_sensitive": "yes", + "is_hexbin": "no", + "pattern": "head" + } + ] + }, + ] +} \ No newline at end of file diff --git a/test/table_info.conf b/test/table_info.conf index 768eb06..ca2905b 100644 --- a/test/table_info.conf +++ b/test/table_info.conf @@ -47,6 +47,7 @@ "valid_column":7, "custom": { "scan_mode":"block", + "pattern_type":"literal", "item_id":1, "group_id":2, "keywords":3, @@ -62,6 +63,7 @@ "valid_column":7, "custom": { "scan_mode":"block", + "pattern_type":"literal", "item_id":1, "group_id":2, "keywords":3, @@ -77,6 +79,7 @@ "valid_column":7, "custom": { "scan_mode":"block", + "pattern_type":"regex", "item_id":1, "group_id":2, "keywords":3, @@ -89,7 +92,7 @@ "table_id":5, "table_name":"IP_CONFIG", "table_type":"ip_plus", - "valid_column":7, + "valid_column":18, "custom": { "item_id":1, "group_id":2, @@ -128,6 +131,7 @@ "valid_column":8, "custom": { "scan_mode":"block", + "pattern_type":"literal", "item_id":1, "group_id":2, "district":3, @@ -144,6 +148,7 @@ "valid_column":7, "custom": { "scan_mode":"block", + "pattern_type":"literal", "item_id":1, "group_id":2, "keywords":3, @@ -218,6 +223,7 @@ "valid_column":8, "custom": { "scan_mode":"block", + "pattern_type":"literal", "item_id":1, "group_id":2, "district":3, @@ -234,6 +240,7 @@ "valid_column":7, "custom": { "scan_mode":"block", + "pattern_type":"literal", "item_id":1, "group_id":2, "keywords":3, @@ -250,6 +257,7 @@ "valid_column":7, "custom": { "scan_mode":"block", + "pattern_type":"literal", "item_id":1, "group_id":2, "keywords":3, @@ -262,7 +270,7 @@ "table_id":20, "table_name":"IP_PLUS_CONFIG", "table_type":"ip_plus", - "valid_column":7, + "valid_column":18, "custom": { "item_id":1, "group_id":2, @@ -326,6 +334,7 @@ "valid_column":7, "custom": { "scan_mode":"block", + "pattern_type":"literal", "item_id":1, "group_id":2, "keywords":3, @@ -353,6 +362,7 @@ "valid_column":7, "custom": { "scan_mode":"block", + "pattern_type":"literal", "item_id":1, "group_id":2, "keywords":3, @@ -410,6 +420,7 @@ "valid_column":7, "custom": { "scan_mode":"block", + "pattern_type":"literal", "item_id":1, "group_id":2, "keywords":3, @@ -451,5 +462,19 @@ "flag":3, "flag_mask":4 } + }, + { + "table_id":43, + "table_name":"PORT_PROTO_CONFIG", + "table_type":"port_proto", + "valid_column":7, + "custom": { + "item_id":1, + "group_id":2, + "port_format":3, + "port1":4, + "port2":5, + "proto":6 + } } ] \ No newline at end of file diff --git a/vendor/CMakeLists.txt b/vendor/CMakeLists.txt index f3eb0c4..4b6a964 100644 --- a/vendor/CMakeLists.txt +++ b/vendor/CMakeLists.txt @@ -5,6 +5,9 @@ include(ExternalProject) set(VENDOR_ROOT ${CMAKE_BINARY_DIR}/vendor) set(VENDOR_BUILD ${CMAKE_BINARY_DIR}/vendor/vbuild) +set(CMAKE_C_FLAGS "-std=gnu99 -fPIC -Wall") +set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fPIC -Wall") + # GoogleTest ExternalProject_Add(GoogleTest PREFIX GoogleTest URL ${CMAKE_CURRENT_SOURCE_DIR}/googletest-release-1.8.0.tar.gz @@ -40,7 +43,7 @@ ExternalProject_Add(ragel PREFIX ragel ExternalProject_Add(hyperscan PREFIX hyperscan URL ${CMAKE_CURRENT_SOURCE_DIR}/hyperscan-5.4.0.tar.gz DEPENDS ragel - CMAKE_ARGS -DCMAKE_INSTALL_PREFIX=${VENDOR_BUILD} -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} -DCMAKE_C_FLAGS="-fPIC") + CMAKE_ARGS -DCMAKE_INSTALL_PREFIX=${VENDOR_BUILD} -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} -DCMAKE_C_FLAGS="-fPIC" -DCMAKE_CXX_FLAGS="-fPIC") ExternalProject_Get_Property(hyperscan INSTALL_DIR) file(MAKE_DIRECTORY ${VENDOR_BUILD}/include) @@ -81,7 +84,7 @@ file(MAKE_DIRECTORY ${VENDOR_BUILD}/include) add_library(hiredis-static STATIC IMPORTED GLOBAL) add_dependencies(hiredis-static hiredis) -set_property(TARGET hiredis-static PROPERTY IMPORTED_LOCATION ${VENDOR_BUILD}/lib64/libhiredisd.a) +set_property(TARGET hiredis-static PROPERTY IMPORTED_LOCATION ${VENDOR_BUILD}/lib64/libhiredis.a) set_property(TARGET hiredis-static PROPERTY INTERFACE_INCLUDE_DIRECTORIES ${VENDOR_BUILD}/include) #igraph-0.7.1 @@ -103,7 +106,7 @@ set_property(TARGET igraph-static PROPERTY INTERFACE_INCLUDE_DIRECTORIES ${INSTA #flag-matcher ExternalProject_Add(flagmatcher PREFIX flagmatcher - URL ${CMAKE_CURRENT_SOURCE_DIR}/flag-matcher.tar.gz + URL ${CMAKE_CURRENT_SOURCE_DIR}/flag-matcher-1.0.1.tar.gz CMAKE_ARGS -DCMAKE_INSTALL_PREFIX=${VENDOR_BUILD} -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} -DCMAKE_C_FLAGS="-fPIC") ExternalProject_Get_Property(flagmatcher INSTALL_DIR) @@ -116,7 +119,7 @@ set_property(TARGET flagmatcher-static PROPERTY INTERFACE_INCLUDE_DIRECTORIES $ #interval-matcher ExternalProject_Add(intervalmatcher PREFIX intervalmatcher - URL ${CMAKE_CURRENT_SOURCE_DIR}/interval-matcher.tar.gz + URL ${CMAKE_CURRENT_SOURCE_DIR}/interval-matcher-1.0.1.tar.gz CMAKE_ARGS -DCMAKE_INSTALL_PREFIX=${VENDOR_BUILD} -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} -DCMAKE_C_FLAGS="-fPIC") ExternalProject_Get_Property(intervalmatcher INSTALL_DIR) @@ -125,4 +128,4 @@ file(MAKE_DIRECTORY ${VENDOR_BUILD}/include) add_library(intervalmatcher-static STATIC IMPORTED GLOBAL) add_dependencies(intervalmatcher-static flagmatcher) set_property(TARGET intervalmatcher-static PROPERTY IMPORTED_LOCATION ${VENDOR_BUILD}/lib/libinterval_matcher.a) -set_property(TARGET intervalmatcher-static PROPERTY INTERFACE_INCLUDE_DIRECTORIES ${VENDOR_BUILD}/include) \ No newline at end of file +set_property(TARGET intervalmatcher-static PROPERTY INTERFACE_INCLUDE_DIRECTORIES ${VENDOR_BUILD}/include) diff --git a/vendor/flag-matcher.tar.gz b/vendor/flag-matcher-1.0.1.tar.gz similarity index 72% rename from vendor/flag-matcher.tar.gz rename to vendor/flag-matcher-1.0.1.tar.gz index b01742b..1cfab0e 100644 Binary files a/vendor/flag-matcher.tar.gz and b/vendor/flag-matcher-1.0.1.tar.gz differ diff --git a/vendor/interval-matcher.tar.gz b/vendor/interval-matcher-1.0.1.tar.gz similarity index 72% rename from vendor/interval-matcher.tar.gz rename to vendor/interval-matcher-1.0.1.tar.gz index 4889b35..4f8a88f 100644 Binary files a/vendor/interval-matcher.tar.gz and b/vendor/interval-matcher-1.0.1.tar.gz differ