diff --git a/scanner/adapter_hs.cpp b/scanner/adapter_hs.cpp index 2222f40..fb3dad9 100644 --- a/scanner/adapter_hs.cpp +++ b/scanner/adapter_hs.cpp @@ -414,7 +414,7 @@ int adapter_hs_scan(struct adapter_hs *hs_instance, int thread_id, size_t n_result, size_t *n_hit_result) { if (NULL == hs_instance || NULL == data || (0 == data_len) || - NULL == results || NULL == n_results) { + NULL == results || 0 == n_result || NULL == n_hit_result) { return -1; } @@ -515,9 +515,8 @@ struct adapter_hs_stream *adapter_hs_stream_open(struct adapter_hs *hs_instance, return hs_stream; } -int adapter_hs_scan_stream(struct adapter_hs_stream *hs_stream, - const char *data, size_t data_len, - int results[], size_t *n_results) +int adapter_hs_scan_stream(struct adapter_hs_stream *hs_stream, const char *data, size_t data_len, + struct hs_scan_result *results, size_t n_result, size_t *n_hit_result) { hs_error_t err; @@ -549,19 +548,30 @@ int adapter_hs_scan_stream(struct adapter_hs_stream *hs_stream, items[i] = *(unsigned long long *)utarray_eltptr(hs_stream->pattern_id_set, i); } - size_t matched_index = 0; + int ret = 0; + int matched_index = 0; struct bool_expr_match *bool_matcher_results = NULL; + bool_matcher_results = ALLOC(struct bool_expr_match, hs_stream->n_expr); - size_t bool_matcher_ret = bool_matcher_match(hs_stream->hs_rt->bm, items, pattern_set_size, + int bool_matcher_ret = bool_matcher_match(hs_stream->hs_rt->bm, items, pattern_set_size, bool_matcher_results, hs_stream->n_expr); - for (matched_index = 0; matched_index < bool_matcher_ret; matched_index++) { - results[matched_index] = bool_matcher_results[matched_index].expr_id; + if (bool_matcher_ret < 0) { + ret = -1; + goto next; } - *n_results = bool_matcher_ret; + if (bool_matcher_ret > n_result) { + bool_matcher_ret = n_result; + } + + for (matched_index = 0; matched_index < bool_matcher_ret; matched_index++) { + results[matched_index].item_id = bool_matcher_results[matched_index].expr_id; + results[matched_index].user_tag = bool_matcher_results[matched_index].user_tag; + } + *n_hit_result = bool_matcher_ret; +next: FREE(bool_matcher_results); - - return 0; + return ret; } void adapter_hs_stream_close(struct adapter_hs_stream *hs_stream) diff --git a/scanner/adapter_hs.h b/scanner/adapter_hs.h index 5cc4eec..73335b5 100644 --- a/scanner/adapter_hs.h +++ b/scanner/adapter_hs.h @@ -106,7 +106,7 @@ struct adapter_hs_stream; struct adapter_hs_stream *adapter_hs_stream_open(struct adapter_hs *hs_instance, int thread_id); int adapter_hs_scan_stream(struct adapter_hs_stream *stream, const char *data, size_t data_len, - int results[], size_t *n_results); + struct hs_scan_result *results, size_t n_result, size_t *n_hit_result); void adapter_hs_stream_close(struct adapter_hs_stream *stream); diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 85ac26a..4d8babe 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -10,7 +10,7 @@ message(STATUS "Maat Frame, Version: ${MAAT_FRAME_VERSION}") add_definitions(-fPIC) set(MAAT_SRC json2iris.cpp maat_api.cpp rcu_hash.cpp maat_garbage_collection.cpp maat_config_monitor.cpp maat_rule.cpp maat_kv.cpp maat_ex_data.cpp maat_utils.cpp maat_command.cpp maat_redis_monitor.cpp - maat_table.cpp maat_compile.cpp maat_group.cpp maat_ip.cpp maat_intval.cpp maat_flag.cpp + maat_table.cpp maat_compile.cpp maat_group.cpp maat_ip.cpp maat_flag.cpp maat_interval.cpp maat_expr.cpp maat_plugin.cpp maat_ip_plugin.cpp maat_virtual.cpp) set(LIB_SOURCE_FILES @@ -27,7 +27,7 @@ set_target_properties(maat_frame_static PROPERTIES LINKER_LANGUAGE CXX) set_target_properties(maat_frame_static PROPERTIES OUTPUT_NAME maatframe) set_target_properties(maat_frame_static PROPERTIES CLEAN_DIRECT_OUTPUT 1) -target_link_libraries(maat_frame_static adapter-static hiredis-static igraph-static pthread crypto z) +target_link_libraries(maat_frame_static adapter-static hiredis-static igraph-static flagmatcher-static intervalmatcher-static pthread crypto z) # Shared Library Output #add_library(maat_frame_shared SHARED ${MAAT_SRC} ${LIB_SOURCE_FILES}) diff --git a/src/inc_internal/maat_expr.h b/src/inc_internal/maat_expr.h index b2d0950..fa230f0 100644 --- a/src/inc_internal/maat_expr.h +++ b/src/inc_internal/maat_expr.h @@ -42,13 +42,14 @@ int expr_runtime_commit(void *expr_runtime); * @retval the num of hit group_id */ int expr_runtime_scan_string(struct expr_runtime *expr_rt, int thread_id, - const char *data, size_t data_len, + const char *data, size_t data_len, int *group_ids, size_t group_ids_size, - int vtable_id, struct maat_state *state); + int vtable_ids, struct maat_state *state); void expr_runtime_stream_open(struct expr_runtime *expr_rt, int thread_id); -int expr_runtime_scan_stream(struct expr_runtime *expr_rt, const char *data, - size_t data_len, int result[], size_t *n_result); +int expr_runtime_scan_stream(struct expr_runtime *expr_rt, const char *data, + size_t data_len, int *group_ids, size_t group_ids_size, + int vtable_id, struct maat_state *state); void expr_runtime_stream_close(struct expr_runtime *expr_rt); void expr_runtime_scan_hit_inc(struct expr_runtime *expr_rt, int thread_id); diff --git a/src/inc_internal/maat_flag.h b/src/inc_internal/maat_flag.h index f535730..2818858 100644 --- a/src/inc_internal/maat_flag.h +++ b/src/inc_internal/maat_flag.h @@ -16,6 +16,8 @@ extern "C" { #endif +#include + #include "log/log.h" #include "cJSON/cJSON.h" @@ -42,9 +44,8 @@ int flag_runtime_commit(void *flag_runtime); * @retval the num of hit group_id */ int flag_runtime_scan_flag(struct flag_runtime *flag_rt, int thread_id, - const char *data, size_t data_len, - int *group_ids, size_t group_ids_size, - int vtable_id, struct maat_state *state); + uint64_t flag, int *group_ids, size_t group_ids_size, + int vtable_id, struct maat_state *state); void flag_runtime_scan_hit_inc(struct flag_runtime *flag_rt, int thread_id); long long flag_runtime_scan_hit_sum(struct flag_runtime *flag_rt, int n_thread); diff --git a/src/inc_internal/maat_interval.h b/src/inc_internal/maat_interval.h new file mode 100644 index 0000000..7b13399 --- /dev/null +++ b/src/inc_internal/maat_interval.h @@ -0,0 +1,57 @@ +/* +********************************************************************************************** +* File: maat_interval.h +* Description: +* Authors: Liu WenTan +* Date: 2022-10-31 +* Copyright: (c) 2018-2022 Geedge Networks, Inc. All rights reserved. +*********************************************************************************************** +*/ + +#ifndef _MAAT_INTERVAL_H_ +#define _MAAT_INTERVAL_H_ + +#ifdef __cpluscplus +extern "C" +{ +#endif + +#include + +#include "log/log.h" +#include "cJSON/cJSON.h" + +struct interval_runtime; + +void *interval_schema_new(cJSON *json, struct table_manager *tbl_mgr, + const char *table_name, struct log_handle *logger); +void interval_schema_free(void *interval_schema); + +/* interval runtime API */ +void *interval_runtime_new(void *interval_schema, int max_thread_num, + struct maat_garbage_bin *garbage_bin, + struct log_handle *logger); +void interval_runtime_free(void *interval_runtime); + +int interval_runtime_update(void *interval_runtime, void *interval_schema, + const char *line, int valid_column); +int interval_runtime_commit(void *interval_runtime); + +/* interval runtime scan API */ +/** + * @brief scan intval to get hit group_ids + * + * @retval the num of hit group_id +*/ +int interval_runtime_scan_intval(struct interval_runtime *interval_rt, int thread_id, + uint64_t integer, int *group_ids, size_t group_ids_size, + int vtable_id, struct maat_state *state); + +void interval_runtime_scan_hit_inc(struct interval_runtime *interval_rt, int thread_id); +long long interval_runtime_scan_hit_sum(struct interval_runtime *interval_rt, int n_thread); + +#ifdef __cpluscplus +} +#endif + +#endif diff --git a/src/inc_internal/maat_intval.h b/src/inc_internal/maat_intval.h deleted file mode 100644 index adab665..0000000 --- a/src/inc_internal/maat_intval.h +++ /dev/null @@ -1,56 +0,0 @@ -/* -********************************************************************************************** -* File: maat_intval.h -* Description: -* Authors: Liu WenTan -* Date: 2022-10-31 -* Copyright: (c) 2018-2022 Geedge Networks, Inc. All rights reserved. -*********************************************************************************************** -*/ - -#ifndef _MAAT_INTVAL_H_ -#define _MAAT_INTVAL_H_ - -#ifdef __cpluscplus -extern "C" -{ -#endif - -#include "log/log.h" -#include "cJSON/cJSON.h" - -struct intval_runtime; - -void *intval_schema_new(cJSON *json, struct table_manager *tbl_mgr, - const char *table_name, struct log_handle *logger); -void _schema_free(void *intval_schema); - -/* intval runtime API */ -void *intval_runtime_new(void *intval_schema, int max_thread_num, - struct maat_garbage_bin *garbage_bin, - struct log_handle *logger); -void intval_runtime_free(void *intval_runtime); - -int intval_runtime_update(void *intval_runtime, void *intval_schema, - const char *line, int valid_column); -int intval_runtime_commit(void *intval_runtime); - -/* intval runtime scan API */ -/** - * @brief scan intval to get hit group_ids - * - * @retval the num of hit group_id -*/ -int intval_runtime_scan_intval(struct intval_runtime *intval_rt, int thread_id, - const char *data, size_t data_len, - int *group_ids, size_t group_ids_size, - int vtable_id, struct maat_state *state); - -void intval_runtime_scan_hit_inc(struct intval_runtime *intval_rt, int thread_id); -long long intval_runtime_scan_hit_sum(struct intval_runtime *intval_rt, int n_thread); - -#ifdef __cpluscplus -} -#endif - -#endif diff --git a/src/inc_internal/maat_rule.h b/src/inc_internal/maat_rule.h index 74a94dd..55107a9 100644 --- a/src/inc_internal/maat_rule.h +++ b/src/inc_internal/maat_rule.h @@ -31,6 +31,7 @@ extern "C" #include "IPMatcher.h" #include "maat_kv.h" #include "maat_table.h" +#include "maat_virtual.h" #define MAX_TABLE_NUM 256 @@ -38,6 +39,11 @@ extern "C" #define MAAT_UPDATE_TYPE_FULL 1 #define MAAT_UPDATE_TYPE_INC 2 +#define DISTRICT_ANY -1 +#define DISTRICT_UNKNOWN -2 + +#define MAX_DISTRICT_STR 128 + enum last_scan_flag { LAST_SCAN_UNSET, LAST_SCAN_SET, @@ -253,6 +259,7 @@ struct maat_state { struct maat_compile_state *compile_state; }; +enum scan_type maat_table_get_scan_type(enum table_type table_type); size_t parse_accept_tag(const char *value, struct rule_tag **result, struct log_handle *logger); diff --git a/src/inc_internal/maat_table.h b/src/inc_internal/maat_table.h index b9f7fcd..e5038ed 100644 --- a/src/inc_internal/maat_table.h +++ b/src/inc_internal/maat_table.h @@ -67,6 +67,7 @@ int table_manager_accept_tags_match(struct table_manager *tbl_mgr, const char *t int table_manager_set_scan_district(struct table_manager *tbl_mgr, const char *district_str, size_t district_str_len, int *district_id); int table_manager_get_district_id(struct table_manager *tbl_mgr, const char *district); + void *table_manager_get_schema(struct table_manager *tbl_mgr, int table_id); void *table_manager_get_runtime(struct table_manager *tbl_mgr, int table_id); diff --git a/src/inc_internal/maat_virtual.h b/src/inc_internal/maat_virtual.h index b315ce4..e73c53d 100644 --- a/src/inc_internal/maat_virtual.h +++ b/src/inc_internal/maat_virtual.h @@ -26,6 +26,7 @@ enum scan_type { SCAN_TYPE_IP, SCAN_TYPE_INTERVAL, SCAN_TYPE_STRING, + SCAN_TYPE_FLAG, SCAN_TYPE_MAX }; @@ -34,6 +35,7 @@ void *virtual_schema_new(cJSON *json, struct table_manager *tbl_mgr, void virtual_schema_free(void *virtual_schema); +int virtual_table_get_physical_table_id(void *virtual_schema, enum scan_type type); #ifdef __cpluscplus } diff --git a/src/maat_api.cpp b/src/maat_api.cpp index 31203b6..4d3d088 100644 --- a/src/maat_api.cpp +++ b/src/maat_api.cpp @@ -28,6 +28,7 @@ #include "maat_garbage_collection.h" #include "maat_group.h" #include "maat_expr.h" +#include "maat_flag.h" #include "maat_ip.h" #include "maat_plugin.h" #include "maat_ip_plugin.h" @@ -35,9 +36,6 @@ #define MODULE_MAAT_API module_name_str("maat.api") -#define DISTRICT_ANY -1 -#define DISTRICT_UNKNOWN -2 - enum district_set_flag { DISTRICT_FLAG_UNSET, DISTRICT_FLAG_SET @@ -53,17 +51,13 @@ enum scan_type maat_table_get_scan_type(enum table_type table_type) break; case TABLE_TYPE_EXPR: case TABLE_TYPE_EXPR_PLUS: - case TABLE_TYPE_SIMILARITY: - case TABLE_TYPE_DIGEST: ret = SCAN_TYPE_STRING; break; case TABLE_TYPE_INTERVAL: case TABLE_TYPE_INTERVAL_PLUS: ret = SCAN_TYPE_INTERVAL; break; - case TABLE_TYPE_IP: case TABLE_TYPE_IP_PLUS: - case TABLE_TYPE_COMPOSITION: ret = SCAN_TYPE_IP; break; case TABLE_TYPE_PLUGIN: @@ -84,6 +78,7 @@ enum scan_type maat_table_get_scan_type(enum table_type table_type) default: break; } + return ret; } @@ -299,14 +294,6 @@ struct maat *maat_new(struct maat_options *opts, const char *table_info_path) maat_instance->logger = log_handle_create(log_path, opts->log_level); } - maat_instance->tbl_mgr = table_manager_create(table_info_path, opts->accept_tags, - maat_instance->garbage_bin, maat_instance->logger); - if (NULL == maat_instance->tbl_mgr) { - goto failed; - } - - maat_instance->default_compile_table_id = table_manager_get_defaut_compile_table_id(maat_instance->tbl_mgr); - maat_instance->g2g_table_id = table_manager_get_group2group_table_id(maat_instance->tbl_mgr); maat_instance->input_mode = opts->input_mode; switch (maat_instance->input_mode) { @@ -343,6 +330,15 @@ struct maat *maat_new(struct maat_options *opts, const char *table_info_path) (maat_instance->gc_timeout_ms / 1000); maat_instance->garbage_bin = maat_garbage_bin_new(garbage_gc_timeout_s); + maat_instance->tbl_mgr = table_manager_create(table_info_path, opts->accept_tags, + maat_instance->garbage_bin, maat_instance->logger); + if (NULL == maat_instance->tbl_mgr) { + goto failed; + } + + maat_instance->default_compile_table_id = table_manager_get_defaut_compile_table_id(maat_instance->tbl_mgr); + maat_instance->g2g_table_id = table_manager_get_group2group_table_id(maat_instance->tbl_mgr); + maat_instance->outer_state_cnt = alignment_int64_array_alloc(opts->nr_worker_threads); maat_instance->compile_state_cnt = alignment_int64_array_alloc(opts->nr_worker_threads); maat_instance->thread_call_cnt = alignment_int64_array_alloc(opts->nr_worker_threads); @@ -698,11 +694,97 @@ int hit_group_to_compile(void *compile_runtime, int *compile_ids, size_t compile } } -int maat_scan_flag(struct maat *instance, int table_id, int thread_id, +int maat_scan_flag(struct maat *maat_instance, int table_id, int thread_id, uint64_t flag, int *results, size_t n_result, size_t *n_hit_result, struct maat_state **state) { - return 0; + if ((NULL == maat_instance) || (table_id < 0) || (table_id >= MAX_TABLE_NUM) + || (thread_id < 0) || (NULL == results) || (0 == n_result) || (NULL == state)) { + return MAAT_SCAN_ERR; + } + + struct maat_state *mid = NULL; + mid = grab_state(state, maat_instance, thread_id); + mid->scan_cnt++; + + int physical_table_id = 0; + int vtable_id = 0; + + enum table_type table_type = table_manager_get_table_type(maat_instance->tbl_mgr, table_id); + if (table_type == TABLE_TYPE_VIRTUAL) { + //find physical table id + void *virtual_schema = table_manager_get_schema(maat_instance->tbl_mgr, table_id); + physical_table_id = virtual_table_get_physical_table_id(virtual_schema, SCAN_TYPE_FLAG); + if (physical_table_id < 0) { + return MAAT_SCAN_ERR; + } + } else { + physical_table_id = table_id; + } + + if (NULL == maat_instance->maat_rt) { + log_error(maat_instance->logger, MODULE_MAAT_API, + "maat_scan_string error because of maat_runtime is NULL"); + return MAAT_SCAN_OK; + } + + maat_runtime_ref_inc(maat_instance->maat_rt, thread_id); + alignment_int64_array_add(maat_instance->thread_call_cnt, thread_id, 1); + + int group_ids[MAX_SCANNER_HIT_GROUP_NUM] = {-1}; + void *flag_rt = table_manager_get_runtime(maat_instance->tbl_mgr, table_id); + if (NULL == flag_rt) { + return MAAT_SCAN_ERR; + } + + int group_hit_cnt = flag_runtime_scan_flag((struct flag_runtime *)flag_rt, + thread_id, + flag, group_ids, + MAX_SCANNER_HIT_GROUP_NUM, + vtable_id, mid); + if (group_hit_cnt < 0) { + return MAAT_SCAN_ERR; + } + + int compile_ret = 0; + if (group_hit_cnt > 0 || scan_status_should_compile_NOT(mid)) { + // come here means group_hit_cnt > 0, at least MAAT_SCAN_HALF_HIT, or MAAT_SCAN_HIT + if (group_hit_cnt > 0) { + flag_runtime_scan_hit_inc((struct flag_runtime *)flag_rt, thread_id); + } + + int compile_table_id = -1; + if (mid->compile_table_id == -1) { + compile_table_id = maat_instance->default_compile_table_id; + } else { + compile_table_id = mid->compile_table_id; + } + + void *compile_rt = table_manager_get_runtime(maat_instance->tbl_mgr, compile_table_id); + compile_ret = hit_group_to_compile(compile_rt, results, n_result, n_hit_result, mid); + + assert(mid->is_last_scan < LAST_SCAN_FINISHED); + if (LAST_SCAN_SET == mid->is_last_scan) { + mid->is_last_scan = LAST_SCAN_FINISHED; + } + } + + if (compile_ret > 0) { + alignment_int64_array_add(maat_instance->hit_cnt, thread_id, 1); + } + + if (0 == group_hit_cnt && compile_ret > 0) { + // hit NOT group + alignment_int64_array_add(maat_instance->not_grp_hit_cnt, thread_id, 1); + } + + maat_runtime_ref_dec(maat_instance->maat_rt, thread_id); + + if (0 == compile_ret && group_hit_cnt > 0) { + return MAAT_SCAN_HALF_HIT; + } + + return MAAT_SCAN_HIT; } int maat_scan_integer(struct maat *instance, int table_id, int thread_id, @@ -894,11 +976,14 @@ int maat_scan_string(struct maat *maat_instance, int table_id, int thread_id, mid = grab_state(state, maat_instance, thread_id); mid->scan_cnt++; - int physical_table_id = -1; + int physical_table_id = 0; + int vtable_id = 0; + enum table_type table_type = table_manager_get_table_type(maat_instance->tbl_mgr, table_id); if (table_type == TABLE_TYPE_VIRTUAL) { //find physical table id - physical_table_id = virtual_table_get_physical_table_id(table_id, SCAN_TYPE_STRING); + void *virtual_schema = table_manager_get_schema(maat_instance->tbl_mgr, table_id); + physical_table_id = virtual_table_get_physical_table_id(virtual_schema, SCAN_TYPE_STRING); if (physical_table_id < 0) { return MAAT_SCAN_ERR; } @@ -914,8 +999,7 @@ int maat_scan_string(struct maat *maat_instance, int table_id, int thread_id, maat_runtime_ref_inc(maat_instance->maat_rt, thread_id); - //TODO: is TABLE_TYPE_EXPR_PLUS - enum table_type table_type = table_manager_get_table_type(maat_instance->tbl_mgr, physical_table_id); + table_type = table_manager_get_table_type(maat_instance->tbl_mgr, physical_table_id); if ((table_type == TABLE_TYPE_EXPR_PLUS) && (NULL == mid || DISTRICT_FLAG_UNSET == mid->is_set_district)) { maat_instance->scan_err_cnt++; @@ -930,7 +1014,8 @@ int maat_scan_string(struct maat *maat_instance, int table_id, int thread_id, return MAAT_SCAN_ERR; } - int group_hit_cnt = expr_runtime_scan_string((struct expr_runtime *)expr_rt, thread_id, + int group_hit_cnt = expr_runtime_scan_string((struct expr_runtime *)expr_rt, + thread_id, data, data_len, group_ids, MAX_SCANNER_HIT_GROUP_NUM, vtable_id, mid); @@ -939,17 +1024,12 @@ int maat_scan_string(struct maat *maat_instance, int table_id, int thread_id, } int compile_ret = 0; - int district_id = DISTRICT_ANY; if (group_hit_cnt > 0 || scan_status_should_compile_NOT(mid)) { // come here means group_hit_cnt > 0, at least MAAT_SCAN_HALF_HIT, or MAAT_SCAN_HIT if (group_hit_cnt > 0) { expr_runtime_scan_hit_inc((struct expr_runtime *)expr_rt, thread_id); } - if (group_hit_cnt > 0 && table_type == TABLE_TYPE_EXPR_PLUS) { - district_id = mid->district_id; - } - int compile_table_id = -1; if (mid->compile_table_id == -1) { compile_table_id = maat_instance->default_compile_table_id; diff --git a/src/maat_compile.cpp b/src/maat_compile.cpp index ce45f53..a65b124 100644 --- a/src/maat_compile.cpp +++ b/src/maat_compile.cpp @@ -992,7 +992,7 @@ maat_compile_bool_matcher_new(struct maat_compile *compile_hash, } //TODO:mytest need to delete - #if 0 + #if 1 struct maat_literal_id *p = NULL; for(p = (struct maat_literal_id *)utarray_front(compile->clause_states[i].literal_ids); p!=NULL; p=(struct maat_literal_id *)utarray_next(compile->clause_states[i].literal_ids,p)) { printf("compile_id:%d, clause_id:%llu, literal{%d: %d}\n", @@ -1067,16 +1067,16 @@ size_t maat_compile_bool_matcher_match(struct bool_matcher *bm, int is_last_scan size_t ud_result_cnt = 0; //TODO:mytest need to delete - #if 0 + #if 1 unsigned long long *p; - for (p = (unsigned long long *)utarray_front(compile_state->all_hit_clause_array); p != NULL; p = (unsigned long long *)utarray_next(compile_state->all_hit_clause_array, p)) + printf("utarray_len:%u\n", utarray_len(compile_state->all_hit_clauses)); + for (p = (unsigned long long *)utarray_front(compile_state->all_hit_clauses); p != NULL; p = (unsigned long long *)utarray_next(compile_state->all_hit_clauses, p)) { - printf(" before bool_matcher_match compile_state clause_id:%llu\n", *p); + printf("before bool_matcher_match compile_state clause_id:%llu\n", *p); } #endif - int bool_match_ret = bool_matcher_match(bm, - (unsigned long long *)utarray_eltptr(compile_state->all_hit_clauses, 0), + int bool_match_ret = bool_matcher_match(bm, (unsigned long long *)utarray_eltptr(compile_state->all_hit_clauses, 0), utarray_len(compile_state->all_hit_clauses), expr_match, MAX_SCANNER_HIT_COMPILE_NUM); for (int i = 0; i < bool_match_ret && ud_result_cnt < ud_array_size; i++) { @@ -1197,8 +1197,8 @@ struct maat_compile_state *maat_compile_state_new(int thread_id) compile_state->thread_id = thread_id; //compile_state->hier_ver = hier->version; utarray_new(compile_state->internal_hit_paths, &ut_hit_path_icd); - utarray_new(compile_state->all_hit_clause_array, &ut_clause_id_icd); - utarray_new(compile_state->this_scan_hit_clause_ids, &ut_clause_id_icd); + utarray_new(compile_state->all_hit_clauses, &ut_clause_id_icd); + utarray_new(compile_state->this_scan_hit_clauses, &ut_clause_id_icd); return compile_state; } @@ -1206,8 +1206,8 @@ struct maat_compile_state *maat_compile_state_new(int thread_id) void maat_compile_state_free(struct maat_compile_state *compile_state) { utarray_free(compile_state->internal_hit_paths); - utarray_free(compile_state->all_hit_clause_array); - utarray_free(compile_state->this_scan_hit_clause_ids); + utarray_free(compile_state->all_hit_clauses); + utarray_free(compile_state->this_scan_hit_clauses); free(compile_state); } @@ -1316,8 +1316,8 @@ size_t compile_runtime_get_hit_paths(struct compile_runtime *compile_rt, struct maat_hit_path tmp_path; struct bool_expr_match *expr_match = compile_rt->expr_match_buff + compile_state->thread_id * MAX_SCANNER_HIT_COMPILE_NUM; int bool_match_ret = bool_matcher_match(compile_rt->bm, - (unsigned long long *)utarray_eltptr(compile_state->all_hit_clause_array, 0), - utarray_len(compile_state->all_hit_clause_array), expr_match, + (unsigned long long *)utarray_eltptr(compile_state->all_hit_clauses, 0), + utarray_len(compile_state->all_hit_clauses), expr_match, MAX_SCANNER_HIT_COMPILE_NUM); for (int idx = 0; idx < bool_match_ret; idx++) { compile = (struct maat_compile *)expr_match[idx].user_tag; @@ -1387,7 +1387,6 @@ void maat_compile_state_update_hit_clause(struct maat_compile_state *compile_sta struct maat_literal_id *tmp = NULL; unsigned long long *clause_id = 0; - size_t new_clause_idx = utarray_len(compile_state->this_scan_hit_clauses); HASH_ITER(hh, compile_rt->compile_hash, compile, tmp_compile) { for (size_t i = 0; i < MAX_ITEMS_PER_BOOL_EXPR; i++) { @@ -1396,6 +1395,7 @@ void maat_compile_state_update_hit_clause(struct maat_compile_state *compile_sta continue; } + size_t new_clause_idx = utarray_len(compile_state->this_scan_hit_clauses); tmp = (struct maat_literal_id *)utarray_find(clause_state->literal_ids, &literal_id, compare_literal_id); if (tmp) { @@ -1404,6 +1404,7 @@ void maat_compile_state_update_hit_clause(struct maat_compile_state *compile_sta compare_clause_id)) { continue; } + utarray_push_back(compile_state->this_scan_hit_clauses, &(clause_state->clause_id)); } @@ -1779,6 +1780,11 @@ int maat_compile_state_update(struct maat_item *item_hash, int vtable_id, int top_group_ids[MAX_SCANNER_HIT_GROUP_NUM] = {-1}; int top_group_cnt = group2group_runtime_get_top_groups(g2g_rt, &group_ids[i], 1, top_group_ids); + if (0 == top_group_cnt) { + maat_compile_state_update_hit_clause(state->compile_state, compile_rt, + group_ids[i], vtable_id); + } + for (int j = 0; j < top_group_cnt; j++) { maat_compile_state_update_hit_clause(state->compile_state, compile_rt, top_group_ids[j], vtable_id); diff --git a/src/maat_expr.cpp b/src/maat_expr.cpp index 7adb95e..8ddd425 100644 --- a/src/maat_expr.cpp +++ b/src/maat_expr.cpp @@ -23,8 +23,6 @@ #include "alignment.h" #include "maat_garbage_collection.h" -#define MAX_DISTRICT_STR 128 - #define MODULE_EXPR module_name_str("maat.expr") struct expr_schema { @@ -161,7 +159,7 @@ struct expr_item *expr_item_new(const char *line, struct expr_schema *expr_schem } expr_item->group_id = atoi(line + column_offset); - table_type = table_manager_get_table_type(expr_schema->tbl_mgr, expr_schema->table_id); + table_type = table_manager_get_table_type(expr_schema->ref_tbl_mgr, expr_schema->table_id); if (table_type == TABLE_TYPE_EXPR_PLUS) { ret = get_column_pos(line, expr_schema->district_column, &column_offset, &column_len); if (ret < 0) { @@ -339,7 +337,7 @@ void *expr_schema_new(cJSON *json, struct table_manager *tbl_mgr, read_cnt++; } - expr_schema->tbl_mgr = tbl_mgr; + expr_schema->ref_tbl_mgr = tbl_mgr; if (read_cnt < 8) { goto error; @@ -443,7 +441,7 @@ void expr_runtime_free(void *expr_runtime) } int expr_runtime_update_row(struct expr_runtime *expr_rt, char *key, - size_t key_len, and_expr_t *expr_rule, + size_t key_len, int item_id, and_expr_t *expr_rule, int is_valid, struct log_handle *logger) { void *data = NULL; @@ -454,7 +452,7 @@ int expr_runtime_update_row(struct expr_runtime *expr_rt, char *key, if (NULL == data) { log_error(logger, MODULE_EXPR, "the key of expr rule not exist, can't be deleted, expr_id:%d", - expr_rule->expr_id); + item_id); return -1; } rcu_hash_del(expr_rt->htable, key, key_len); @@ -464,7 +462,7 @@ int expr_runtime_update_row(struct expr_runtime *expr_rt, char *key, if (data != NULL) { log_error(logger, MODULE_EXPR, "the key of expr rule already exist, can't be added, expr_id:%d", - expr_rule->expr_id); + item_id); return -1; } rcu_hash_add(expr_rt->htable, key, key_len, (void *)expr_rule); @@ -492,7 +490,7 @@ enum pattern_type expr_type2pattern_type(enum expr_type expr_type) } #define MAAT_MAX_EXPR_ITEM_NUM 8 -and_expr_t *expr_item_to_expr_rule(struct expr_item *expr_item, +and_expr_t *expr_item_to_expr_rule(struct expr_item *expr_item, void *user_data, struct log_handle *logger) { size_t i = 0; @@ -543,9 +541,8 @@ and_expr_t *expr_item_to_expr_rule(struct expr_item *expr_item, expr_rule->patterns[i].pat_len = strlen(sub_key_array[i]); expr_rule->patterns[i].type = expr_type2pattern_type(expr_item->expr_type); } - int *district_tag = ALLOC(int, 1); - *district_tag = expr_item->district_id; - expr_rule->user_tag = district_tag; + + expr_rule->user_tag = user_data; expr_rule->n_patterns = sub_expr_cnt; return expr_rule; @@ -554,7 +551,8 @@ and_expr_t *expr_item_to_expr_rule(struct expr_item *expr_item, int expr_runtime_update(void *expr_runtime, void *expr_schema, const char *line, int valid_column) { - if (NULL == expr_runtime || NULL == expr_schema) { + if (NULL == expr_runtime || NULL == expr_schema || + NULL == line) { return -1; } @@ -604,7 +602,7 @@ int expr_runtime_update(void *expr_runtime, void *expr_schema, item = maat_item_new(item_id, expr_item->group_id, u_para); HASH_ADD_INT(expr_rt->item_hash, item_id, item); - expr_rule = expr_item_to_expr_rule(expr_item, expr_rt->logger); + expr_rule = expr_item_to_expr_rule(expr_item, u_para, expr_rt->logger); expr_item_free(expr_item); if (NULL == expr_rule) { log_error(expr_rt->logger, MODULE_EXPR, @@ -615,8 +613,8 @@ int expr_runtime_update(void *expr_runtime, void *expr_schema, } char *key = (char *)&item_id; - ret = expr_runtime_update_row(expr_rt, key, sizeof(int), expr_rule, - is_valid, expr_rt->logger); + ret = expr_runtime_update_row(expr_rt, key, sizeof(int), item_id, + expr_rule, is_valid, expr_rt->logger); if (ret < 0) { if (expr_rule != NULL) { expr_rule_free(expr_rule); @@ -653,8 +651,9 @@ int expr_runtime_commit(void *expr_runtime) for (size_t i = 0; i < rule_cnt; i++) { rules[i] = *(and_expr_t *)ex_data_array[i]; + printf("rule_id:%d\n", rules[i].expr_id); } - + printf("\n\n"); struct adapter_hs *new_adapter_hs = NULL; struct adapter_hs *old_adapter_hs = NULL; @@ -688,15 +687,15 @@ int expr_runtime_commit(void *expr_runtime) } int expr_runtime_scan_string(struct expr_runtime *expr_rt, int thread_id, - const char *data, size_t data_len, - int *group_ids, size_t group_ids_size, - int vtable_id, struct maat_state *state) + const char *data, size_t data_len, + int *group_ids, size_t group_ids_size, + int vtable_id, struct maat_state *state) { size_t n_hit_item = 0; - struct hs_scan_results hit_results[MAX_SCANNER_HIT_ITEM_NUM] = {0}; + struct hs_scan_result hit_results[MAX_SCANNER_HIT_ITEM_NUM] = {0}; int ret = adapter_hs_scan(expr_rt->hs, thread_id, data, data_len, - hit_item_ids, MAX_SCANNER_HIT_ITEM_NUM, + hit_results, MAX_SCANNER_HIT_ITEM_NUM, &n_hit_item); if (ret < 0) { return -1; @@ -713,9 +712,20 @@ int expr_runtime_scan_string(struct expr_runtime *expr_rt, int thread_id, n_hit_item = MAX_SCANNER_HIT_ITEM_NUM; } + int hit_item_ids[MAX_SCANNER_HIT_ITEM_NUM] = {-1}; + struct maat_item_inner *item = NULL; + int real_hit_item_cnt = 0; + int district_id = state->district_id; + for (size_t i = 0; i < n_hit_item; i++) { + item = (struct maat_item_inner *)(hit_results[i].user_tag); + if (item->district_id == district_id || district_id == DISTRICT_ANY) { + hit_item_ids[real_hit_item_cnt++] = hit_results[i].item_id; + } + } + size_t group_hit_cnt = 0; ret = maat_compile_state_update(expr_rt->item_hash, vtable_id, hit_item_ids, - n_hit_item, group_ids, group_ids_size, + real_hit_item_cnt, group_ids, group_ids_size, &group_hit_cnt, state); if (ret < 0) { return -1; @@ -735,14 +745,47 @@ void expr_runtime_stream_open(struct expr_runtime *expr_rt, int thread_id) } int expr_runtime_scan_stream(struct expr_runtime *expr_rt, const char *data, - size_t data_len, int result[], size_t *n_result) + size_t data_len, int *group_ids, size_t group_ids_size, + int vtable_id, struct maat_state *state) { if (NULL == expr_rt) { return -1; } - return adapter_hs_scan_stream(expr_rt->hs_stream, data, data_len, - result, n_result); + size_t n_hit_item = 0; + struct hs_scan_result hit_results[MAX_SCANNER_HIT_ITEM_NUM] = {0}; + int ret = adapter_hs_scan_stream(expr_rt->hs_stream, data, data_len, + hit_results, MAX_SCANNER_HIT_ITEM_NUM, + &n_hit_item); + if (ret < 0) { + return -1; + } + + if (0 == n_hit_item) { + return 0; + } + + if (n_hit_item > MAX_SCANNER_HIT_ITEM_NUM) { + log_info(expr_rt->logger, MODULE_EXPR, + "hit expr item count:%d exceed maxium:%d", + n_hit_item, MAX_SCANNER_HIT_ITEM_NUM); + n_hit_item = MAX_SCANNER_HIT_ITEM_NUM; + } + + int hit_item_ids[MAX_SCANNER_HIT_ITEM_NUM] = {-1}; + for (size_t i = 0; i < n_hit_item; i++) { + hit_item_ids[i] = hit_results[i].item_id; + } + + size_t group_hit_cnt = 0; + ret = maat_compile_state_update(expr_rt->item_hash, vtable_id, hit_item_ids, + n_hit_item, group_ids, group_ids_size, + &group_hit_cnt, state); + if (ret < 0) { + return -1; + } + + return group_hit_cnt; } void expr_runtime_stream_close(struct expr_runtime *expr_rt) diff --git a/src/maat_flag.cpp b/src/maat_flag.cpp index 3610d92..402b6dd 100644 --- a/src/maat_flag.cpp +++ b/src/maat_flag.cpp @@ -8,64 +8,449 @@ *********************************************************************************************** */ +#include + #include "maat_flag.h" +#include "flag_matcher.h" +#include "maat_utils.h" +#include "maat_rule.h" +#include "log/log.h" +#include "uthash/uthash.h" +#include "rcu_hash.h" +#include "maat_table.h" +#include "alignment.h" +#include "maat_compile.h" +#include "maat_garbage_collection.h" + +#define MODULE_FLAG module_name_str("maat.flag") struct flag_schema { + int item_id_column; + int group_id_column; + int flag_column; + int flag_mask_column; + int table_id; + struct table_manager *ref_tbl_mgr; +}; +struct flag_item { + int item_id; + int group_id; + uint64_t flag; + uint64_t flag_mask; }; struct flag_runtime { + struct flag_matcher *matcher; + struct rcu_hash_table *htable; + uint32_t rule_num; + uint32_t updating_rule_num; + struct maat_item *item_hash; + void (*item_user_data_free)(void *); + + struct maat_garbage_bin *ref_garbage_bin; + struct log_handle *logger; + + long long *scan_cnt; + long long *hit_cnt; }; void *flag_schema_new(cJSON *json, struct table_manager *tbl_mgr, const char *table_name, struct log_handle *logger) { + int read_cnt = 0; + struct flag_schema *schema = ALLOC(struct flag_schema, 1); + cJSON *custom_item = NULL; + cJSON *item = cJSON_GetObjectItem(json, "table_id"); + if (item != NULL && item->type == cJSON_Number) { + schema->table_id = item->valueint; + read_cnt++; + } + + item = cJSON_GetObjectItem(json, "custom"); + if (item == NULL || item->type != cJSON_Object) { + log_error(logger, MODULE_FLAG, + "table %s has no custom column", table_name); + goto error; + } + + custom_item = cJSON_GetObjectItem(item, "item_id"); + if (custom_item != NULL && custom_item->type == cJSON_Number) { + schema->item_id_column = custom_item->valueint; + read_cnt++; + } + + custom_item = cJSON_GetObjectItem(item, "group_id"); + if (custom_item != NULL && custom_item->type == cJSON_Number) { + schema->group_id_column = custom_item->valueint; + read_cnt++; + } + + custom_item = cJSON_GetObjectItem(item, "flag"); + if (custom_item != NULL && custom_item->type == cJSON_Number) { + schema->flag_column = custom_item->valueint; + read_cnt++; + } + + custom_item = cJSON_GetObjectItem(item, "flag_mask"); + if (custom_item != NULL && custom_item->type == cJSON_Number) { + schema->flag_mask_column = custom_item->valueint; + read_cnt++; + } + + schema->ref_tbl_mgr = tbl_mgr; + + if (read_cnt < 5) { + goto error; + } + + return schema; +error: + FREE(schema); + return NULL; } void flag_schema_free(void *flag_schema) { + FREE(flag_schema); +} +void flag_ex_data_free(void *user_ctx, void *data) +{ + struct flag_item *flag_item = (struct flag_item *)data; + FREE(flag_item); } void *flag_runtime_new(void *flag_schema, int max_thread_num, struct maat_garbage_bin *garbage_bin, struct log_handle *logger) { + if (NULL == flag_schema) { + return NULL; + } + struct flag_runtime *flag_rt = ALLOC(struct flag_runtime, 1); + + flag_rt->htable = rcu_hash_new(flag_ex_data_free); + flag_rt->item_user_data_free = maat_item_inner_free; + flag_rt->ref_garbage_bin = garbage_bin; + flag_rt->logger = logger; + + flag_rt->hit_cnt = alignment_int64_array_alloc(max_thread_num); + flag_rt->scan_cnt = alignment_int64_array_alloc(max_thread_num); + + return flag_rt; } void flag_runtime_free(void *flag_runtime) { + if (NULL == flag_runtime) { + return; + } + struct flag_runtime *flag_rt = (struct flag_runtime *)flag_runtime; + if (flag_rt->htable != NULL) { + rcu_hash_free(flag_rt->htable); + flag_rt->htable = NULL; + } + + struct maat_item *item = NULL, *tmp = NULL; + HASH_ITER(hh, flag_rt->item_hash, item, tmp) { + HASH_DELETE(hh, flag_rt->item_hash, item); + maat_item_free(item, flag_rt->item_user_data_free); + } + + if (flag_rt->hit_cnt != NULL) { + alignment_int64_array_free(flag_rt->hit_cnt); + flag_rt->hit_cnt = NULL; + } + + if (flag_rt->scan_cnt != NULL) { + alignment_int64_array_free(flag_rt->scan_cnt); + flag_rt->scan_cnt = NULL; + } + + FREE(flag_rt); +} + +int flag_runtime_update_row(struct flag_runtime *flag_rt, char *key, size_t key_len, + int item_id, struct flag_rule *rule, int is_valid) +{ + void *data = NULL; + + if (0 == is_valid) { + //delete + data = rcu_hash_find(flag_rt->htable, key, key_len); + if (NULL == data) { + log_error(flag_rt->logger, MODULE_FLAG, + "the key of flag rule not exist, can't be deleted, item_id:%d", + item_id); + return -1; + } + rcu_hash_del(flag_rt->htable, key, key_len); + } else { + //add + data = rcu_hash_find(flag_rt->htable, key, key_len); + if (data != NULL) { + log_error(flag_rt->logger, MODULE_FLAG, + "the key of flag rule already exist, can't be added, item_id:%d", + item_id); + return -1; + } + rcu_hash_add(flag_rt->htable, key, key_len, (void *)rule); + } + + return 0; +} + +struct flag_item *flag_item_new(const char *line, struct flag_schema *flag_schema, + struct log_handle *logger) +{ + size_t column_offset = 0; + size_t column_len = 0; + struct flag_item *flag_item = ALLOC(struct flag_item, 1); + + int ret = get_column_pos(line, flag_schema->item_id_column, &column_offset, &column_len); + if (ret < 0) { + log_error(logger, MODULE_FLAG, + "flag table(table_id:%d) line:%s has no item_id", + flag_schema->table_id, line); + goto error; + } + flag_item->item_id = atoi(line + column_offset); + + ret = get_column_pos(line, flag_schema->group_id_column, &column_offset, &column_len); + if (ret < 0) { + log_error(logger, MODULE_FLAG, + "flag table(table_id:%d) line:%s has no group_id", + flag_schema->table_id, line); + goto error; + } + flag_item->group_id = atoi(line + column_offset); + + ret = get_column_pos(line, flag_schema->flag_column, &column_offset, &column_len); + if (ret < 0) { + log_error(logger, MODULE_FLAG, + "flag table(table_id:%d) line:%s has no flag", + flag_schema->table_id, line); + goto error; + } + + flag_item->flag = strtoull(line + column_offset, NULL, 0); + + ret = get_column_pos(line, flag_schema->flag_mask_column, &column_offset, &column_len); + if (ret < 0) { + log_error(logger, MODULE_FLAG, + "flag table(table_id:%d) line:%s has no flag_mask", + flag_schema->table_id, line); + goto error; + } + flag_item->flag_mask = strtoull(line + column_offset, NULL, 0); + + return flag_item; +error: + FREE(flag_item); + return NULL; +} + +void flag_item_free(struct flag_item *item) +{ + FREE(item); +} + +struct flag_rule *flag_item_to_flag_rule(struct flag_item *item, void *user_data) +{ + struct flag_rule *rule = ALLOC(struct flag_rule, 1); + + rule->rule_id = item->item_id; + rule->flag = item->flag; + rule->mask = item->flag_mask; + rule->user_tag = user_data; + + return rule; +} + +void flag_rule_free(struct flag_rule *rule) +{ + FREE(rule); } int flag_runtime_update(void *flag_runtime, void *flag_schema, const char *line, int valid_column) { + if (NULL == flag_runtime || NULL == flag_schema || + NULL == line) { + return -1; + } + struct maat_item_inner *u_para = NULL; + struct maat_item *item = NULL; + struct flag_rule *flag_rule = NULL; + struct flag_schema *schema = (struct flag_schema *)flag_schema; + struct flag_runtime *flag_rt = (struct flag_runtime *)flag_runtime; + + int item_id = get_column_value(line, schema->item_id_column); + int is_valid = get_column_value(line, valid_column); + if (is_valid < 0) { + return -1; + } else if (0 == is_valid) { + //delete + HASH_FIND_INT(flag_rt->item_hash, &item_id, item); + if (NULL == item) { + return -1; + } + + u_para = (struct maat_item_inner *)item->user_data; + item->user_data = NULL; + + if (NULL == u_para) { + return -1; + } + + HASH_DELETE(hh, flag_rt->item_hash, item); + maat_garbage_bagging(flag_rt->ref_garbage_bin, u_para, maat_item_inner_free); + } else { + //add + HASH_FIND_INT(flag_rt->item_hash, &item_id, item); + if (item) { + log_error(flag_rt->logger, MODULE_FLAG, + "flag runtime add item %d to item_hash failed, already exist", + item_id); + return -1; + } + + struct flag_item *flag_item = flag_item_new(line, schema, flag_rt->logger); + if (NULL == flag_item) { + return -1; + } + + u_para = maat_item_inner_new(flag_item->group_id, item_id, 0); + item = maat_item_new(item_id, flag_item->group_id, u_para); + HASH_ADD_INT(flag_rt->item_hash, item_id, item); + + flag_rule = flag_item_to_flag_rule(flag_item, u_para); + flag_item_free(flag_item); + if (NULL == flag_rule) { + log_error(flag_rt->logger, MODULE_FLAG, + "transform flag table(table_id:%d) item to flag_rule failed, item_id:%d", + schema->table_id, item_id); + return -1; + } + } + + char *key = (char *)&item_id; + int ret = flag_runtime_update_row(flag_rt, key, sizeof(int), item_id, flag_rule, is_valid); + if (ret < 0) { + if (flag_rule != NULL) { + flag_rule_free(flag_rule); + flag_rule = NULL; + } + return -1; + } else { + if (0 == is_valid) { + flag_rt->rule_num--; + } else { + flag_rt->rule_num++; + } + } + + return 0; } int flag_runtime_commit(void *flag_runtime) { + if (NULL == flag_runtime) { + return -1; + } + int ret = 0; + struct flag_runtime *flag_rt = (struct flag_runtime *)flag_runtime; + void **ex_data_array = NULL; + + size_t rule_cnt = rcu_hash_list_updating_data(flag_rt->htable, &ex_data_array); + if (0 == rule_cnt) { + FREE(ex_data_array); + return 0; + } + + struct flag_rule *rules = ALLOC(struct flag_rule, rule_cnt); + + for (size_t i = 0; i < rule_cnt; i++) { + rules[i] = *(struct flag_rule *)ex_data_array[i]; + } + + struct flag_matcher *new_flag_matcher = NULL; + struct flag_matcher *old_flag_matcher = NULL; + + log_info(flag_rt->logger, MODULE_FLAG, + "committing %zu flag rules for rebuilding flag_matcher engine", + rule_cnt); + new_flag_matcher = flag_matcher_new(rules, rule_cnt); + if (NULL == new_flag_matcher) { + log_error(flag_rt->logger, MODULE_FLAG, + "rebuild flag_matcher engine failed when update %zu flag rules", + rule_cnt); + ret = -1; + } + + old_flag_matcher = flag_rt->matcher; + flag_rt->matcher = new_flag_matcher; + + maat_garbage_bagging(flag_rt->ref_garbage_bin, old_flag_matcher, + (void (*)(void*))flag_matcher_free); + rcu_hash_commit(flag_rt->htable); + rule_cnt = rcu_hash_updating_count(flag_rt->htable); + assert(rule_cnt == 0); + + FREE(rules); + FREE(ex_data_array); + + return ret; } int flag_runtime_scan_flag(struct flag_runtime *flag_rt, int thread_id, - const char *data, size_t data_len, - int *group_ids, size_t group_ids_size, - int vtable_id, struct maat_state *state) + uint64_t flag, int *group_ids, size_t group_ids_size, + int vtable_id, struct maat_state *state) { + struct flag_result hit_results[MAX_SCANNER_HIT_ITEM_NUM] = {0}; + + int n_hit_item = flag_matcher_match(flag_rt->matcher, flag, + hit_results, MAX_SCANNER_HIT_ITEM_NUM); + if (n_hit_item <= 0) { + return n_hit_item; + } + if (n_hit_item > MAX_SCANNER_HIT_ITEM_NUM) { + log_info(flag_rt->logger, MODULE_FLAG, + "hit flag item count:%d exceed maxium:%d", + n_hit_item, MAX_SCANNER_HIT_ITEM_NUM); + n_hit_item = MAX_SCANNER_HIT_ITEM_NUM; + } + + int hit_item_ids[MAX_SCANNER_HIT_ITEM_NUM] = {-1}; + for (int i = 0; i < n_hit_item; i++) { + hit_item_ids[i] = hit_results[i].rule_id; + } + + size_t group_hit_cnt = 0; + int ret = maat_compile_state_update(flag_rt->item_hash, vtable_id, hit_item_ids, + n_hit_item, group_ids, group_ids_size, + &group_hit_cnt, state); + if (ret < 0) { + return -1; + } + + return group_hit_cnt; } void flag_runtime_scan_hit_inc(struct flag_runtime *flag_rt, int thread_id) { - + alignment_int64_array_add(flag_rt->hit_cnt, thread_id, 1); } long long flag_runtime_scan_hit_sum(struct flag_runtime *flag_rt, int n_thread) { - + return alignment_int64_array_sum(flag_rt->hit_cnt, n_thread); } \ No newline at end of file diff --git a/src/maat_interval.cpp b/src/maat_interval.cpp new file mode 100644 index 0000000..fda0621 --- /dev/null +++ b/src/maat_interval.cpp @@ -0,0 +1,486 @@ +/* +********************************************************************************************** +* File: maat_interval.cpp +* Description: +* Authors: Liu WenTan +* Date: 2022-10-31 +* Copyright: (c) 2018-2022 Geedge Networks, Inc. All rights reserved. +*********************************************************************************************** +*/ + +#include + +#include "maat_utils.h" +#include "log/log.h" +#include "maat_interval.h" +#include "maat_rule.h" +#include "rcu_hash.h" +#include "alignment.h" +#include "uthash/uthash.h" +#include "maat_garbage_collection.h" +#include "maat_compile.h" +#include "interval_matcher.h" + +#define MODULE_INTERVAL module_name_str("maat.interval") + +struct interval_schema { + int item_id_column; + int group_id_column; + int district_column; + int low_bound_column; + int up_bound_column; + int table_id; + struct table_manager *ref_tbl_mgr; +}; + +struct interval_item { + int item_id; + int group_id; + int district_id; + int low_bound; + int up_bound; +}; + +struct interval_runtime { + struct interval_matcher *matcher; + struct rcu_hash_table *htable; + + uint32_t rule_num; + uint32_t updating_rule_num; + struct maat_item *item_hash; + void (*item_user_data_free)(void *); + + struct maat_garbage_bin *ref_garbage_bin; + struct log_handle *logger; + + long long *scan_cnt; + long long *hit_cnt; +}; + +void *interval_schema_new(cJSON *json, struct table_manager *tbl_mgr, + const char *table_name, struct log_handle *logger) +{ + int read_cnt = 0; + struct interval_schema *schema = ALLOC(struct interval_schema, 1); + + cJSON *custom_item = NULL; + cJSON *item = cJSON_GetObjectItem(json, "table_id"); + if (item != NULL && item->type == cJSON_Number) { + schema->table_id = item->valueint; + read_cnt++; + } + + item = cJSON_GetObjectItem(json, "custom"); + if (item == NULL || item->type != cJSON_Object) { + log_error(logger, MODULE_INTERVAL, + "table %s has no custom column", table_name); + goto error; + } + + custom_item = cJSON_GetObjectItem(item, "item_id"); + if (custom_item != NULL && custom_item->type == cJSON_Number) { + schema->item_id_column = custom_item->valueint; + read_cnt++; + } + + custom_item = cJSON_GetObjectItem(item, "group_id"); + if (custom_item != NULL && custom_item->type == cJSON_Number) { + schema->group_id_column = custom_item->valueint; + read_cnt++; + } + + custom_item = cJSON_GetObjectItem(item, "district"); + if (custom_item != NULL && custom_item->type == cJSON_Number) { + schema->district_column = custom_item->valueint; + read_cnt++; + } + + custom_item = cJSON_GetObjectItem(item, "low_bound"); + if (custom_item != NULL && custom_item->type == cJSON_Number) { + schema->low_bound_column = custom_item->valueint; + read_cnt++; + } + + custom_item = cJSON_GetObjectItem(item, "up_bound"); + if (custom_item != NULL && custom_item->type == cJSON_Number) { + schema->up_bound_column = custom_item->valueint; + read_cnt++; + } + + schema->ref_tbl_mgr = tbl_mgr; + + if (read_cnt < 5) { + goto error; + } + + return schema; +error: + FREE(schema); + return NULL; +} + +void interval_schema_free(void *interval_schema) +{ + FREE(interval_schema); +} + +void interval_ex_data_free(void *user_ctx, void *data) +{ + struct interval_item *item = (struct interval_item *)data; + FREE(item); +} + +void *interval_runtime_new(void *interval_schema, int max_thread_num, + struct maat_garbage_bin *garbage_bin, + struct log_handle *logger) +{ + if (NULL == interval_schema) { + return NULL; + } + + struct interval_runtime *interval_rt = ALLOC(struct interval_runtime, 1); + + interval_rt->htable = rcu_hash_new(interval_ex_data_free); + interval_rt->item_user_data_free = maat_item_inner_free; + interval_rt->ref_garbage_bin = garbage_bin; + interval_rt->logger = logger; + + interval_rt->hit_cnt = alignment_int64_array_alloc(max_thread_num); + interval_rt->scan_cnt = alignment_int64_array_alloc(max_thread_num); + + return interval_rt; +} + +void interval_runtime_free(void *interval_runtime) +{ + if (NULL == interval_runtime) { + return; + } + + struct interval_runtime *interval_rt = (struct interval_runtime *)interval_runtime; + if (interval_rt->htable != NULL) { + rcu_hash_free(interval_rt->htable); + interval_rt->htable = NULL; + } + + struct maat_item *item = NULL, *tmp = NULL; + HASH_ITER(hh, interval_rt->item_hash, item, tmp) { + HASH_DELETE(hh, interval_rt->item_hash, item); + maat_item_free(item, interval_rt->item_user_data_free); + } + + if (interval_rt->hit_cnt != NULL) { + alignment_int64_array_free(interval_rt->hit_cnt); + interval_rt->hit_cnt = NULL; + } + + if (interval_rt->scan_cnt != NULL) { + alignment_int64_array_free(interval_rt->scan_cnt); + interval_rt->scan_cnt = NULL; + } + + FREE(interval_rt); +} + +struct interval_item *interval_item_new(const char *line, struct interval_schema *schema, + struct log_handle *logger) +{ + size_t column_offset = 0; + size_t column_len = 0; + enum table_type table_type = TABLE_TYPE_INVALID; + struct interval_item *item = ALLOC(struct interval_item, 1); + + int ret = get_column_pos(line, schema->item_id_column, &column_offset, &column_len); + if (ret < 0) { + log_error(logger, MODULE_INTERVAL, + "interval table(table_id:%d) line:%s has no item_id", + schema->table_id, line); + goto error; + } + item->item_id = atoi(line + column_offset); + + ret = get_column_pos(line, schema->group_id_column, &column_offset, &column_len); + if (ret < 0) { + log_error(logger, MODULE_INTERVAL, + "interval table(table_id:%d) line:%s has no group_id", + schema->table_id, line); + goto error; + } + item->group_id = atoi(line + column_offset); + + table_type = table_manager_get_table_type(schema->ref_tbl_mgr, schema->table_id); + if (table_type == TABLE_TYPE_INTERVAL_PLUS) { + ret = get_column_pos(line, schema->district_column, &column_offset, &column_len); + if (ret < 0) { + goto error; + } + + if (column_len >= MAX_DISTRICT_STR) { + log_error(logger, MODULE_INTERVAL, + "interval table(table_id:%d) line:%s district length too long", + schema->table_id, line); + goto error; + } + + char district[MAX_DISTRICT_STR] = {0}; + memcpy(district, (line + column_offset), column_len); + assert(strlen(district) > 0); + str_unescape(district); + item->district_id = table_manager_get_district_id(schema->ref_tbl_mgr, district); + } + + ret = get_column_pos(line, schema->low_bound_column, &column_offset, &column_len); + if (ret < 0) { + log_error(logger, MODULE_INTERVAL, + "interval table(table_id:%d) line:%s has no low_bound", + schema->table_id, line); + goto error; + } + item->low_bound = atoi(line + column_offset); + + ret = get_column_pos(line, schema->up_bound_column, &column_offset, &column_len); + if (ret < 0) { + log_error(logger, MODULE_INTERVAL, + "interval table(table_id:%d) line:%s has no up_bound", + schema->table_id, line); + goto error; + } + item->up_bound = atoi(line + column_offset); + + return item; +error: + FREE(item); + return NULL; +} + +void interval_item_free(void *interval_item) +{ + FREE(interval_item); +} + +struct interval_rule *interval_item_to_interval_rule(struct interval_item *item, void *user_data) +{ + struct interval_rule *rule = ALLOC(struct interval_rule, 1); + + rule->start = item->low_bound; + rule->end = item->up_bound; + rule->result.rule_id = item->item_id; + rule->result.user_tag = user_data; + + return rule; +} + +void interval_rule_free(struct interval_rule *rule) +{ + FREE(rule); +} + +int interval_runtime_update_row(struct interval_runtime *interval_rt, char *key, size_t key_len, + int item_id, struct interval_rule *rule, int is_valid) +{ + void *data = NULL; + + if (0 == is_valid) { + //delete + data = rcu_hash_find(interval_rt->htable, key, key_len); + if (NULL == data) { + log_error(interval_rt->logger, MODULE_INTERVAL, + "the key of interval rule not exist, can't be deleted, item_id:%d", + item_id); + return -1; + } + rcu_hash_del(interval_rt->htable, key, key_len); + } else { + //add + data = rcu_hash_find(interval_rt->htable, key, key_len); + if (data != NULL) { + log_error(interval_rt->logger, MODULE_INTERVAL, + "the key of interval rule already exist, can't be added, item_id:%d", + item_id); + return -1; + } + rcu_hash_add(interval_rt->htable, key, key_len, (void *)rule); + } + + return 0; +} + +int interval_runtime_update(void *interval_runtime, void *interval_schema, + const char *line, int valid_column) +{ + if (NULL == interval_runtime || NULL == interval_schema || + NULL == line) { + return -1; + } + + struct maat_item_inner *u_para = NULL; + struct maat_item *item = NULL; + struct interval_rule *interval_rule = NULL; + struct interval_schema *schema = (struct interval_schema *)interval_schema; + struct interval_runtime *interval_rt = (struct interval_runtime *)interval_runtime; + + int item_id = get_column_value(line, schema->item_id_column); + int is_valid = get_column_value(line, valid_column); + if (is_valid < 0) { + return -1; + } else if (0 == is_valid) { + //delete + HASH_FIND_INT(interval_rt->item_hash, &item_id, item); + if (NULL == item) { + return -1; + } + + u_para = (struct maat_item_inner *)item->user_data; + item->user_data = NULL; + + if (NULL == u_para) { + return -1; + } + + HASH_DELETE(hh, interval_rt->item_hash, item); + maat_garbage_bagging(interval_rt->ref_garbage_bin, u_para, maat_item_inner_free); + } else { + //add + HASH_FIND_INT(interval_rt->item_hash, &item_id, item); + if (item) { + log_error(interval_rt->logger, MODULE_INTERVAL, + "interval runtime add item %d to item_hash failed, already exist", + item_id); + return -1; + } + + struct interval_item *interval_item = interval_item_new(line, schema, interval_rt->logger); + if (NULL == interval_item) { + return -1; + } + + u_para = maat_item_inner_new(interval_item->group_id, item_id, 0); + item = maat_item_new(item_id, interval_item->group_id, u_para); + HASH_ADD_INT(interval_rt->item_hash, item_id, item); + + interval_rule = interval_item_to_interval_rule(interval_item, u_para); + interval_item_free(interval_item); + if (NULL == interval_rule) { + log_error(interval_rt->logger, MODULE_INTERVAL, + "transform interval table(table_id:%d) item to interval_rule failed, item_id:%d", + schema->table_id, item_id); + return -1; + } + } + + char *key = (char *)&item_id; + int ret = interval_runtime_update_row(interval_rt, key, sizeof(int), item_id, interval_rule, is_valid); + if (ret < 0) { + if (interval_rule != NULL) { + interval_rule_free(interval_rule); + interval_rule = NULL; + } + return -1; + } else { + if (0 == is_valid) { + interval_rt->rule_num--; + } else { + interval_rt->rule_num++; + } + } + + return 0; +} + +int interval_runtime_commit(void *interval_runtime) +{ + if (NULL == interval_runtime) { + return -1; + } + + int ret = 0; + struct interval_runtime *interval_rt = (struct interval_runtime *)interval_runtime; + void **ex_data_array = NULL; + + size_t rule_cnt = rcu_hash_list_updating_data(interval_rt->htable, &ex_data_array); + if (0 == rule_cnt) { + FREE(ex_data_array); + return 0; + } + + struct interval_rule *rules = ALLOC(struct interval_rule, rule_cnt); + + for (size_t i = 0; i < rule_cnt; i++) { + rules[i] = *(struct interval_rule *)ex_data_array[i]; + } + + struct interval_matcher *new_interval_matcher = NULL; + struct interval_matcher *old_interval_matcher = NULL; + + log_info(interval_rt->logger, MODULE_INTERVAL, + "committing %zu interval rules for rebuilding interval_matcher engine", + rule_cnt); + + new_interval_matcher = interval_matcher_new(rules, rule_cnt); + if (NULL == new_interval_matcher) { + log_error(interval_rt->logger, MODULE_INTERVAL, + "rebuild interval_matcher engine failed when update %zu interval rules", + rule_cnt); + ret = -1; + } + + old_interval_matcher = interval_rt->matcher; + interval_rt->matcher = new_interval_matcher; + + maat_garbage_bagging(interval_rt->ref_garbage_bin, old_interval_matcher, + (void (*)(void*))interval_matcher_free); + rcu_hash_commit(interval_rt->htable); + rule_cnt = rcu_hash_updating_count(interval_rt->htable); + assert(rule_cnt == 0); + + FREE(rules); + FREE(ex_data_array); + + return ret; +} + +int interval_runtime_scan_intval(struct interval_runtime *interval_rt, int thread_id, + uint64_t integer, + int *group_ids, size_t group_ids_size, + int vtable_id, struct maat_state *state) +{ + struct interval_result hit_results[MAX_SCANNER_HIT_ITEM_NUM] = {0}; + + int n_hit_item = interval_matcher_match(interval_rt->matcher, integer, + hit_results, MAX_SCANNER_HIT_ITEM_NUM); + if (n_hit_item <= 0) { + return n_hit_item; + } + + if (n_hit_item > MAX_SCANNER_HIT_ITEM_NUM) { + log_info(interval_rt->logger, MODULE_INTERVAL, + "hit interval item count:%d exceed maxium:%d", + n_hit_item, MAX_SCANNER_HIT_ITEM_NUM); + n_hit_item = MAX_SCANNER_HIT_ITEM_NUM; + } + + int hit_item_ids[MAX_SCANNER_HIT_ITEM_NUM] = {-1}; + for (int i = 0; i < n_hit_item; i++) { + hit_item_ids[i] = hit_results[i].rule_id; + } + + size_t group_hit_cnt = 0; + int ret = maat_compile_state_update(interval_rt->item_hash, vtable_id, hit_item_ids, + n_hit_item, group_ids, group_ids_size, + &group_hit_cnt, state); + if (ret < 0) { + return -1; + } + + return group_hit_cnt; +} + +void interval_runtime_scan_hit_inc(struct interval_runtime *interval_rt, int thread_id) +{ + alignment_int64_array_add(interval_rt->hit_cnt, thread_id, 1); +} + +long long interval_runtime_scan_hit_sum(struct interval_runtime *interval_rt, int n_thread) +{ + return alignment_int64_array_sum(interval_rt->hit_cnt, n_thread); +} \ No newline at end of file diff --git a/src/maat_intval.cpp b/src/maat_intval.cpp deleted file mode 100644 index 28f069f..0000000 --- a/src/maat_intval.cpp +++ /dev/null @@ -1,72 +0,0 @@ -/* -********************************************************************************************** -* File: maat_intval.cpp -* Description: -* Authors: Liu WenTan -* Date: 2022-10-31 -* Copyright: (c) 2018-2022 Geedge Networks, Inc. All rights reserved. -*********************************************************************************************** -*/ - -#include "maat_intval.h" - -struct intval_schema { - -}; - -struct intval_runtime { - -}; - -void *intval_schema_new(cJSON *json, struct table_manager *tbl_mgr, - const char *table_name, struct log_handle *logger) -{ - -} - -void _schema_free(void *intval_schema) -{ - -} - -/* intval runtime API */ -void *intval_runtime_new(void *intval_schema, int max_thread_num, - struct maat_garbage_bin *garbage_bin, - struct log_handle *logger) -{ - -} - -void intval_runtime_free(void *intval_runtime) -{ - -} - -int intval_runtime_update(void *intval_runtime, void *intval_schema, - const char *line, int valid_column) -{ - -} - -int intval_runtime_commit(void *intval_runtime) -{ - -} - -int intval_runtime_scan_intval(struct intval_runtime *intval_rt, int thread_id, - const char *data, size_t data_len, - int *group_ids, size_t group_ids_size, - int vtable_id, struct maat_state *state) -{ - -} - -void intval_runtime_scan_hit_inc(struct intval_runtime *intval_rt, int thread_id) -{ - -} - -long long intval_runtime_scan_hit_sum(struct intval_runtime *intval_rt, int n_thread) -{ - -} \ No newline at end of file diff --git a/src/maat_plugin.cpp b/src/maat_plugin.cpp index 945fe58..1eddfaa 100644 --- a/src/maat_plugin.cpp +++ b/src/maat_plugin.cpp @@ -369,7 +369,8 @@ int plugin_accept_tag_match(struct plugin_schema *schema, const char *line, int plugin_runtime_update(void *plugin_runtime, void *plugin_schema, const char *line, int valid_column) { - if (NULL == plugin_runtime || NULL == plugin_schema) { + if (NULL == plugin_runtime || NULL == plugin_schema || + NULL == line) { return -1; } diff --git a/src/maat_rule.cpp b/src/maat_rule.cpp index 67304db..64475c1 100644 --- a/src/maat_rule.cpp +++ b/src/maat_rule.cpp @@ -350,7 +350,7 @@ uint32_t maat_runtime_rule_num(struct maat_runtime *maat_rt) void maat_plugin_table_all_callback_finish(struct table_manager *tbl_mgr) { size_t table_cnt = table_manager_table_count(tbl_mgr); - enum table_type table_type = TABLE_TYPE_MAX; + enum table_type table_type = TABLE_TYPE_INVALID; for (size_t i = 0; i < table_cnt; i++) { table_type = table_manager_get_table_type(tbl_mgr, i); diff --git a/src/maat_table.cpp b/src/maat_table.cpp index c27a53e..6a645de 100644 --- a/src/maat_table.cpp +++ b/src/maat_table.cpp @@ -21,6 +21,7 @@ #include "maat_ip.h" #include "maat_compile.h" #include "maat_group.h" +#include "maat_flag.h" #include "maat_plugin.h" #include "maat_ip_plugin.h" #include "maat_virtual.h" @@ -75,14 +76,17 @@ struct table_operations { }; struct table_operations table_ops[TABLE_TYPE_MAX] = { + // { + // .type = TABLE_TYPE_FLAG, + // .new_schema = flag_schema_new, + // .free_schema = flag_schema_free, + // .new_runtime = flag_runtime_new, + // .free_runtime = flag_runtime_free, + // .update_runtime = flag_runtime_update, + // .commit_runtime = flag_runtime_commit + // }, { - .type = TABLE_TYPE_FLAG, - .new_schema = flag_schema_new, - .free_schema = flag_schema_free, - .new_runtime = flag_runtime_new, - .free_runtime = flag_runtime_free, - .update_runtime = flag_runtime_update, - .commit_runtime = flag_runtime_commit + .type = TABLE_TYPE_FLAG }, { .type = TABLE_TYPE_EXPR, @@ -130,22 +134,10 @@ struct table_operations table_ops[TABLE_TYPE_MAX] = { .commit_runtime = NULL }, { - .type = TABLE_TYPE_DIGEST, - .new_schema = NULL, - .free_schema = NULL, - .new_runtime = NULL, - .free_runtime = NULL, - .update_runtime = NULL, - .commit_runtime = NULL + .type = TABLE_TYPE_DIGEST }, { - .type = TABLE_TYPE_SIMILARITY, - .new_schema = NULL, - .free_schema = NULL, - .new_runtime = NULL, - .free_runtime = NULL, - .update_runtime = NULL, - .commit_runtime = NULL + .type = TABLE_TYPE_SIMILARITY }, { .type = TABLE_TYPE_CONJUNCTION, @@ -262,9 +254,13 @@ static void register_reserved_word(struct maat_kv_store *reserved_word_map) maat_kv_register(reserved_word_map, "group2group", TABLE_TYPE_GROUP2GROUP); maat_kv_register(reserved_word_map, "expr", TABLE_TYPE_EXPR); maat_kv_register(reserved_word_map, "expr_plus", TABLE_TYPE_EXPR_PLUS); + maat_kv_register(reserved_word_map, "intval", TABLE_TYPE_INTERVAL); + maat_kv_register(reserved_word_map, "intval_plus", TABLE_TYPE_INTERVAL_PLUS); maat_kv_register(reserved_word_map, "ip_plus", TABLE_TYPE_IP_PLUS); maat_kv_register(reserved_word_map, "plugin", TABLE_TYPE_PLUGIN); maat_kv_register(reserved_word_map, "ip_plugin", TABLE_TYPE_IP_PLUGIN); + maat_kv_register(reserved_word_map, "bool_plugin", TABLE_TYPE_BOOL_PLUGIN); + maat_kv_register(reserved_word_map, "fqdn_plugin", TABLE_TYPE_FQDN_PLUGIN); maat_kv_register(reserved_word_map, "virtual", TABLE_TYPE_VIRTUAL); } @@ -511,7 +507,7 @@ int table_manager_runtime_create(struct table_manager *tbl_mgr, int max_thread_n assert(tbl_mgr->n_table != 0); size_t i = 0; - enum table_type table_type = TABLE_TYPE_MAX; + enum table_type table_type = TABLE_TYPE_INVALID; for (i = 0; i < MAX_TABLE_NUM; i++) { void *schema = table_manager_get_schema(tbl_mgr, i); @@ -629,11 +625,11 @@ int table_manager_get_table_id(struct table_manager *tbl_mgr, const char *name) enum table_type table_manager_get_table_type(struct table_manager *tbl_mgr, int table_id) { if (NULL == tbl_mgr || table_id < 0 || table_id >= MAX_TABLE_NUM) { - return TABLE_TYPE_MAX; + return TABLE_TYPE_INVALID; } if (NULL == tbl_mgr->tbl[table_id]) { - return TABLE_TYPE_MAX; + return TABLE_TYPE_INVALID; } return tbl_mgr->tbl[table_id]->table_type; @@ -750,14 +746,14 @@ int table_manager_update_runtime(struct table_manager *tbl_mgr, int table_id, } enum table_type table_type = table_manager_get_table_type(tbl_mgr, table_id); - if (table_type == TABLE_TYPE_MAX) { + if (table_type == TABLE_TYPE_INVALID) { return -1; } if (NULL == table_ops[table_type].update_runtime) { return -1; } - + int ret = table_ops[table_type].update_runtime(runtime, schema, line, valid_column); if (tbl_mgr->tmp_district_map != NULL) { @@ -773,7 +769,7 @@ int table_manager_update_runtime(struct table_manager *tbl_mgr, int table_id, void table_manager_commit_runtime(struct table_manager *tbl_mgr, int table_id) { enum table_type table_type = table_manager_get_table_type(tbl_mgr, table_id); - if (table_type == TABLE_TYPE_MAX) { + if (table_type == TABLE_TYPE_INVALID) { return; } @@ -782,6 +778,7 @@ void table_manager_commit_runtime(struct table_manager *tbl_mgr, int table_id) return; } + printf("table_id:%d\n", table_id); if ( table_ops[table_type].commit_runtime != NULL) { table_ops[table_type].commit_runtime(runtime); } diff --git a/src/maat_virtual.cpp b/src/maat_virtual.cpp index 5786a59..364d462 100644 --- a/src/maat_virtual.cpp +++ b/src/maat_virtual.cpp @@ -8,11 +8,15 @@ *********************************************************************************************** */ +#include + #include "cJSON/cJSON.h" #include "maat_kv.h" #include "maat_utils.h" #include "log/log.h" #include "maat_virtual.h" +#include "maat_rule.h" +#include "maat_table.h" #define MODULE_VIRTUAL module_name_str("maat.virtual") @@ -25,8 +29,6 @@ struct virtual_schema { void *virtual_schema_new(cJSON *json, struct table_manager *tbl_mgr, const char *table_name, struct log_handle *logger) { - //size_t read_cnt = 0; - cJSON *item = cJSON_GetObjectItem(json, "physical_table"); if (NULL == item || item->type != cJSON_Array) { log_error(logger, MODULE_VIRTUAL, @@ -35,37 +37,43 @@ void *virtual_schema_new(cJSON *json, struct table_manager *tbl_mgr, } struct virtual_schema *vt_schema = ALLOC(struct virtual_schema, 1); - vt_schema->tbl_mgr = tbl_mgr; - return vt_schema; -#if 0 - struct virtual_schema *vt_schema = ALLOC(struct virtual_schema, 1); + int cnt = cJSON_GetArraySize(item); for (int i = 0; i < cnt; i++) { cJSON *tmp_item = cJSON_GetArrayItem(item, i); if (tmp_item != NULL && tmp_item->type == cJSON_String) { - int table_id = -1; + int table_id = table_manager_get_table_id(tbl_mgr, tmp_item->valuestring); /* physical table should already exist */ - int ret = maat_kv_read(tablename2id_map, tmp_item->valuestring, &table_id); - if (ret < 0) { - return -1; - } + if (table_id < 0) { + log_error(logger, MODULE_VIRTUAL, "table:%s is not registered", + tmp_item->valuestring); + FREE(vt_schema); + return NULL; + } - enum scan_type table_scan_type = table_schema_get_scan_type(table_array[table_id]); - vt_schema->physical_table_id[table_scan_type]= table_id; + enum table_type table_type = table_manager_get_table_type(tbl_mgr, table_id); + assert(table_type != TABLE_TYPE_INVALID); + enum scan_type scan_type = maat_table_get_scan_type(table_type); + assert(scan_type != SCAN_TYPE_INVALID); + vt_schema->physical_table_id[scan_type]= table_id; } } - if (read_cnt < 3) { - FREE(vt_schema); - return NULL; - } - return vt_schema; -#endif } void virtual_schema_free(void *virtual_schema) { FREE(virtual_schema); +} + +int virtual_table_get_physical_table_id(void *virtual_schema, enum scan_type type) +{ + if (NULL == virtual_schema) { + return -1; + } + + struct virtual_schema *schema = (struct virtual_schema *)virtual_schema; + return schema->physical_table_id[type]; } \ No newline at end of file diff --git a/test/maat_framework_gtest.cpp b/test/maat_framework_gtest.cpp index 6006506..c6c3576 100644 --- a/test/maat_framework_gtest.cpp +++ b/test/maat_framework_gtest.cpp @@ -113,7 +113,7 @@ TEST_F(MaatIPScan, hit_ip_and_port) { state = NULL; ret = maat_scan_ipv4(g_maat_instance, table_id, 0, sip, results, sizeof(results), &n_hit_result, &state); - EXPECT_EQ(ret, MAAT_OK); + EXPECT_EQ(ret, MAAT_SCAN_OK); EXPECT_EQ(n_hit_result, 0); maat_state_free(&state); @@ -148,7 +148,7 @@ TEST_F(MaatStringScan, hit_ip_and_port_range) { n_hit_result = 0; state = NULL; ret = maat_scan_ipv4(g_maat_instance, table_id, 0, sip, results, sizeof(results), &n_hit_result, &state); - EXPECT_EQ(ret, MAAT_OK); + EXPECT_EQ(ret, MAAT_SCAN_OK); EXPECT_EQ(n_hit_result, 0); } @@ -183,7 +183,7 @@ TEST(maat_scan_ipv4, hit_ip_range_and_port_range) { EXPECT_EQ(ret, 1); state = NULL; ret = maat_scan_ipv4(g_maat_instance, table_id, 0, sip, results, sizeof(results), &n_hit_result, &state); - EXPECT_EQ(ret, MAAT_OK); + EXPECT_EQ(ret, MAAT_SCAN_OK); EXPECT_EQ(n_hit_result, 0); maat_state_free(&state); } @@ -209,7 +209,7 @@ TEST(maat_scan_ipv4, hit_ip_cidr_and_port_range) { EXPECT_EQ(ret, 1); state = NULL; ret = maat_scan_ipv4(g_maat_instance, table_id, 0, sip, results, sizeof(results), &n_hit_result, &state); - EXPECT_EQ(ret, MAAT_OK); + EXPECT_EQ(ret, MAAT_SCAN_OK); EXPECT_EQ(n_hit_result, 0); maat_state_free(&state); } @@ -233,7 +233,7 @@ TEST(maat_scan_ipv4, hit_ip_cidr_and_port_mask) { state = NULL; ret = maat_scan_ipv4(g_maat_instance, table_id, 0, sip, results, sizeof(results), &n_hit_result, &state); - EXPECT_EQ(ret, MAAT_OK); + EXPECT_EQ(ret, MAAT_SCAN_OK); EXPECT_EQ(n_hit_result, 0); maat_state_free(&state); } @@ -256,7 +256,7 @@ TEST(maat_scan_ipv6, hit_ip_range_and_port_mask) { state = NULL; ret = maat_scan_ipv6(g_maat_instance, table_id, 0, sip6, results, sizeof(results), &n_hit_result, &state); - EXPECT_EQ(ret, MAAT_OK); + EXPECT_EQ(ret, MAAT_SCAN_OK); EXPECT_EQ(n_hit_result, 0); maat_state_free(&state); } @@ -269,7 +269,7 @@ TEST(maat_scan_string, dynamic_config) { size_t n_hit_result = 0; struct maat_state *state = NULL; int ret = maat_scan_string(g_maat_instance, table_id, 0, data, strlen(data), results, sizeof(results), &n_hit_result, &state); - EXPECT_EQ(ret, MAAT_OK); + EXPECT_EQ(ret, MAAT_SCAN_OK); EXPECT_EQ(n_hit_result, 0); maat_state_free(&state); @@ -303,7 +303,7 @@ TEST(maat_scan_ip, dynamic_config) { size_t n_hit_result = 0; struct maat_state *state = NULL; ret = maat_scan_ipv4(g_maat_instance, table_id, 0, sip, results, sizeof(results), &n_hit_result, &state); - EXPECT_EQ(ret, MAAT_OK); + EXPECT_EQ(ret, MAAT_SCAN_OK); EXPECT_EQ(n_hit_result, 0); maat_state_free(&state); diff --git a/test/maat_input_mode_gtest.cpp b/test/maat_input_mode_gtest.cpp index 391aa41..2b920da 100644 --- a/test/maat_input_mode_gtest.cpp +++ b/test/maat_input_mode_gtest.cpp @@ -10,6 +10,7 @@ const char *table_info_path = "./table_info.conf"; const char *json_filename = "maat_json.json"; +#if 0 TEST(json_mode, maat_scan_string) { char json_iris_path[128] = {0}; snprintf(json_iris_path, sizeof(json_iris_path), "./%s_iris_tmp", json_filename); @@ -26,16 +27,16 @@ TEST(json_mode, maat_scan_string) { struct maat *maat_instance = maat_new(opts, table_info_path); EXPECT_NE(maat_instance, nullptr); - int table_id = table_manager_get_table_id(maat_instance->tbl_mgr, "HTTP_URL"); + int table_id = table_manager_get_table_id(maat_instance->tbl_mgr, "KEYWORDS_TABLE"); - char scan_data[128] = "hello"; + char scan_data[128] = "Batman\\:Take me Home.Superman/:Fine,stay with me."; int results[5] = {0}; size_t n_hit_result = 0; struct maat_state *state = NULL; int ret = maat_scan_string(maat_instance, table_id, 0, scan_data, strlen(scan_data), results, sizeof(results), &n_hit_result, &state); EXPECT_EQ(ret, MAAT_SCAN_HIT); EXPECT_EQ(n_hit_result, 1); - EXPECT_EQ(results[0], 191); + EXPECT_EQ(results[0], 132); maat_options_free(opts); maat_state_free(&state); @@ -69,22 +70,22 @@ TEST(iris_mode, maat_scan_string) { struct maat *maat_instance = maat_new(opts, table_info_path); EXPECT_NE(maat_instance, nullptr); - int table_id = table_manager_get_table_id(maat_instance->tbl_mgr, "HTTP_URL"); + int table_id = table_manager_get_table_id(maat_instance->tbl_mgr, "KEYWORDS_TABLE"); - char scan_data[128] = "hello"; + char scan_data[128] = "Batman\\:Take me Home.Superman/:Fine,stay with me."; int results[5] = {0}; size_t n_hit_result = 0; struct maat_state *state = NULL; int ret = maat_scan_string(maat_instance, table_id, 0, scan_data, strlen(scan_data), results, sizeof(results), &n_hit_result, &state); EXPECT_EQ(ret, MAAT_SCAN_HIT); EXPECT_EQ(n_hit_result, 1); - EXPECT_EQ(results[0], 191); + EXPECT_EQ(results[0], 132); maat_options_free(opts); maat_state_free(&state); maat_free(maat_instance); } - +#endif int count_line_num_cb(const char *table_name, const char *line, void *u_para) { (*((unsigned int *)u_para))++; @@ -131,6 +132,8 @@ int make_serial_rule(const char *table_name, const char *line, void *u_para) return 0; } + +#if 1 TEST(redis_mode, maat_scan_string) { char json_iris_path[128] = {0}; char redis_ip[64] = "127.0.0.1"; @@ -189,23 +192,23 @@ TEST(redis_mode, maat_scan_string) { maat_options_set_logger(opts, logger); struct maat *maat_instance = maat_new(opts, table_info_path); - int table_id = table_manager_get_table_id(maat_instance->tbl_mgr, "HTTP_URL"); + int table_id = table_manager_get_table_id(maat_instance->tbl_mgr, "KEYWORDS_TABLE"); - char scan_data[128] = "hello"; + char scan_data[128] = "Batman\\:Take me Home.Superman/:Fine,stay with me."; int results[5] = {0}; size_t n_hit_result = 0; struct maat_state *state = NULL; int ret = maat_scan_string(maat_instance, table_id, 0, scan_data, strlen(scan_data), results, sizeof(results), &n_hit_result, &state); EXPECT_EQ(ret, MAAT_SCAN_HIT); EXPECT_EQ(n_hit_result, 1); - EXPECT_EQ(results[0], 191); + EXPECT_EQ(results[0], 132); maat_options_free(opts); maat_state_free(&state); log_handle_destroy(maat_instance->logger); maat_free(maat_instance); } - +#endif int main(int argc, char ** argv) { int ret=0; diff --git a/test/table_info.conf b/test/table_info.conf index 6eb2ce8..5a5c2cd 100644 --- a/test/table_info.conf +++ b/test/table_info.conf @@ -20,7 +20,7 @@ "table_id":1, "table_name":"GROUP2COMPILE", "table_type":"group2compile", - "associated_compile_table_id":1, + "associated_compile_table_id":0, "valid_column":3, "custom": { "group_id":1, @@ -113,6 +113,7 @@ "table_type":"expr_plus", "valid_column":8, "custom": { + "scan_mode":"block", "item_id":1, "group_id":2, "district":3, @@ -210,7 +211,6 @@ "expr_type":5, "match_method":6, "is_hexbin":7 - } }, { @@ -377,12 +377,6 @@ "fqdn":3 } }, - { - "table_id":37, - "table_name":"VIRTUAL_SSL_SNI", - "table_type":"virtual", - "physical_table": ["KEYWORDS_TABLE", "INTERGER_PLUS"] - }, { "table_id":38, "table_name":"APP_ID", diff --git a/vendor/CMakeLists.txt b/vendor/CMakeLists.txt index 78859fa..f3eb0c4 100644 --- a/vendor/CMakeLists.txt +++ b/vendor/CMakeLists.txt @@ -99,4 +99,30 @@ file(MAKE_DIRECTORY ${INSTALL_DIR}/include) add_library(igraph-static STATIC IMPORTED GLOBAL) add_dependencies(igraph-static igraph) set_property(TARGET igraph-static PROPERTY IMPORTED_LOCATION ${INSTALL_DIR}/lib/libigraph.a) -set_property(TARGET igraph-static PROPERTY INTERFACE_INCLUDE_DIRECTORIES ${INSTALL_DIR}/include) \ No newline at end of file +set_property(TARGET igraph-static PROPERTY INTERFACE_INCLUDE_DIRECTORIES ${INSTALL_DIR}/include) + +#flag-matcher +ExternalProject_Add(flagmatcher PREFIX flagmatcher + URL ${CMAKE_CURRENT_SOURCE_DIR}/flag-matcher.tar.gz + CMAKE_ARGS -DCMAKE_INSTALL_PREFIX=${VENDOR_BUILD} -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} -DCMAKE_C_FLAGS="-fPIC") + +ExternalProject_Get_Property(flagmatcher INSTALL_DIR) +file(MAKE_DIRECTORY ${VENDOR_BUILD}/include) + +add_library(flagmatcher-static STATIC IMPORTED GLOBAL) +add_dependencies(flagmatcher-static flagmatcher) +set_property(TARGET flagmatcher-static PROPERTY IMPORTED_LOCATION ${VENDOR_BUILD}/lib/libflag_matcher.a) +set_property(TARGET flagmatcher-static PROPERTY INTERFACE_INCLUDE_DIRECTORIES ${VENDOR_BUILD}/include) + +#interval-matcher +ExternalProject_Add(intervalmatcher PREFIX intervalmatcher + URL ${CMAKE_CURRENT_SOURCE_DIR}/interval-matcher.tar.gz + CMAKE_ARGS -DCMAKE_INSTALL_PREFIX=${VENDOR_BUILD} -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} -DCMAKE_C_FLAGS="-fPIC") + +ExternalProject_Get_Property(intervalmatcher INSTALL_DIR) +file(MAKE_DIRECTORY ${VENDOR_BUILD}/include) + +add_library(intervalmatcher-static STATIC IMPORTED GLOBAL) +add_dependencies(intervalmatcher-static flagmatcher) +set_property(TARGET intervalmatcher-static PROPERTY IMPORTED_LOCATION ${VENDOR_BUILD}/lib/libinterval_matcher.a) +set_property(TARGET intervalmatcher-static PROPERTY INTERFACE_INCLUDE_DIRECTORIES ${VENDOR_BUILD}/include) \ No newline at end of file diff --git a/vendor/flag_matcher.tar.gz b/vendor/flag-matcher.tar.gz similarity index 69% rename from vendor/flag_matcher.tar.gz rename to vendor/flag-matcher.tar.gz index 8f21dd5..b01742b 100644 Binary files a/vendor/flag_matcher.tar.gz and b/vendor/flag-matcher.tar.gz differ diff --git a/vendor/interval-matcher.tar.gz b/vendor/interval-matcher.tar.gz new file mode 100644 index 0000000..4889b35 Binary files /dev/null and b/vendor/interval-matcher.tar.gz differ diff --git a/vendor/interval_matcher.tar.gz b/vendor/interval_matcher.tar.gz deleted file mode 100644 index b6eb22c..0000000 Binary files a/vendor/interval_matcher.tar.gz and /dev/null differ