diff --git a/scanner/adapter_hs.cpp b/scanner/adapter_hs.cpp index 02229b9..2222f40 100644 --- a/scanner/adapter_hs.cpp +++ b/scanner/adapter_hs.cpp @@ -189,9 +189,10 @@ void adpt_hs_compile_data_free(struct adpt_hs_compile_data *hs_cd, size_t n_patt FREE(hs_cd); } -struct adapter_hs *adapter_hs_initialize(int scan_mode, size_t n_worker_thread, - and_expr_t *expr_array, size_t n_expr_array, - struct log_handle *logger) +struct adapter_hs * +adapter_hs_initialize(int scan_mode, size_t n_worker_thread, + and_expr_t *expr_array, size_t n_expr_array, + struct log_handle *logger) { if ((scan_mode != HS_SCAN_MODE_BLOCK && scan_mode != HS_SCAN_MODE_STREAM) || 0 == n_worker_thread || NULL == expr_array || 0 == n_expr_array) { @@ -275,6 +276,7 @@ struct adapter_hs *adapter_hs_initialize(int scan_mode, size_t n_worker_thread, } exprs[i].expr_id = expr_array[i].expr_id; exprs[i].item_num = expr_array[i].n_patterns; + exprs[i].user_tag = expr_array[i].user_tag; } if (literal_cd != NULL) { @@ -409,7 +411,7 @@ int matched_event_cb(unsigned int id, unsigned long long from, int adapter_hs_scan(struct adapter_hs *hs_instance, int thread_id, const char *data, size_t data_len, struct hs_scan_result *results, - size_t *n_results) + size_t n_result, size_t *n_hit_result) { if (NULL == hs_instance || NULL == data || (0 == data_len) || NULL == results || NULL == n_results) { @@ -466,11 +468,15 @@ int adapter_hs_scan(struct adapter_hs *hs_instance, int thread_id, goto next; } + if (bool_matcher_ret > n_result) { + bool_matcher_ret = n_result; + } + for (matched_index = 0; matched_index < bool_matcher_ret; matched_index++) { results[matched_index].item_id = bool_matcher_results[matched_index].expr_id; results[matched_index].user_tag = bool_matcher_results[matched_index].user_tag; } - *n_results = bool_matcher_ret; + *n_hit_result = bool_matcher_ret; next: FREE(bool_matcher_results); utarray_free(pattern_id_set); diff --git a/scanner/adapter_hs.h b/scanner/adapter_hs.h index fd0756d..5cc4eec 100644 --- a/scanner/adapter_hs.h +++ b/scanner/adapter_hs.h @@ -71,8 +71,10 @@ typedef struct { * * @retval the pointer to adapter_hs instance */ -struct adapter_hs *adapter_hs_initialize(int scan_mode, size_t nr_worker_threads, and_expr_t *expr_array, - size_t n_expr_array, struct log_handle *logger); +struct adapter_hs * +adapter_hs_initialize(int scan_mode, size_t n_worker_thread, + and_expr_t *expr_array, size_t n_expr_array, + struct log_handle *logger); /** * @brief scan input data to match logic AND expression, return all matched expr_id @@ -84,8 +86,10 @@ struct adapter_hs *adapter_hs_initialize(int scan_mode, size_t nr_worker_threads * @param results: the array of expr_id * @param n_results: number of elements in array of expr_id */ -int adapter_hs_scan(struct adapter_hs *instance, int thread_id, const char *data, size_t data_len, - struct hs_scan_result *results, size_t *n_results); +int adapter_hs_scan(struct adapter_hs *hs_instance, int thread_id, + const char *data, size_t data_len, + struct hs_scan_result *results, + size_t n_result, size_t *n_hit_result); /** * @brief destroy adapter_hs instance diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index df304cc..85ac26a 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -10,7 +10,7 @@ message(STATUS "Maat Frame, Version: ${MAAT_FRAME_VERSION}") add_definitions(-fPIC) set(MAAT_SRC json2iris.cpp maat_api.cpp rcu_hash.cpp maat_garbage_collection.cpp maat_config_monitor.cpp maat_rule.cpp maat_kv.cpp maat_ex_data.cpp maat_utils.cpp maat_command.cpp maat_redis_monitor.cpp - maat_table.cpp maat_compile.cpp maat_group.cpp maat_ip.cpp + maat_table.cpp maat_compile.cpp maat_group.cpp maat_ip.cpp maat_intval.cpp maat_flag.cpp maat_expr.cpp maat_plugin.cpp maat_ip_plugin.cpp maat_virtual.cpp) set(LIB_SOURCE_FILES diff --git a/src/inc_internal/maat_flag.h b/src/inc_internal/maat_flag.h new file mode 100644 index 0000000..f535730 --- /dev/null +++ b/src/inc_internal/maat_flag.h @@ -0,0 +1,56 @@ +/* +********************************************************************************************** +* File: maat_flag.h +* Description: +* Authors: Liu WenTan +* Date: 2022-10-31 +* Copyright: (c) 2018-2022 Geedge Networks, Inc. All rights reserved. +*********************************************************************************************** +*/ + +#ifndef _MAAT_FLAG_H_ +#define _MAAT_FLAG_H_ + +#ifdef __cpluscplus +extern "C" +{ +#endif + +#include "log/log.h" +#include "cJSON/cJSON.h" + +struct flag_runtime; + +void *flag_schema_new(cJSON *json, struct table_manager *tbl_mgr, + const char *table_name, struct log_handle *logger); +void flag_schema_free(void *flag_schema); + +/* flag runtime API */ +void *flag_runtime_new(void *flag_schema, int max_thread_num, + struct maat_garbage_bin *garbage_bin, + struct log_handle *logger); +void flag_runtime_free(void *flag_runtime); + +int flag_runtime_update(void *flag_runtime, void *flag_schema, + const char *line, int valid_column); +int flag_runtime_commit(void *flag_runtime); + +/* flag runtime scan API */ +/** + * @brief scan flag to get hit group_ids + * + * @retval the num of hit group_id +*/ +int flag_runtime_scan_flag(struct flag_runtime *flag_rt, int thread_id, + const char *data, size_t data_len, + int *group_ids, size_t group_ids_size, + int vtable_id, struct maat_state *state); + +void flag_runtime_scan_hit_inc(struct flag_runtime *flag_rt, int thread_id); +long long flag_runtime_scan_hit_sum(struct flag_runtime *flag_rt, int n_thread); + +#ifdef __cpluscplus +} +#endif + +#endif diff --git a/src/inc_internal/maat_intval.h b/src/inc_internal/maat_intval.h new file mode 100644 index 0000000..adab665 --- /dev/null +++ b/src/inc_internal/maat_intval.h @@ -0,0 +1,56 @@ +/* +********************************************************************************************** +* File: maat_intval.h +* Description: +* Authors: Liu WenTan +* Date: 2022-10-31 +* Copyright: (c) 2018-2022 Geedge Networks, Inc. All rights reserved. +*********************************************************************************************** +*/ + +#ifndef _MAAT_INTVAL_H_ +#define _MAAT_INTVAL_H_ + +#ifdef __cpluscplus +extern "C" +{ +#endif + +#include "log/log.h" +#include "cJSON/cJSON.h" + +struct intval_runtime; + +void *intval_schema_new(cJSON *json, struct table_manager *tbl_mgr, + const char *table_name, struct log_handle *logger); +void _schema_free(void *intval_schema); + +/* intval runtime API */ +void *intval_runtime_new(void *intval_schema, int max_thread_num, + struct maat_garbage_bin *garbage_bin, + struct log_handle *logger); +void intval_runtime_free(void *intval_runtime); + +int intval_runtime_update(void *intval_runtime, void *intval_schema, + const char *line, int valid_column); +int intval_runtime_commit(void *intval_runtime); + +/* intval runtime scan API */ +/** + * @brief scan intval to get hit group_ids + * + * @retval the num of hit group_id +*/ +int intval_runtime_scan_intval(struct intval_runtime *intval_rt, int thread_id, + const char *data, size_t data_len, + int *group_ids, size_t group_ids_size, + int vtable_id, struct maat_state *state); + +void intval_runtime_scan_hit_inc(struct intval_runtime *intval_rt, int thread_id); +long long intval_runtime_scan_hit_sum(struct intval_runtime *intval_rt, int n_thread); + +#ifdef __cpluscplus +} +#endif + +#endif diff --git a/src/inc_internal/maat_rule.h b/src/inc_internal/maat_rule.h index 34593a6..74a94dd 100644 --- a/src/inc_internal/maat_rule.h +++ b/src/inc_internal/maat_rule.h @@ -121,12 +121,6 @@ struct maat_runtime { uint32_t rule_num; struct maat_garbage_bin *ref_garbage_bin; - - struct maat_kv_store *district_map; - struct maat_kv_store *tmp_district_map; - - unsigned int district_num; - struct log_handle *logger; }; @@ -259,6 +253,7 @@ struct maat_state { struct maat_compile_state *compile_state; }; + size_t parse_accept_tag(const char *value, struct rule_tag **result, struct log_handle *logger); int compare_accept_tag(const char *value, const struct rule_tag *accept_tags, size_t n_accept_tag); diff --git a/src/inc_internal/maat_table.h b/src/inc_internal/maat_table.h index 4b6916c..b9f7fcd 100644 --- a/src/inc_internal/maat_table.h +++ b/src/inc_internal/maat_table.h @@ -21,7 +21,8 @@ extern "C" enum table_type { TABLE_TYPE_INVALID = -1, - TABLE_TYPE_EXPR = 0, + TABLE_TYPE_FLAG = 0, + TABLE_TYPE_EXPR, TABLE_TYPE_EXPR_PLUS, TABLE_TYPE_IP_PLUS, TABLE_TYPE_INTERVAL, @@ -65,7 +66,7 @@ int table_manager_accept_tags_match(struct table_manager *tbl_mgr, const char *t int table_manager_set_scan_district(struct table_manager *tbl_mgr, const char *district_str, size_t district_str_len, int *district_id); - +int table_manager_get_district_id(struct table_manager *tbl_mgr, const char *district); void *table_manager_get_schema(struct table_manager *tbl_mgr, int table_id); void *table_manager_get_runtime(struct table_manager *tbl_mgr, int table_id); diff --git a/src/inc_internal/maat_virtual.h b/src/inc_internal/maat_virtual.h index 8b4214b..b315ce4 100644 --- a/src/inc_internal/maat_virtual.h +++ b/src/inc_internal/maat_virtual.h @@ -34,6 +34,7 @@ void *virtual_schema_new(cJSON *json, struct table_manager *tbl_mgr, void virtual_schema_free(void *virtual_schema); + #ifdef __cpluscplus } #endif diff --git a/src/maat_api.cpp b/src/maat_api.cpp index b7e0ac7..31203b6 100644 --- a/src/maat_api.cpp +++ b/src/maat_api.cpp @@ -31,6 +31,7 @@ #include "maat_ip.h" #include "maat_plugin.h" #include "maat_ip_plugin.h" +#include "maat_virtual.h" #define MODULE_MAAT_API module_name_str("maat.api") @@ -42,6 +43,50 @@ enum district_set_flag { DISTRICT_FLAG_SET }; +enum scan_type maat_table_get_scan_type(enum table_type table_type) +{ + enum scan_type ret = SCAN_TYPE_INVALID; + + switch (table_type) { + case TABLE_TYPE_FLAG: + ret = SCAN_TYPE_FLAG; + break; + case TABLE_TYPE_EXPR: + case TABLE_TYPE_EXPR_PLUS: + case TABLE_TYPE_SIMILARITY: + case TABLE_TYPE_DIGEST: + ret = SCAN_TYPE_STRING; + break; + case TABLE_TYPE_INTERVAL: + case TABLE_TYPE_INTERVAL_PLUS: + ret = SCAN_TYPE_INTERVAL; + break; + case TABLE_TYPE_IP: + case TABLE_TYPE_IP_PLUS: + case TABLE_TYPE_COMPOSITION: + ret = SCAN_TYPE_IP; + break; + case TABLE_TYPE_PLUGIN: + ret = SCAN_TYPE_PLUGIN; + break; + case TABLE_TYPE_IP_PLUGIN: + ret = SCAN_TYPE_IP; + break; + case TABLE_TYPE_FQDN_PLUGIN: + ret = SCAN_TYPE_FQDN_PLUGIN; + break; + case TABLE_TYPE_BOOL_PLUGIN: + ret = SCAN_TYPE_BOOL_PLUGIN; + break; + case TABLE_TYPE_COMPILE: + ret = SCAN_TYPE_NONE; + break; + default: + break; + } + return ret; +} + struct maat_options* maat_options_new(void) { struct maat_options *options = ALLOC(struct maat_options, 1); @@ -849,13 +894,17 @@ int maat_scan_string(struct maat *maat_instance, int table_id, int thread_id, mid = grab_state(state, maat_instance, thread_id); mid->scan_cnt++; - int vtable_id = 0; - //TODO: by luis get virtual_table_id - // struct table_schema *table_schema = table_schema_get_by_scan_type(maat_instance->table_schema_mgr, table_id, - // SCAN_TYPE_STRING, &virtual_table_id); - // if (NULL == table_schema) { - // return MAAT_SCAN_ERR; - // } + int physical_table_id = -1; + enum table_type table_type = table_manager_get_table_type(maat_instance->tbl_mgr, table_id); + if (table_type == TABLE_TYPE_VIRTUAL) { + //find physical table id + physical_table_id = virtual_table_get_physical_table_id(table_id, SCAN_TYPE_STRING); + if (physical_table_id < 0) { + return MAAT_SCAN_ERR; + } + } else { + physical_table_id = table_id; + } if (NULL == maat_instance->maat_rt) { log_error(maat_instance->logger, MODULE_MAAT_API, @@ -866,7 +915,7 @@ int maat_scan_string(struct maat *maat_instance, int table_id, int thread_id, maat_runtime_ref_inc(maat_instance->maat_rt, thread_id); //TODO: is TABLE_TYPE_EXPR_PLUS - enum table_type table_type = table_manager_get_table_type(maat_instance->tbl_mgr, table_id); + enum table_type table_type = table_manager_get_table_type(maat_instance->tbl_mgr, physical_table_id); if ((table_type == TABLE_TYPE_EXPR_PLUS) && (NULL == mid || DISTRICT_FLAG_UNSET == mid->is_set_district)) { maat_instance->scan_err_cnt++; @@ -882,13 +931,15 @@ int maat_scan_string(struct maat *maat_instance, int table_id, int thread_id, } int group_hit_cnt = expr_runtime_scan_string((struct expr_runtime *)expr_rt, thread_id, - data, data_len, group_ids, sizeof(group_ids), + data, data_len, group_ids, + MAX_SCANNER_HIT_GROUP_NUM, vtable_id, mid); if (group_hit_cnt < 0) { return MAAT_SCAN_ERR; } int compile_ret = 0; + int district_id = DISTRICT_ANY; if (group_hit_cnt > 0 || scan_status_should_compile_NOT(mid)) { // come here means group_hit_cnt > 0, at least MAAT_SCAN_HALF_HIT, or MAAT_SCAN_HIT if (group_hit_cnt > 0) { @@ -896,7 +947,7 @@ int maat_scan_string(struct maat *maat_instance, int table_id, int thread_id, } if (group_hit_cnt > 0 && table_type == TABLE_TYPE_EXPR_PLUS) { - + district_id = mid->district_id; } int compile_table_id = -1; @@ -964,8 +1015,6 @@ int maat_state_set_scan_district(struct maat *maat_instance, int ret = table_manager_set_scan_district(maat_instance->tbl_mgr, district, district_len, &(mid->district_id)); - // int map_ret = maat_kv_read_unNull(maat_instance->maat_rt->district_map, - // district, district_len, &(mid->district_id)); if (ret < 0) { mid->district_id = DISTRICT_UNKNOWN; } diff --git a/src/maat_expr.cpp b/src/maat_expr.cpp index 20b9170..7adb95e 100644 --- a/src/maat_expr.cpp +++ b/src/maat_expr.cpp @@ -162,8 +162,6 @@ struct expr_item *expr_item_new(const char *line, struct expr_schema *expr_schem expr_item->group_id = atoi(line + column_offset); table_type = table_manager_get_table_type(expr_schema->tbl_mgr, expr_schema->table_id); - //TODO - #if 1 if (table_type == TABLE_TYPE_EXPR_PLUS) { ret = get_column_pos(line, expr_schema->district_column, &column_offset, &column_len); if (ret < 0) { @@ -181,9 +179,9 @@ struct expr_item *expr_item_new(const char *line, struct expr_schema *expr_schem memcpy(district, (line + column_offset), column_len); assert(strlen(district) > 0); str_unescape(district); - expr_item->district_id = get_district_id() + expr_item->district_id = table_manager_get_district_id(expr_schema->ref_tbl_mgr, district); } - #endif + ret = get_column_pos(line, expr_schema->keywords_column, &column_offset, &column_len); if (ret < 0) { log_error(logger, MODULE_EXPR, @@ -545,6 +543,9 @@ and_expr_t *expr_item_to_expr_rule(struct expr_item *expr_item, expr_rule->patterns[i].pat_len = strlen(sub_key_array[i]); expr_rule->patterns[i].type = expr_type2pattern_type(expr_item->expr_type); } + int *district_tag = ALLOC(int, 1); + *district_tag = expr_item->district_id; + expr_rule->user_tag = district_tag; expr_rule->n_patterns = sub_expr_cnt; return expr_rule; @@ -599,10 +600,7 @@ int expr_runtime_update(void *expr_runtime, void *expr_schema, return -1; } - // TODO: by luis - //int district_id = get_district_id(maat_rt, expr_item->district); - int district_id = -1; - u_para = maat_item_inner_new(expr_item->group_id, item_id, district_id); + u_para = maat_item_inner_new(expr_item->group_id, item_id, expr_item->district_id); item = maat_item_new(item_id, expr_item->group_id, u_para); HASH_ADD_INT(expr_rt->item_hash, item_id, item); @@ -694,9 +692,12 @@ int expr_runtime_scan_string(struct expr_runtime *expr_rt, int thread_id, int *group_ids, size_t group_ids_size, int vtable_id, struct maat_state *state) { - int hit_item_ids[MAX_SCANNER_HIT_ITEM_NUM] = {-1}; size_t n_hit_item = 0; - int ret = adapter_hs_scan(expr_rt->hs, thread_id, data, data_len, hit_item_ids, &n_hit_item); + struct hs_scan_results hit_results[MAX_SCANNER_HIT_ITEM_NUM] = {0}; + + int ret = adapter_hs_scan(expr_rt->hs, thread_id, data, data_len, + hit_item_ids, MAX_SCANNER_HIT_ITEM_NUM, + &n_hit_item); if (ret < 0) { return -1; } @@ -713,8 +714,9 @@ int expr_runtime_scan_string(struct expr_runtime *expr_rt, int thread_id, } size_t group_hit_cnt = 0; - ret = maat_compile_state_update(expr_rt->item_hash, vtable_id, hit_item_ids, n_hit_item, - group_ids, group_ids_size, &group_hit_cnt, state); + ret = maat_compile_state_update(expr_rt->item_hash, vtable_id, hit_item_ids, + n_hit_item, group_ids, group_ids_size, + &group_hit_cnt, state); if (ret < 0) { return -1; } diff --git a/src/maat_flag.cpp b/src/maat_flag.cpp new file mode 100644 index 0000000..3610d92 --- /dev/null +++ b/src/maat_flag.cpp @@ -0,0 +1,71 @@ +/* +********************************************************************************************** +* File: maat_flag.cpp +* Description: +* Authors: Liu WenTan +* Date: 2022-10-31 +* Copyright: (c) 2018-2022 Geedge Networks, Inc. All rights reserved. +*********************************************************************************************** +*/ + +#include "maat_flag.h" + +struct flag_schema { + +}; + +struct flag_runtime { + +}; + +void *flag_schema_new(cJSON *json, struct table_manager *tbl_mgr, + const char *table_name, struct log_handle *logger) +{ + +} + +void flag_schema_free(void *flag_schema) +{ + +} + +void *flag_runtime_new(void *flag_schema, int max_thread_num, + struct maat_garbage_bin *garbage_bin, + struct log_handle *logger) +{ + +} + +void flag_runtime_free(void *flag_runtime) +{ + +} + +int flag_runtime_update(void *flag_runtime, void *flag_schema, + const char *line, int valid_column) +{ + +} + +int flag_runtime_commit(void *flag_runtime) +{ + +} + +int flag_runtime_scan_flag(struct flag_runtime *flag_rt, int thread_id, + const char *data, size_t data_len, + int *group_ids, size_t group_ids_size, + int vtable_id, struct maat_state *state) +{ + +} + +void flag_runtime_scan_hit_inc(struct flag_runtime *flag_rt, int thread_id) +{ + +} + +long long flag_runtime_scan_hit_sum(struct flag_runtime *flag_rt, int n_thread) +{ + +} \ No newline at end of file diff --git a/src/maat_intval.cpp b/src/maat_intval.cpp new file mode 100644 index 0000000..28f069f --- /dev/null +++ b/src/maat_intval.cpp @@ -0,0 +1,72 @@ +/* +********************************************************************************************** +* File: maat_intval.cpp +* Description: +* Authors: Liu WenTan +* Date: 2022-10-31 +* Copyright: (c) 2018-2022 Geedge Networks, Inc. All rights reserved. +*********************************************************************************************** +*/ + +#include "maat_intval.h" + +struct intval_schema { + +}; + +struct intval_runtime { + +}; + +void *intval_schema_new(cJSON *json, struct table_manager *tbl_mgr, + const char *table_name, struct log_handle *logger) +{ + +} + +void _schema_free(void *intval_schema) +{ + +} + +/* intval runtime API */ +void *intval_runtime_new(void *intval_schema, int max_thread_num, + struct maat_garbage_bin *garbage_bin, + struct log_handle *logger) +{ + +} + +void intval_runtime_free(void *intval_runtime) +{ + +} + +int intval_runtime_update(void *intval_runtime, void *intval_schema, + const char *line, int valid_column) +{ + +} + +int intval_runtime_commit(void *intval_runtime) +{ + +} + +int intval_runtime_scan_intval(struct intval_runtime *intval_rt, int thread_id, + const char *data, size_t data_len, + int *group_ids, size_t group_ids_size, + int vtable_id, struct maat_state *state) +{ + +} + +void intval_runtime_scan_hit_inc(struct intval_runtime *intval_rt, int thread_id) +{ + +} + +long long intval_runtime_scan_hit_sum(struct intval_runtime *intval_rt, int n_thread) +{ + +} \ No newline at end of file diff --git a/src/maat_ip.cpp b/src/maat_ip.cpp index 4fbceaf..5757ba8 100644 --- a/src/maat_ip.cpp +++ b/src/maat_ip.cpp @@ -24,11 +24,6 @@ #define MODULE_IP module_name_str("maat.ip") -struct port_range { - uint16_t min_port; - uint16_t max_port; -}; - struct ip_schema { int item_id_column; int group_id_column; @@ -36,11 +31,6 @@ struct ip_schema { int saddr_format_column; int sip1_column; int sip2_column; - int sport_format_column; - int sport1_column; - int sport2_column; - int proto_column; - int direction_column; int table_id; //ugly struct table_manager *tbl_mgr; }; @@ -48,19 +38,11 @@ struct ip_schema { struct ipv4_item_rule { uint32_t min_sip; /* 源地址下界；0表示忽略本字段 */ uint32_t max_sip; /* 源地址上界；0表示固定IP=min_saddr */ - uint16_t min_sport; /* 源端口范围下界；0表示忽略本字段 */ - uint16_t max_sport; /* 源端口范围上界；0表示固定端口=min_sport */ - uint16_t proto; /* 传输层协议，6表示TCP，17表示UDP；0表示忽略本字段 */ - uint16_t direction; /* 方向，0表示双向，1表示单向 */ }; struct ipv6_item_rule { uint32_t min_sip[4]; /* 源地址下界；全0表示忽略本字段 */ uint32_t max_sip[4]; /* 源地址上界；全0表示固定IP=min_saddr */ - uint16_t min_sport; /* 源端口范围下界；0表示忽略本字段 */ - uint16_t max_sport; /* 源端口范围上界；0表示固定端口=min_sport */ - uint16_t proto; /* 传输层协议，6表示TCP，17表示UDP，无限制默认为0 */ - uint16_t direction; /* 方向，0表示双向，1表示单向 */ }; struct ip_item { @@ -145,39 +127,9 @@ void *ip_schema_new(cJSON *json, struct table_manager *tbl_mgr, read_cnt++; } - custom_item = cJSON_GetObjectItem(item, "sport_format"); - if (custom_item != NULL && custom_item->type == cJSON_Number) { - ip_schema->sport_format_column = custom_item->valueint; - read_cnt++; - } - - custom_item = cJSON_GetObjectItem(item, "sport1"); - if (custom_item != NULL && custom_item->type == cJSON_Number) { - ip_schema->sport1_column = custom_item->valueint; - read_cnt++; - } - - custom_item = cJSON_GetObjectItem(item, "sport2"); - if (custom_item != NULL && custom_item->type == cJSON_Number) { - ip_schema->sport2_column = custom_item->valueint; - read_cnt++; - } - - custom_item = cJSON_GetObjectItem(item, "proto"); - if (custom_item != NULL && custom_item->type == cJSON_Number) { - ip_schema->proto_column = custom_item->valueint; - read_cnt++; - } - - custom_item = cJSON_GetObjectItem(item, "direction"); - if (custom_item != NULL && custom_item->type == cJSON_Number) { - ip_schema->direction_column = custom_item->valueint; - read_cnt++; - } - ip_schema->tbl_mgr = tbl_mgr; - if (read_cnt < 12) { + if (read_cnt < 7) { goto error; } @@ -330,38 +282,6 @@ struct ip_item *ip_item_new(const char *line, struct ip_schema *ip_schema, } memcpy(sip2_str, (line + column_offset), column_len); - ret = get_column_pos(line, ip_schema->sport_format_column, &column_offset, &column_len); - if (ret < 0) { - log_error(logger, MODULE_IP, - "ip table(table_id:%d) line:%s has no sport_format", - ip_schema->table_id, line); - goto error; - } - memcpy(sport_format, (line + column_offset), column_len); - if (IP_FORMAT_UNKNOWN == ip_format_str2int(sport_format)) { - log_error(logger, MODULE_IP, - "ip table(table_id:%d) line:%s has invalid sport_format, should be range/mask/CIDR", - ip_schema->table_id, line); - goto error; - } - - ret = get_column_pos(line, ip_schema->sport1_column, &column_offset, &column_len); - if (ret < 0) { - log_error(logger, MODULE_IP, "ip table(table_id:%d) line:%s has no sport1", - ip_schema->table_id, line); - goto error; - } - sport1 = atoi(line + column_offset); - - ret = get_column_pos(line, ip_schema->sport2_column, &column_offset, &column_len); - if (ret < 0) { - log_error(logger, MODULE_IP, - "ip table(table_id:%d) line:%s has no sport2", - ip_schema->table_id, line); - goto error; - } - sport2 = atoi(line + column_offset); - if (IPv4 == ip_item->addr_type) { ret = ip_format2range(ip_item->addr_type, ip_format_str2int(saddr_format), sip1_str, sip2_str, &ip_item->ipv4.min_sip, &ip_item->ipv4.max_sip); @@ -371,34 +291,6 @@ struct ip_item *ip_item_new(const char *line, struct ip_schema *ip_schema, ip_schema->table_id, line); goto error; } - - if(IP_FORMAT_MASK == ip_format_str2int(sport_format)) { - ip_item->ipv4.min_sport = sport1 & sport2; - ip_item->ipv4.max_sport = sport1 | ~sport2; - } else { - ip_item->ipv4.min_sport = sport1; - ip_item->ipv4.max_sport = sport2; - } - - ret = get_column_pos(line, ip_schema->proto_column, &column_offset, &column_len); - if (ret < 0) { - log_error(logger, MODULE_IP, - "ip table(table_id:%d) line:%s has no proto", - ip_schema->table_id, line); - goto error; - } - ip_item->ipv4.proto = atoi(line + column_offset); - protocol = ip_item->ipv4.proto; - - ret = get_column_pos(line, ip_schema->direction_column, &column_offset, &column_len); - if (ret < 0) { - log_error(logger, MODULE_IP, - "ip table(table_id:%d) line:%s has no direction", - ip_schema->table_id, line); - goto error; - } - ip_item->ipv4.direction = atoi(line + column_offset); - direction = ip_item->ipv4.direction; } else { //ipv6 ret = ip_format2range(ip_item->addr_type, ip_format_str2int(saddr_format), sip1_str, sip2_str, @@ -409,46 +301,6 @@ struct ip_item *ip_item_new(const char *line, struct ip_schema *ip_schema, ip_schema->table_id, line); goto error; } - - if(IP_FORMAT_MASK == ip_format_str2int(sport_format)) { - ip_item->ipv6.min_sport = sport1 & sport2; - ip_item->ipv6.max_sport = sport1 | ~sport2; - } else { - ip_item->ipv6.min_sport = sport1; - ip_item->ipv6.max_sport = sport2; - } - - ret = get_column_pos(line, ip_schema->proto_column, &column_offset, &column_len); - if (ret < 0) { - log_error(logger, MODULE_IP, "ip table(table_id:%d) line:%s has no proto", - ip_schema->table_id, line); - goto error; - } - ip_item->ipv6.proto = atoi(line + column_offset); - protocol = ip_item->ipv6.proto; - - ret = get_column_pos(line, ip_schema->direction_column, &column_offset, &column_len); - if (ret < 0) { - log_error(logger, MODULE_IP, "ip table(table_id:%d) line:%s has no direction", - ip_schema->table_id, line); - goto error; - } - ip_item->ipv6.direction = atoi(line + column_offset); - direction = ip_item->ipv6.direction; - } - - if (protocol > 65535 || protocol < 0) { - log_error(logger, MODULE_IP, - "ip table(table_id:%d) line:%s has invalid proto:%d", - ip_schema->table_id, line, protocol); - goto error; - } - - if (direction != 0 && direction != 1) { - log_error(logger, MODULE_IP, - "ip table(table_id:%d) line:%s has invalid direction:%d", - ip_schema->table_id, line, direction); - goto error; } return ip_item; @@ -464,24 +316,18 @@ void ip_item_free(struct ip_item *ip_item) void ip_item_to_ip_rule(struct ip_item *item, struct ip_rule *rule) { - struct port_range *sport_range = ALLOC(struct port_range, 1); if (IPv4 == item->addr_type) { rule->type = IPv4; - sport_range->min_port = item->ipv4.min_sport; - sport_range->max_port = item->ipv4.max_sport; rule->ipv4_rule.start_ip = item->ipv4.min_sip; rule->ipv4_rule.end_ip = item->ipv4.max_sip; } else { rule->type = IPv6; - sport_range->min_port = item->ipv6.min_sport; - sport_range->max_port = item->ipv6.max_sport; memcpy(rule->ipv6_rule.start_ip, item->ipv6.min_sip, sizeof(item->ipv6.min_sip)); memcpy(rule->ipv6_rule.end_ip, item->ipv6.max_sip, sizeof(item->ipv6.max_sip)); } rule->rule_id = item->item_id; - rule->user_tag = sport_range; } struct ex_data_runtime *ip_runtime_get_ex_data_rt(struct ip_runtime *ip_rt) @@ -656,7 +502,7 @@ int ip_runtime_scan_ip(struct ip_runtime *ip_rt, int thread_id, int ip_type, } } - n_hit_item = ip_matcher_match(ip_rt->ip_matcher, &scan_data, scan_results, sizeof(scan_result)); + n_hit_item = ip_matcher_match(ip_rt->ip_matcher, &scan_data, scan_results, MAX_SCANNER_HIT_ITEM_NUM); if (n_hit_item <= 0) { return n_hit_item; } diff --git a/src/maat_table.cpp b/src/maat_table.cpp index 9e14640..c27a53e 100644 --- a/src/maat_table.cpp +++ b/src/maat_table.cpp @@ -54,6 +54,7 @@ struct table_manager { struct maat_kv_store *district_map; struct maat_kv_store *tmp_district_map; + uint32_t district_num; struct maat_garbage_bin *ref_garbage_bin; struct log_handle *logger; @@ -74,6 +75,15 @@ struct table_operations { }; struct table_operations table_ops[TABLE_TYPE_MAX] = { + { + .type = TABLE_TYPE_FLAG, + .new_schema = flag_schema_new, + .free_schema = flag_schema_free, + .new_runtime = flag_runtime_new, + .free_runtime = flag_runtime_free, + .update_runtime = flag_runtime_update, + .commit_runtime = flag_runtime_commit + }, { .type = TABLE_TYPE_EXPR, .new_schema = expr_schema_new, @@ -415,7 +425,6 @@ table_manager_create(const char *table_info_path, const char *accept_tags, tbl_mgr->n_accept_tag = parse_accept_tag(accept_tags, &tbl_mgr->accept_tags, logger); tbl_mgr->logger = logger; tbl_mgr->tablename2id_map = maat_kv_store_new(); - tbl_mgr->district_map = maat_kv_store_new(); tbl_mgr->ref_garbage_bin = garbage_bin; int default_compile_table_id = MAX_TABLE_NUM; @@ -682,11 +691,31 @@ int table_manager_accept_tags_match(struct table_manager *tbl_mgr, const char *t return compare_accept_tag(tags, tbl_mgr->accept_tags, tbl_mgr->n_accept_tag); } -int table_manager_set_scan_district(struct table_manager *tbl_mgr, const char *district_str, - size_t district_str_len, int *district_id) +int table_manager_set_scan_district(struct table_manager *tbl_mgr, const char *district, + size_t district_len, int *district_id) { - return maat_kv_read_unNull(tbl_mgr->district_map, - district_str, district_str_len, district_id); + return maat_kv_read_unNull(tbl_mgr->district_map, district, district_len, district_id); +} + +int table_manager_get_district_id(struct table_manager *tbl_mgr, const char *district) +{ + int district_id = -1; + + int map_ret = maat_kv_read(tbl_mgr->district_map, district, &district_id); + if (map_ret < 0) { + if (NULL == tbl_mgr->tmp_district_map) { + tbl_mgr->tmp_district_map = maat_kv_store_duplicate(tbl_mgr->district_map); + } + + map_ret = maat_kv_read(tbl_mgr->tmp_district_map, district, &district_id); + if (map_ret < 0) { + district_id = tbl_mgr->district_num; + maat_kv_register(tbl_mgr->tmp_district_map, district, district_id); + tbl_mgr->district_num++; + } + } + + return district_id; } void *table_manager_get_runtime(struct table_manager *tbl_mgr, int table_id) diff --git a/test/table_info.conf b/test/table_info.conf index 5aab16f..6eb2ce8 100644 --- a/test/table_info.conf +++ b/test/table_info.conf @@ -1,6 +1,6 @@ [ { - "table_id":1, + "table_id":0, "table_name":"COMPILE", "table_type":"compile", "valid_column":8, @@ -17,7 +17,7 @@ } }, { - "table_id":2, + "table_id":1, "table_name":"GROUP2COMPILE", "table_type":"group2compile", "associated_compile_table_id":1, @@ -31,38 +31,7 @@ } }, { - "table_id":3, - "table_name":"COMPILE_2", - "table_type":"compile", - "valid_column":8, - "custom": { - "compile_id":1, - "service_id":2, - "action":3, - "do_blacklist":4, - "do_log":5, - "tags":6, - "user_region":7, - "clause_num":9, - "evaluation_order":10 - } - }, - { - "table_id":4, - "table_name":"GROUP2COMPILE_2", - "table_type":"group2compile", - "associated_compile_table_id":3, - "valid_column":3, - "custom": { - "group_id":1, - "compile_id":2, - "not_flag":4, - "virtual_table_name":5, - "clause_index":6 - } - }, - { - "table_id":5, + "table_id":2, "table_name":"GROUP2GROUP", "table_type":"group2group", "valid_column":3, @@ -72,7 +41,7 @@ } }, { - "table_id":6, + "table_id":3, "table_name":"HTTP_URL", "table_type":"expr", "valid_column":7, @@ -87,10 +56,248 @@ } }, { - "table_id":7, - "table_name":"IP_PLUGIN_TABLE", - "table_type":"ip_plugin", + "table_id":3, + "table_name":"HTTP_HOST", + "table_type":"expr", + "valid_column":7, + "custom": { + "scan_mode":"block", + "item_id":1, + "group_id":2, + "keywords":3, + "expr_type":4, + "match_method":5, + "is_hexbin":6 + } + }, + { + "table_id":4, + "table_name":"KEYWORDS_TABLE", + "table_type":"expr", + "valid_column":7, + "custom": { + "scan_mode":"block", + "item_id":1, + "group_id":2, + "keywords":3, + "expr_type":4, + "match_method":5, + "is_hexbin":6 + } + }, + { + "table_id":6, + "table_name":"CONTENT_SIZE", + "table_type":"intval", "valid_column":5, + "custom": { + "item_id":1, + "group_id":2, + "low_bound":3, + "up_bound":4 + } + }, + { + "table_id":7, + "table_name":"QD_ENTRY_INFO", + "table_type":"plugin", + "valid_column":4, + "custom": { + "key":1, + "tag":3 + } + }, + { + "table_id":9, + "table_name":"HTTP_SIGNATURE", + "table_type":"expr_plus", + "valid_column":8, + "custom": { + "item_id":1, + "group_id":2, + "district":3, + "keywords":4, + "expr_type":5, + "match_method":6, + "db_hexbin":7 + } + }, + { + "table_id":11, + "table_name":"IMAGE_FP", + "table_type":"expr", + "valid_column":7, + "custom": { + "scan_mode":"block", + "item_id":1, + "group_id":2, + "keywords":3, + "expr_type":4, + "match_method":5, + "is_hexbin":6 + } + }, + { + "table_id":12, + "table_name":"TEST_EFFECTIVE_RANGE_TABLE", + "table_type":"plugin", + "valid_column":4, + "custom": { + "key":1, + "tag":5 + } + }, + { + "table_id":13, + "table_name":"TEST_FOREIGN_KEY", + "table_type":"plugin", + "valid_column":4, + "custom": { + "key":2, + "tag":3, + "foreign": [6,8] + } + }, + { + "table_id":14, + "table_name":"COMPILE_ALIAS", + "table_type":"compile", + "valid_column":4, + "custom": { + "compile_id":1, + "service_id":2, + "action":3, + "do_blacklist":4, + "do_log":5, + "tags":6, + "user_region":7, + "clause_num":9, + "evaluation_order":10 + } + }, + { + "table_id":15, + "table_name":"TEST_PLUGIN_EXDATA_TABLE", + "table_type":"plugin", + "valid_column":4, + "custom": { + "key":2, + "tag":5, + "estimate_size": 1024 + } + }, + { + "table_id":16, + "table_name":"IR_INTERCEPT_IP", + "table_type":"plugin", + "valid_column":14, + "custom": { + "key":2, + "tag":18 + } + }, + { + "table_id":17, + "table_name":"APP_PAYLOAD", + "table_type":"expr_plus", + "valid_column":8, + "custom": { + "scan_mode":"block", + "item_id":1, + "group_id":2, + "district":3, + "keywords":4, + "expr_type":5, + "match_method":6, + "is_hexbin":7 + + } + }, + { + "table_id":18, + "table_name":"TROJAN_PAYLOAD", + "table_type":"expr", + "valid_column":7, + "custom": { + "scan_mode":"block", + "item_id":1, + "group_id":2, + "keywords":3, + "expr_type":4, + "match_method":5, + "is_hexbin":6 + + } + }, + { + "table_id":19, + "table_name":"MAIL_ADDR", + "table_type":"expr", + "valid_column":7, + "custom": { + "scan_mode":"block", + "item_id":1, + "group_id":2, + "keywords":3, + "expr_type":4, + "match_method":5, + "is_hexbin":6 + } + }, + { + "table_id":20, + "table_name":"IP_PLUS_CONFIG", + "table_type":"ip_plus", + "valid_column":7, + "custom": { + "item_id":1, + "group_id":2, + "addr_type":3, + "saddr_format":4, + "sip1":5, + "sip2":6 + } + }, + { + "table_id":21, + "table_name":"HTTP_RESPONSE_KEYWORDS", + "table_type":"virtual", + "physical_table": ["KEYWORDS_TABLE"] + }, + { + "table_id":22, + "table_name":"HTTP_REQUEST_HEADER", + "table_type":"virtual", + "physical_table": ["HTTP_SIGNATURE"] + }, + { + "table_id":23, + "table_name":"HTTP_RESPONSE_HEADER", + "table_type":"virtual", + "physical_table": ["HTTP_SIGNATURE"] + }, + { + "table_id":24, + "table_name":"VIRTUAL_IP_PLUS_TABLE", + "table_type":"virtual", + "physical_table": ["IP_PLUS_CONFIG"] + }, + { + "table_id":24, + "table_name":"VIRTUAL_IP_PLUS_SOURCE", + "table_type":"virtual", + "physical_table": ["IP_PLUS_CONFIG"] + }, + { + "table_id":24, + "table_name":"VIRTUAL_IP_PLUS_DESTINATION", + "table_type":"virtual", + "physical_table": ["IP_PLUS_CONFIG"] + }, + { + "table_id":29, + "table_name":"TEST_IP_PLUGIN_WITH_EXDATA", + "table_type":"ip_plugin", "custom": { "item_id":1, "ip_type":2, @@ -99,40 +306,142 @@ } }, { - "table_id":8, - "table_name":"PLUGIN_TABLE", - "table_type":"plugin", - "valid_column":4, + "table_id":30, + "table_name":"AS_NUMBER", + "table_type":"expr", + "valid_column":7, "custom": { + "scan_mode":"block", "item_id":1, - "key":2, - "tag":3, - "foreign":[6,8,10] + "group_id":2, + "keywords":3, + "expr_type":4, + "match_method":5, + "is_hexbin":6 } }, { - "table_id":9, - "table_name":"IP_PLUS_CONFIG", - "table_type":"ip_plus", - "valid_column":18, + "table_id":31, + "table_name":"SOURCE_IP_ASN", + "table_type":"virtual", + "physical_table":["AS_NUMBER"] + }, + { + "table_id":32, + "table_name":"DESTINATION_IP_ASN", + "table_type":"virtual", + "physical_table":["AS_NUMBER"] + }, + { + "table_id":33, + "table_name":"GeoLocation", + "table_type":"expr", + "valid_column":7, + "custom": { + "scan_mode":"block", + "item_id":1, + "group_id":2, + "keywords":3, + "expr_type":4, + "match_method":5, + "is_hexbin":6 + } + }, + { + "table_id":34, + "table_name":"SOURCE_IP_GEO", + "table_type":"virtual", + "physical_table":["GeoLocation"] + }, + { + "table_id":35, + "table_name":"INTERGER_PLUS", + "table_type":"intval_plus", + "valid_column":6, "custom": { "item_id":1, "group_id":2, - "addr_type":3, - "saddr_format":4, - "sip1":5, - "sip2":6, - "sport_format":7, - "sport1":8, - "sport2":9, - "proto":16, - "direction":17 + "district":3, + "low_bound":4, + "up_bound":5 } }, { - "table_id":10, - "table_name":"VIRTUAL_IP_PLUS_TABLE", + "table_id":36, + "table_name":"TEST_FQDN_PLUGIN_WITH_EXDATA", + "table_type":"fqdn_plugin", + "valid_column":5, + "custom": { + "row_id":1, + "is_suffix_match":2, + "fqdn":3 + } + }, + { + "table_id":37, + "table_name":"VIRTUAL_SSL_SNI", "table_type":"virtual", - "physical_table": ["IP_PLUS_CONFIG"] + "physical_table": ["KEYWORDS_TABLE", "INTERGER_PLUS"] + }, + { + "table_id":38, + "table_name":"APP_ID", + "table_type":"intval", + "valid_column":5, + "custom": { + "item_id":1, + "group_id":2, + "low_bound":3, + "up_bound":4 + } + }, + { + "table_id":39, + "table_name":"EMPTY_KEYWORD", + "table_type":"expr", + "valid_column":7, + "custom": { + "scan_mode":"block", + "item_id":1, + "group_id":2, + "keywords":3, + "expr_type":4, + "match_method":5, + "is_hexbin":6 + } + }, + { + "table_id":40, + "table_name":"EMPTY_INTERGER", + "table_type":"intval", + "valid_column":5, + "custom": { + "item_id":1, + "group_id":2, + "low_bound":3, + "up_bound":4 + } + }, + { + "table_id":41, + "table_name":"TEST_BOOL_PLUGIN_WITH_EXDATA", + "table_type":"bool_plugin", + "valid_column":4, + "custom": { + "item_id":1, + "bool_expr":2 + } + }, + { + "table_id":42, + "table_name":"FLAG_CONFIG", + "table_type":"flag", + "valid_column":5, + "custom": { + "item_id":1, + "group_id":2, + "flag":3, + "flag_mask":4 + } } ] \ No newline at end of file diff --git a/vendor/flag_matcher.tar.gz b/vendor/flag_matcher.tar.gz new file mode 100644 index 0000000..8f21dd5 Binary files /dev/null and b/vendor/flag_matcher.tar.gz differ diff --git a/vendor/interval_matcher.tar.gz b/vendor/interval_matcher.tar.gz new file mode 100644 index 0000000..b6eb22c Binary files /dev/null and b/vendor/interval_matcher.tar.gz differ