diff --git a/include/maat.h b/include/maat.h index a22d275..733a424 100644 --- a/include/maat.h +++ b/include/maat.h @@ -56,6 +56,20 @@ struct ip_addr { }; }; +struct ipv4_tuple { + unsigned int sip; /* network order */ + unsigned int dip; /* network order */ + unsigned short sport; /* network order */ + unsigned short dport; /* network order */ +}; + +struct ipv6_tuple { + unsigned int sip[4] ; /* network order */ + unsigned int dip[4] ; /* network order */ + unsigned short sport; /* network order */ + unsigned short dport; /* network order */ +}; + typedef void maat_start_callback_t(int update_type, void *u_param); typedef void maat_update_callback_t(int table_id, const char *table_line, void *u_para); typedef void maat_finish_callback_t(void *u_para); @@ -86,7 +100,7 @@ struct maat *maat_new(struct maat_options *opts, const char *table_info_path); void maat_free(struct maat *instance); /* maat table API */ -int maat_table_get_id(struct maat *instance, const char *table_name); +int maat_get_table_id(struct maat *instance, const char *table_name); /* return 0 if success, otherwise return -1 */ int maat_table_callback_register(struct maat *instance, int table_id, @@ -138,17 +152,27 @@ int maat_scan_flag(struct maat *instance, int table_id, int thread_id, long long flag, long long *results, size_t n_result, size_t *n_hit_result, struct maat_state **state); int maat_scan_integer(struct maat *instance, int table_id, int thread_id, - unsigned int intval, long long *results, size_t n_result, + long long integer, long long *results, size_t n_result, size_t *n_hit_result, struct maat_state **state); int maat_scan_ipv4(struct maat *instance, int table_id, int thread_id, uint32_t ip_addr, long long *results, size_t n_result, size_t *n_hit_result, struct maat_state **state); +int maat_scan_ipv4_tuple4(struct maat *instance, int table_id, int thread_id, + const struct ipv4_tuple *tuple, long long *results, + size_t n_result, size_t *n_hit_result, + struct maat_state **state); + int maat_scan_ipv6(struct maat *instance, int table_id, int thread_id, uint8_t *ip_addr, long long *results, size_t n_result, size_t *n_hit_result, struct maat_state **state); +int maat_scan_ipv6_tuple4(struct maat *instance, int table_id, int thread_id, + const struct ipv6_tuple *tuple, long long *results, + size_t n_result, size_t *n_hit_result, + struct maat_state **state); + int maat_scan_string(struct maat *instance, int table_id, int thread_id, const char *data, size_t data_len, long long *results, size_t n_result, size_t *n_hit_result, diff --git a/src/inc_internal/maat_table.h b/src/inc_internal/maat_table.h index 57aaba1..11f2bbb 100644 --- a/src/inc_internal/maat_table.h +++ b/src/inc_internal/maat_table.h @@ -25,13 +25,13 @@ extern "C" enum table_type { TABLE_TYPE_INVALID = -1, TABLE_TYPE_FLAG = 0, + TABLE_TYPE_FLAG_PLUS, TABLE_TYPE_EXPR, TABLE_TYPE_EXPR_PLUS, TABLE_TYPE_IP_PLUS, + TABLE_TYPE_PORT, TABLE_TYPE_INTERVAL, TABLE_TYPE_INTERVAL_PLUS, - TABLE_TYPE_DIGEST, - TABLE_TYPE_SIMILARITY, TABLE_TYPE_CONJUNCTION, TABLE_TYPE_PLUGIN, TABLE_TYPE_IP_PLUGIN, @@ -39,6 +39,7 @@ enum table_type { TABLE_TYPE_BOOL_PLUGIN, //above are physical table TABLE_TYPE_VIRTUAL, + TABLE_TYPE_COMPOSITION, TABLE_TYPE_COMPILE, TABLE_TYPE_GROUP2GROUP, TABLE_TYPE_GROUP2COMPILE, diff --git a/src/json2iris.c b/src/json2iris.c index 8f60cf6..ed55387 100644 --- a/src/json2iris.c +++ b/src/json2iris.c @@ -139,7 +139,9 @@ int set_iris_descriptor(const char *json_file, cJSON *json, maat_kv_register(iris_cfg->str2int_map, "no", 0); maat_kv_register(iris_cfg->str2int_map, "flag", TABLE_TYPE_FLAG); + maat_kv_register(iris_cfg->str2int_map, "flag_plus", TABLE_TYPE_FLAG_PLUS); maat_kv_register(iris_cfg->str2int_map, "ip_plus", TABLE_TYPE_IP_PLUS); + maat_kv_register(iris_cfg->str2int_map, "port", TABLE_TYPE_PORT); maat_kv_register(iris_cfg->str2int_map, "string", TABLE_TYPE_EXPR); maat_kv_register(iris_cfg->str2int_map, "expr", TABLE_TYPE_EXPR); maat_kv_register(iris_cfg->str2int_map, "expr_plus", TABLE_TYPE_EXPR_PLUS); @@ -147,8 +149,6 @@ int set_iris_descriptor(const char *json_file, cJSON *json, maat_kv_register(iris_cfg->str2int_map, "interval", TABLE_TYPE_INTERVAL); maat_kv_register(iris_cfg->str2int_map, "intval_plus", TABLE_TYPE_INTERVAL_PLUS); maat_kv_register(iris_cfg->str2int_map, "interval_plus", TABLE_TYPE_INTERVAL_PLUS); - maat_kv_register(iris_cfg->str2int_map, "digest", TABLE_TYPE_DIGEST); - maat_kv_register(iris_cfg->str2int_map, "similar", TABLE_TYPE_SIMILARITY); maat_kv_register(iris_cfg->str2int_map, "ipv4", 4); maat_kv_register(iris_cfg->str2int_map, "ipv6", 6); @@ -423,6 +423,12 @@ int write_flag_line(cJSON *region_json, struct iris_description *p_iris, json_cmd[cmd_cnt].json_type = cJSON_Number; cmd_cnt++; + if (table->table_type==TABLE_TYPE_FLAG_PLUS) { + json_cmd[cmd_cnt].json_string = "district"; + json_cmd[cmd_cnt].json_type = cJSON_String; + cmd_cnt++; + } + json_cmd[cmd_cnt].json_string = "flag"; json_cmd[cmd_cnt].json_type = cJSON_Number; cmd_cnt++; @@ -637,72 +643,6 @@ int write_intval_line(cJSON *region_json, struct iris_description *p_iris, json_cmd, cmd_cnt, table, logger); } -int write_digest_line(cJSON *region_json, struct iris_description *p_iris, - struct iris_table *table, struct log_handle *logger) -{ - struct translate_command json_cmd[MAX_COLUMN_NUM]; - int cmd_cnt = 0; - memset(json_cmd, 0, sizeof(json_cmd)); - - json_cmd[cmd_cnt].json_string = "region_id"; - json_cmd[cmd_cnt].json_type = cJSON_Number; - cmd_cnt++; - - json_cmd[cmd_cnt].json_string = "group_id"; - json_cmd[cmd_cnt].json_type = cJSON_Number; - cmd_cnt++; - - json_cmd[cmd_cnt].json_string = "raw_len"; - json_cmd[cmd_cnt].json_type = cJSON_Number; - cmd_cnt++; - - json_cmd[cmd_cnt].json_string = "digest"; - json_cmd[cmd_cnt].json_type = cJSON_String; - cmd_cnt++; - - json_cmd[cmd_cnt].json_string = "cfds_level"; - json_cmd[cmd_cnt].json_type = cJSON_Number; - cmd_cnt++; - - json_cmd[cmd_cnt].json_string = "is_valid"; - json_cmd[cmd_cnt].json_type = cJSON_Number; - cmd_cnt++; - - return direct_write_rule(region_json, p_iris->str2int_map, - json_cmd, cmd_cnt, table, logger); -} - -int write_similar_line(cJSON *region_json, struct iris_description *p_iris, - struct iris_table *table, struct log_handle *logger) -{ - struct translate_command json_cmd[MAX_COLUMN_NUM]; - int cmd_cnt = 0; - memset(json_cmd, 0, sizeof(json_cmd)); - - json_cmd[cmd_cnt].json_string = "region_id"; - json_cmd[cmd_cnt].json_type = cJSON_Number; - cmd_cnt++; - - json_cmd[cmd_cnt].json_string = "group_id"; - json_cmd[cmd_cnt].json_type = cJSON_Number; - cmd_cnt++; - - json_cmd[cmd_cnt].json_string = "target"; - json_cmd[cmd_cnt].json_type = cJSON_String; - cmd_cnt++; - - json_cmd[cmd_cnt].json_string = "threshold"; - json_cmd[cmd_cnt].json_type = cJSON_Number; - cmd_cnt++; - - json_cmd[cmd_cnt].json_string = "is_valid"; - json_cmd[cmd_cnt].json_type = cJSON_Number; - cmd_cnt++; - - return direct_write_rule(region_json, p_iris->str2int_map, - json_cmd, cmd_cnt, table, logger); -} - int write_region_rule(cJSON *region_json, int compile_id, int group_id, struct iris_description *p_iris, struct log_handle *logger) { @@ -750,6 +690,7 @@ int write_region_rule(cJSON *region_json, int compile_id, int group_id, switch(table_type) { case TABLE_TYPE_FLAG: + case TABLE_TYPE_FLAG_PLUS: ret = write_flag_line(table_content, p_iris, table_info, logger); break; case TABLE_TYPE_EXPR: @@ -763,12 +704,6 @@ int write_region_rule(cJSON *region_json, int compile_id, int group_id, case TABLE_TYPE_INTERVAL_PLUS: ret = write_intval_line(table_content, p_iris, table_info, logger); break; - case TABLE_TYPE_DIGEST: - ret = write_digest_line(table_content, p_iris, table_info, logger); - break; - case TABLE_TYPE_SIMILARITY: - ret = write_similar_line(table_content, p_iris, table_info, logger); - break; default: assert(0); break; @@ -778,10 +713,16 @@ int write_region_rule(cJSON *region_json, int compile_id, int group_id, int write_group2compile_line(int group_id, int compile_id, int group_not_flag, int clause_index, const char *vtable, - struct iris_description *p_iris) + struct iris_description *p_iris, + struct iris_table *g2c_table) { char buff[4096] = {0}; - struct iris_table *table = p_iris->group2compile_table; + struct iris_table *table = NULL; + if (g2c_table != NULL) { + table = g2c_table; + } else { + table = p_iris->group2compile_table; + } snprintf(buff, sizeof(buff), "%d\t%d\t1\t%d\t%s\t%d\n", group_id, compile_id, group_not_flag, vtable, clause_index); @@ -818,6 +759,7 @@ int write_group_rule(cJSON *group_json, int parent_id, const char *str_parent_type[2] = {"compile", "group"}; const char *group_name = NULL; const char *virtual_table = NULL; + struct iris_table *g2c_table = NULL; cJSON *item = cJSON_GetObjectItem(group_json, "group_name"); if (NULL == item || item->type != cJSON_String) { @@ -847,6 +789,12 @@ int write_group_rule(cJSON *group_json, int parent_id, } else { clause_index = item->valueint; } + + item = cJSON_GetObjectItem(group_json, "g2c_table_name"); + if (item != NULL && item->type == cJSON_String) { + g2c_table = query_table_info(p_iris, item->valuestring, + TABLE_TYPE_GROUP2COMPILE); + } } else { group_not_flag = 0; } @@ -892,7 +840,7 @@ int write_group_rule(cJSON *group_json, int parent_id, if (parent_type == PARENT_TYPE_COMPILE) { ret = write_group2compile_line(group_info->group_id, parent_id, group_not_flag, - clause_index, virtual_table, p_iris); + clause_index, virtual_table, p_iris, g2c_table); } else { ret = write_group2group_line(group_info->group_id, parent_id, p_iris); } @@ -998,7 +946,7 @@ int write_compile_line(cJSON *compile, struct iris_description *p_iris, cmd_cnt++; struct iris_table *table_info = NULL; - item = cJSON_GetObjectItem(compile,"table_name"); + item = cJSON_GetObjectItem(compile,"compile_table_name"); if (NULL == item || item->type != cJSON_String) { table_info = p_iris->compile_table; } else { diff --git a/src/maat_api.c b/src/maat_api.c index 988ab92..b26b8fd 100644 --- a/src/maat_api.c +++ b/src/maat_api.c @@ -61,6 +61,7 @@ enum scan_type maat_table_get_scan_type(enum table_type table_type) switch (table_type) { case TABLE_TYPE_FLAG: + case TABLE_TYPE_FLAG_PLUS: ret = SCAN_TYPE_FLAG; break; case TABLE_TYPE_EXPR: @@ -386,7 +387,7 @@ void maat_free(struct maat *maat_instance) pthread_join(maat_instance->cfg_mon_thread, &ret); } -int maat_table_get_id(struct maat *maat_instance, const char *table_name) +int maat_get_table_id(struct maat *maat_instance, const char *table_name) { int table_id = -1; @@ -889,7 +890,12 @@ int flag_scan_hit_group_count(struct table_manager *tbl_mgr, int thread_id, long for (int i = 0; i < physical_table_cnt; i++) { enum table_type table_type = table_manager_get_table_type(tbl_mgr, physical_table_ids[i]); - if (table_type != TABLE_TYPE_FLAG) { + if ((table_type == TABLE_TYPE_FLAG_PLUS) && + (NULL == mid || DISTRICT_FLAG_UNSET == mid->is_set_district)) { + return -1; + } + + if (table_type != TABLE_TYPE_FLAG && table_type != TABLE_TYPE_FLAG_PLUS) { continue; } @@ -913,7 +919,7 @@ int flag_scan_hit_group_count(struct table_manager *tbl_mgr, int thread_id, long return sum_hit_group_cnt; } -int interval_scan_hit_group_count(struct table_manager *tbl_mgr, int thread_id, long long intval, +int interval_scan_hit_group_count(struct table_manager *tbl_mgr, int thread_id, long long integer, int physical_table_ids[], int physical_table_cnt, int vtable_id, struct maat_state *mid) { @@ -924,7 +930,7 @@ int interval_scan_hit_group_count(struct table_manager *tbl_mgr, int thread_id, if ((table_type == TABLE_TYPE_INTERVAL_PLUS) && (NULL == mid || DISTRICT_FLAG_UNSET == mid->is_set_district)) { //maat_instance->scan_err_cnt++; - return MAAT_SCAN_ERR; + return -1; } if (table_type != TABLE_TYPE_INTERVAL && table_type != TABLE_TYPE_INTERVAL_PLUS) { @@ -933,13 +939,13 @@ int interval_scan_hit_group_count(struct table_manager *tbl_mgr, int thread_id, void *interval_rt = table_manager_get_runtime(tbl_mgr, physical_table_ids[i]); if (NULL == interval_rt) { - return MAAT_SCAN_ERR; + return -1; } int group_hit_cnt = interval_runtime_scan((struct interval_runtime *)interval_rt, - thread_id, intval, vtable_id, mid); + thread_id, integer, vtable_id, mid); if (group_hit_cnt < 0) { - return MAAT_SCAN_ERR; + return -1; } if (group_hit_cnt > 0) { @@ -965,13 +971,13 @@ int ipv4_scan_hit_group_count(struct table_manager *tbl_mgr, int thread_id, uint void *ip_rt = table_manager_get_runtime(tbl_mgr, physical_table_ids[i]); if (NULL == ip_rt) { - return MAAT_SCAN_ERR; + return -1; } int group_hit_cnt = ip_runtime_scan((struct ip_runtime *)ip_rt, thread_id, IPv4, (uint8_t *)&ip_addr, vtable_id, mid); if (group_hit_cnt < 0) { - return MAAT_SCAN_ERR; + return -1; } if (group_hit_cnt > 0) { @@ -997,13 +1003,13 @@ int ipv6_scan_hit_group_count(struct table_manager *tbl_mgr, int thread_id, uint void *ip_rt = table_manager_get_runtime(tbl_mgr, physical_table_ids[i]); if (NULL == ip_rt) { - return MAAT_SCAN_ERR; + return -1; } int group_hit_cnt = ip_runtime_scan((struct ip_runtime *)ip_rt, thread_id, IPv6, ip_addr, vtable_id, mid); if (group_hit_cnt < 0) { - return MAAT_SCAN_ERR; + return -1; } if (group_hit_cnt > 0) { @@ -1104,7 +1110,7 @@ size_t group_to_compile(struct maat *maat_instance, long long *results, size_t n compile_table_cnt = 1; } else { for (size_t i = 0; i < mid->n_compile_table; i++) { - compile_table_id[i] = maat_table_get_id(maat_instance, mid->compile_tables[i]); + compile_table_id[i] = maat_get_table_id(maat_instance, mid->compile_tables[i]); } compile_table_cnt = mid->n_compile_table; } @@ -1158,7 +1164,7 @@ int maat_scan_flag(struct maat *maat_instance, int table_id, int thread_id, enum table_type table_type = TABLE_TYPE_INVALID; if (0 == vtable_id) { table_type = table_manager_get_table_type(maat_instance->tbl_mgr, physical_table_ids[0]); - if (table_type != TABLE_TYPE_FLAG) { + if (table_type != TABLE_TYPE_FLAG && table_type != TABLE_TYPE_FLAG_PLUS) { return MAAT_SCAN_ERR; } } @@ -1200,7 +1206,7 @@ int maat_scan_flag(struct maat *maat_instance, int table_id, int thread_id, } int maat_scan_integer(struct maat *maat_instance, int table_id, int thread_id, - unsigned int intval, long long *results, size_t n_result, + long long integer, long long *results, size_t n_result, size_t *n_hit_result, struct maat_state **state) { if ((NULL == maat_instance) || table_id < 0 || table_id >= MAX_TABLE_NUM @@ -1242,7 +1248,7 @@ int maat_scan_integer(struct maat *maat_instance, int table_id, int thread_id, maat_runtime_ref_inc(maat_instance->maat_rt, thread_id); alignment_int64_array_add(maat_instance->thread_call_cnt, thread_id, 1); - int hit_group_cnt = interval_scan_hit_group_count(maat_instance->tbl_mgr, thread_id, intval, + int hit_group_cnt = interval_scan_hit_group_count(maat_instance->tbl_mgr, thread_id, integer, physical_table_ids, physical_table_cnt, vtable_id, mid); if (hit_group_cnt < 0) { @@ -1351,6 +1357,13 @@ int maat_scan_ipv4(struct maat *maat_instance, int table_id, int thread_id, return MAAT_SCAN_OK; } +int maat_scan_ipv4_tuple4(struct maat *instance, int table_id, int thread_id, + const struct ipv4_tuple *tuple4, long long *results, + size_t n_result, size_t *n_hit_result, + struct maat_state **state) +{ + return MAAT_SCAN_OK; +} int maat_scan_ipv6(struct maat *maat_instance, int table_id, int thread_id, uint8_t *ip_addr, long long *results, size_t n_result, @@ -1428,6 +1441,14 @@ int maat_scan_ipv6(struct maat *maat_instance, int table_id, int thread_id, return MAAT_SCAN_OK; } +int maat_scan_ipv6_tuple4(struct maat *instance, int table_id, int thread_id, + const struct ipv6_tuple *tuple, long long *results, + size_t n_result, size_t *n_hit_result, + struct maat_state **state) +{ + return MAAT_SCAN_OK; +} + int maat_scan_string(struct maat *maat_instance, int table_id, int thread_id, const char *data, size_t data_len, long long *results, size_t n_result, size_t *n_hit_result, struct maat_state **state) @@ -1691,7 +1712,7 @@ size_t maat_get_hit_paths(struct maat *maat_instance, struct maat_state *state, compile_table_cnt = 1; } else { for (size_t i = 0; i < state->n_compile_table; i++) { - compile_table_ids[i] = maat_table_get_id(maat_instance, state->compile_tables[i]); + compile_table_ids[i] = maat_get_table_id(maat_instance, state->compile_tables[i]); } compile_table_cnt = state->n_compile_table; } diff --git a/src/maat_command.c b/src/maat_command.c index 18b44b1..15565f8 100644 --- a/src/maat_command.c +++ b/src/maat_command.c @@ -163,9 +163,16 @@ int maat_cmd_get_valid_flag_offset(const char *line, enum table_type table_type, int column_seq = 0; switch (table_type) { + case TABLE_TYPE_FLAG: + case TABLE_TYPE_FLAG_PLUS: //TODO: + column_seq = 5; + break; case TABLE_TYPE_EXPR: column_seq = 7; break; + case TABLE_TYPE_EXPR_PLUS: + column_seq = 8; + break; case TABLE_TYPE_IP_PLUS: column_seq = 18; break; @@ -188,15 +195,6 @@ int maat_cmd_get_valid_flag_offset(const char *line, enum table_type table_type, case TABLE_TYPE_INTERVAL_PLUS: column_seq = 6; break; - case TABLE_TYPE_DIGEST: - column_seq = 6; - break; - case TABLE_TYPE_SIMILARITY: - column_seq = 5; - break; - case TABLE_TYPE_EXPR_PLUS: - column_seq = 8; - break; case TABLE_TYPE_GROUP2COMPILE: case TABLE_TYPE_GROUP2GROUP: column_seq = 3; diff --git a/src/maat_compile.c b/src/maat_compile.c index aace378..163700d 100644 --- a/src/maat_compile.c +++ b/src/maat_compile.c @@ -1807,7 +1807,7 @@ int maat_compile_state_update(struct maat_item *item_hash, int vtable_id, compile_table_cnt = 1; } else { for (size_t i = 0; i < state->n_compile_table; i++) { - compile_table_ids[i] = maat_table_get_id(state->maat_instance, state->compile_tables[i]); + compile_table_ids[i] = maat_get_table_id(state->maat_instance, state->compile_tables[i]); } compile_table_cnt = state->n_compile_table; } diff --git a/src/maat_flag.c b/src/maat_flag.c index 3751607..e9d0865 100644 --- a/src/maat_flag.c +++ b/src/maat_flag.c @@ -27,6 +27,7 @@ struct flag_schema { int item_id_column; int group_id_column; + int district_column; int flag_column; int flag_mask_column; int table_id; @@ -36,6 +37,7 @@ struct flag_schema { struct flag_item { long long item_id; long long group_id; + int district_id; long long flag; long long flag_mask; }; @@ -88,6 +90,12 @@ void *flag_schema_new(cJSON *json, struct table_manager *tbl_mgr, read_cnt++; } + custom_item = cJSON_GetObjectItem(item, "district"); + if (custom_item != NULL && custom_item->type == cJSON_Number) { + schema->district_column = custom_item->valueint; + read_cnt++; + } + custom_item = cJSON_GetObjectItem(item, "flag"); if (custom_item != NULL && custom_item->type == cJSON_Number) { schema->flag_column = custom_item->valueint; @@ -205,53 +213,77 @@ int flag_runtime_update_row(struct flag_runtime *flag_rt, char *key, size_t key_ return 0; } -struct flag_item *flag_item_new(const char *line, struct flag_schema *flag_schema, +struct flag_item *flag_item_new(const char *line, struct flag_schema *schema, struct log_handle *logger) { size_t column_offset = 0; size_t column_len = 0; - struct flag_item *flag_item = ALLOC(struct flag_item, 1); + enum table_type table_type = TABLE_TYPE_INVALID; + struct flag_item *item = ALLOC(struct flag_item, 1); - int ret = get_column_pos(line, flag_schema->item_id_column, &column_offset, &column_len); + int ret = get_column_pos(line, schema->item_id_column, &column_offset, &column_len); if (ret < 0) { log_error(logger, MODULE_FLAG, "flag table(table_id:%d) line:%s has no item_id", - flag_schema->table_id, line); + schema->table_id, line); goto error; } - flag_item->item_id = atoll(line + column_offset); + item->item_id = atoll(line + column_offset); - ret = get_column_pos(line, flag_schema->group_id_column, &column_offset, &column_len); + ret = get_column_pos(line, schema->group_id_column, &column_offset, &column_len); if (ret < 0) { log_error(logger, MODULE_FLAG, "flag table(table_id:%d) line:%s has no group_id", - flag_schema->table_id, line); + schema->table_id, line); goto error; } - flag_item->group_id = atoll(line + column_offset); + item->group_id = atoll(line + column_offset); - ret = get_column_pos(line, flag_schema->flag_column, &column_offset, &column_len); + table_type = table_manager_get_table_type(schema->ref_tbl_mgr, schema->table_id); + if (table_type == TABLE_TYPE_INTERVAL_PLUS) { + ret = get_column_pos(line, schema->district_column, &column_offset, &column_len); + if (ret < 0) { + goto error; + } + + if (column_len >= MAX_DISTRICT_STR) { + log_error(logger, MODULE_FLAG, + "flag_plus table(table_id:%d) line:%s district length too long", + schema->table_id, line); + goto error; + } + + char district[MAX_DISTRICT_STR] = {0}; + memcpy(district, (line + column_offset), column_len); + assert(strlen(district) > 0); + str_unescape(district); + item->district_id = table_manager_get_district_id(schema->ref_tbl_mgr, district); + } else { + item->district_id = DISTRICT_ANY; + } + + ret = get_column_pos(line, schema->flag_column, &column_offset, &column_len); if (ret < 0) { log_error(logger, MODULE_FLAG, "flag table(table_id:%d) line:%s has no flag", - flag_schema->table_id, line); + schema->table_id, line); goto error; } - flag_item->flag = strtoull(line + column_offset, NULL, 0); + item->flag = strtoull(line + column_offset, NULL, 0); - ret = get_column_pos(line, flag_schema->flag_mask_column, &column_offset, &column_len); + ret = get_column_pos(line, schema->flag_mask_column, &column_offset, &column_len); if (ret < 0) { log_error(logger, MODULE_FLAG, "flag table(table_id:%d) line:%s has no flag_mask", - flag_schema->table_id, line); + schema->table_id, line); goto error; } - flag_item->flag_mask = strtoull(line + column_offset, NULL, 0); + item->flag_mask = strtoull(line + column_offset, NULL, 0); - return flag_item; + return item; error: - FREE(flag_item); + FREE(item); return NULL; } @@ -330,7 +362,7 @@ int flag_runtime_update(void *flag_runtime, void *flag_schema, return -1; } - u_para = maat_item_inner_new(flag_item->group_id, item_id, 0); + u_para = maat_item_inner_new(flag_item->group_id, item_id, flag_item->district_id); item = maat_item_new(item_id, flag_item->group_id, u_para); HASH_ADD(hh, flag_rt->item_hash, item_id, sizeof(long long), item); @@ -433,10 +465,17 @@ int flag_runtime_scan(struct flag_runtime *flag_rt, int thread_id, } long long hit_item_ids[MAX_SCANNER_HIT_ITEM_NUM]; + struct maat_item_inner *item = NULL; + int real_hit_item_cnt = 0; + int district_id = state->district_id; + memset(hit_item_ids, 0, sizeof(hit_item_ids)); for (int i = 0; i < n_hit_item; i++) { - hit_item_ids[i] = hit_results[i].rule_id; + item = (struct maat_item_inner *)(hit_results[i].user_tag); + if (item->district_id == district_id || item->district_id == DISTRICT_ANY) { + hit_item_ids[real_hit_item_cnt++] = hit_results[i].rule_id; + } } size_t group_hit_cnt = 0; diff --git a/src/maat_interval.c b/src/maat_interval.c index 4355415..e49bcbc 100644 --- a/src/maat_interval.c +++ b/src/maat_interval.c @@ -215,7 +215,7 @@ struct interval_item *interval_item_new(const char *line, struct interval_schema if (column_len >= MAX_DISTRICT_STR) { log_error(logger, MODULE_INTERVAL, - "interval table(table_id:%d) line:%s district length too long", + "interval_plus table(table_id:%d) line:%s district length too long", schema->table_id, line); goto error; } diff --git a/src/maat_redis_monitor.c b/src/maat_redis_monitor.c index d11e725..913aabf 100644 --- a/src/maat_redis_monitor.c +++ b/src/maat_redis_monitor.c @@ -190,7 +190,7 @@ int _get_maat_redis_value(redisContext *c, struct serial_rule *rule_list, int ru for (i = 0; i < rule_num; i++) { snprintf(redis_cmd, sizeof(redis_cmd), - "GET %s:%s,%lu", mr_key_prefix[rule_list[i].op], + "GET %s:%s,%lld", mr_key_prefix[rule_list[i].op], rule_list[i].table_name, rule_list[i].rule_id); ret = redisAppendCommand(c, redis_cmd); @@ -236,7 +236,7 @@ int _get_maat_redis_value(redisContext *c, struct serial_rule *rule_list, int ru for (i = 0; i < failed_cnt; i++) { idx = retry_ids[i]; snprintf(redis_cmd, sizeof(redis_cmd), - "GET %s:%s,%lu", mr_key_prefix[MAAT_OP_DEL], + "GET %s:%s,%lld", mr_key_prefix[MAAT_OP_DEL], rule_list[idx].table_name, rule_list[idx].rule_id); ret = redisAppendCommand(c, redis_cmd); @@ -454,13 +454,13 @@ int recovery_history_version(const struct serial_rule *current, int current_num, struct serial_rule *s_rule_map = NULL; struct serial_rule *rule_node = NULL; for (i = 0; i < current_num; i++) { - snprintf(hkey, sizeof(hkey), "%ld,%s", current[i].rule_id, current[i].table_name); + snprintf(hkey, sizeof(hkey), "%lld,%s", current[i].rule_id, current[i].table_name); rule_node = serial_rule_clone(current + i); HASH_ADD_KEYPTR(hh, s_rule_map, hkey, strlen(hkey), rule_node); } for (i = changed_num - 1; i >= 0; i--) { - snprintf(hkey, sizeof(hkey), "%ld,%s", changed[i].rule_id, changed[i].table_name); + snprintf(hkey, sizeof(hkey), "%lld,%s", changed[i].rule_id, changed[i].table_name); //newly added rule is need to delete from current, so that history version can be recovered. if (changed[i].op == MAAT_OP_ADD) { rule_node = NULL; @@ -643,7 +643,7 @@ FULL_UPDATE: continue; } - ret = sscanf(sub_reply->element[i]->str, "%*[^:]:%[^,],%ld", + ret = sscanf(sub_reply->element[i]->str, "%*[^:]:%[^,],%lld", s_rule_array[full_idx].table_name, &(s_rule_array[full_idx].rule_id)); s_rule_array[full_idx].op = MAAT_OP_ADD; @@ -1316,7 +1316,7 @@ void check_maat_expiration(redisContext *c, struct log_handle *logger) for (size_t i = 0; i < s_rule_num; i++) { s_rule[i].op = MAAT_OP_DEL; - ret = sscanf(data_reply->element[i]->str, "%[^,],%ld", + ret = sscanf(data_reply->element[i]->str, "%[^,],%lld", s_rule[i].table_name, &(s_rule[i].rule_id)); assert(ret == 2); } diff --git a/src/maat_table.c b/src/maat_table.c index 510ba04..c041d7c 100644 --- a/src/maat_table.c +++ b/src/maat_table.c @@ -86,6 +86,15 @@ struct table_operations table_ops[TABLE_TYPE_MAX] = { .update_runtime = flag_runtime_update, .commit_runtime = flag_runtime_commit }, + { + .type = TABLE_TYPE_FLAG_PLUS, + .new_schema = flag_schema_new, + .free_schema = flag_schema_free, + .new_runtime = flag_runtime_new, + .free_runtime = flag_runtime_free, + .update_runtime = flag_runtime_update, + .commit_runtime = flag_runtime_commit + }, { .type = TABLE_TYPE_EXPR, .new_schema = expr_schema_new, @@ -113,6 +122,15 @@ struct table_operations table_ops[TABLE_TYPE_MAX] = { .update_runtime = ip_runtime_update, .commit_runtime = ip_runtime_commit }, + { + .type = TABLE_TYPE_PORT, + .new_schema = NULL, + .free_schema = NULL, + .new_runtime = NULL, + .free_runtime = NULL, + .update_runtime = NULL, + .commit_runtime = NULL + }, { .type = TABLE_TYPE_INTERVAL, .new_schema = interval_schema_new, @@ -131,12 +149,6 @@ struct table_operations table_ops[TABLE_TYPE_MAX] = { .update_runtime = interval_runtime_update, .commit_runtime = interval_runtime_commit }, - { - .type = TABLE_TYPE_DIGEST - }, - { - .type = TABLE_TYPE_SIMILARITY - }, { .type = TABLE_TYPE_CONJUNCTION, .new_schema = NULL, @@ -191,6 +203,15 @@ struct table_operations table_ops[TABLE_TYPE_MAX] = { .update_runtime = NULL, .commit_runtime = NULL }, + { + .type = TABLE_TYPE_COMPOSITION, + .new_schema = NULL, + .free_schema = NULL, + .new_runtime = NULL, + .free_runtime = NULL, + .update_runtime = NULL, + .commit_runtime = NULL + }, { .type = TABLE_TYPE_COMPILE, .new_schema = compile_schema_new, @@ -251,16 +272,19 @@ static void register_reserved_word(struct maat_kv_store *reserved_word_map) maat_kv_register(reserved_word_map, "group2compile", TABLE_TYPE_GROUP2COMPILE); maat_kv_register(reserved_word_map, "group2group", TABLE_TYPE_GROUP2GROUP); maat_kv_register(reserved_word_map, "flag", TABLE_TYPE_FLAG); + maat_kv_register(reserved_word_map, "flag_plus", TABLE_TYPE_FLAG_PLUS); maat_kv_register(reserved_word_map, "expr", TABLE_TYPE_EXPR); maat_kv_register(reserved_word_map, "expr_plus", TABLE_TYPE_EXPR_PLUS); maat_kv_register(reserved_word_map, "intval", TABLE_TYPE_INTERVAL); maat_kv_register(reserved_word_map, "intval_plus", TABLE_TYPE_INTERVAL_PLUS); maat_kv_register(reserved_word_map, "ip_plus", TABLE_TYPE_IP_PLUS); + maat_kv_register(reserved_word_map, "port", TABLE_TYPE_PORT); maat_kv_register(reserved_word_map, "plugin", TABLE_TYPE_PLUGIN); maat_kv_register(reserved_word_map, "ip_plugin", TABLE_TYPE_IP_PLUGIN); maat_kv_register(reserved_word_map, "bool_plugin", TABLE_TYPE_BOOL_PLUGIN); maat_kv_register(reserved_word_map, "fqdn_plugin", TABLE_TYPE_FQDN_PLUGIN); maat_kv_register(reserved_word_map, "virtual", TABLE_TYPE_VIRTUAL); + maat_kv_register(reserved_word_map, "composition", TABLE_TYPE_COMPOSITION); } static void register_tablename2id(cJSON *json, struct maat_kv_store *tablename2id_map, diff --git a/test/maat_ex_data_gtest.cpp b/test/maat_ex_data_gtest.cpp index a7f9e07..f8eace5 100644 --- a/test/maat_ex_data_gtest.cpp +++ b/test/maat_ex_data_gtest.cpp @@ -51,7 +51,7 @@ void ex_data_dup_cb(int table_id, void **to, void **from, long argl, void *argp) TEST(EXDataRuntime, Update) { const char *table_name = "TEST_PLUGIN_EXDATA_TABLE"; - int table_id = maat_table_get_id(g_maat_instance, table_name); + int table_id = maat_get_table_id(g_maat_instance, table_name); ASSERT_GT(table_id, 0); int ex_data_counter = 0; diff --git a/test/maat_framework_gtest.cpp b/test/maat_framework_gtest.cpp index e111f74..f03e661 100644 --- a/test/maat_framework_gtest.cpp +++ b/test/maat_framework_gtest.cpp @@ -36,7 +36,7 @@ protected: TEST_F(MaatFlagScan, basic) { const char *flag_table_name = "FLAG_CONFIG"; - int flag_table_id = maat_table_get_id(g_maat_instance, flag_table_name); + int flag_table_id = maat_get_table_id(g_maat_instance, flag_table_name); //compile_id:192 flag: 0000 0001 mask: 0000 0011 //scan_data: 0000 1001 or 0000 1101 should hit long long scan_data = 9; @@ -78,8 +78,8 @@ TEST_F(MaatFlagScan, basic) { TEST_F(MaatFlagScan, withExprRegion) { const char *flag_table_name = "FLAG_CONFIG"; const char *expr_table_name = "HTTP_URL_LITERAL"; - int flag_table_id = maat_table_get_id(g_maat_instance, flag_table_name); - int expr_table_id = maat_table_get_id(g_maat_instance, expr_table_name); + int flag_table_id = maat_get_table_id(g_maat_instance, flag_table_name); + int expr_table_id = maat_get_table_id(g_maat_instance, expr_table_name); //compile_id:193 flag: 0000 0010 mask: 0000 0011 //scan_data: 0000 0010 or 0000 0100 should hit long long flag_scan_data = 2; @@ -109,7 +109,7 @@ TEST_F(MaatFlagScan, withExprRegion) { TEST_F(MaatFlagScan, hitMultiCompile) { const char *flag_table_name = "FLAG_CONFIG"; - int flag_table_id = maat_table_get_id(g_maat_instance, flag_table_name); + int flag_table_id = maat_get_table_id(g_maat_instance, flag_table_name); //compile_id:192 flag: 0000 0001 mask: 0000 0011 //compile_id:194 flag: 0001 0101 mask: 0001 1111 //scan_data: 0001 0101 should hit compile192 and compile194 @@ -140,7 +140,7 @@ TEST_F(MaatFlagScan, hitMultiCompile) { TEST_F(MaatFlagScan, hitRepeatedCompile) { const char *flag_table_name = "FLAG_CONFIG"; - int flag_table_id = maat_table_get_id(g_maat_instance, flag_table_name); + int flag_table_id = maat_get_table_id(g_maat_instance, flag_table_name); long long results[ARRAY_SIZE] = {0}; size_t n_hit_result = 0; struct maat_state *state = NULL; @@ -178,6 +178,41 @@ TEST_F(MaatFlagScan, hitRepeatedCompile) { maat_state_free(&state); } + +TEST_F(MaatFlagScan, FlagPlus) { + const char *flag_table_name = "FLAG_PLUS_CONFIG"; + const char *region_name = "I love China"; + int flag_table_id = maat_get_table_id(g_maat_instance, flag_table_name); + //compile_id:196 flag: 0001 1111 mask: 0000 1111 + //scan_data: 0000 1111 or 0001 1111 should hit + long long scan_data1 = 15; + long long results[ARRAY_SIZE] = {0}; + size_t n_hit_result = 0; + struct maat_state *state = NULL; + int ret = maat_scan_flag(g_maat_instance, flag_table_id, 0, scan_data1, results, + ARRAY_SIZE, &n_hit_result, &state); + EXPECT_EQ(ret, MAAT_SCAN_ERR); + + ret = maat_state_set_scan_district(g_maat_instance, &state, region_name, strlen(region_name)); + ASSERT_EQ(ret, 0); + + ret = maat_scan_flag(g_maat_instance, flag_table_id, 0, scan_data1, results, + ARRAY_SIZE, &n_hit_result, &state); + EXPECT_EQ(ret, MAAT_SCAN_HIT); + EXPECT_EQ(n_hit_result, 1); + EXPECT_EQ(results[0], 196); + + ret = maat_scan_flag(g_maat_instance, flag_table_id, 0, scan_data1, results, + ARRAY_SIZE, &n_hit_result, &state); + EXPECT_EQ(ret, MAAT_SCAN_HALF_HIT); + + struct maat_hit_path hit_path[HIT_PATH_SIZE] = {0}; + int n_read = 0; + n_read = maat_state_get_hit_paths(g_maat_instance, &state, hit_path, HIT_PATH_SIZE); + EXPECT_NE(n_read, 0); + maat_state_free(&state); +} + class MaatStringScan : public testing::Test { protected: @@ -191,7 +226,7 @@ protected: TEST_F(MaatStringScan, Expr8) { const char *table_name = "KEYWORDS_TABLE"; - int table_id = maat_table_get_id(g_maat_instance, table_name); + int table_id = maat_get_table_id(g_maat_instance, table_name); char scan_data[128] = "string1, string2, string3, string4, string5, string6, string7, string8"; long long results[ARRAY_SIZE] = {0}; size_t n_hit_result = 0; @@ -216,7 +251,7 @@ TEST_F(MaatStringScan, Regex) { struct maat_state *state = NULL; const char *cookie = "Cookie: Txa123aheadBCAxd"; const char *table_name = "HTTP_URL_REGEX"; - int table_id = maat_table_get_id(g_maat_instance, table_name); + int table_id = maat_get_table_id(g_maat_instance, table_name); ret = maat_scan_string(g_maat_instance, table_id, 0, cookie, strlen(cookie), results, ARRAY_SIZE, &n_hit_result, &state); @@ -256,7 +291,7 @@ TEST_F(MaatStringScan, ExprPlus) { const char *scan_data1 = "http://www.cyberessays.com/search_results.php?action=search&query=abckkk,1234567"; const char *scan_data2 = "Addis Sapphire Hotel"; const char *table_name = "HTTP_SIGNATURE"; - int table_id = maat_table_get_id(g_maat_instance, table_name); + int table_id = maat_get_table_id(g_maat_instance, table_name); int ret = maat_scan_string(g_maat_instance, table_id, 0, scan_data1, strlen(scan_data1), results, ARRAY_SIZE, &n_hit_result, &state); @@ -287,8 +322,8 @@ TEST_F(MaatStringScan, ExprAndExprPlus) { const char *expr_plus_table_name = "HTTP_SIGNATURE"; const char *region_name = "I love China"; const char *scan_data = "today is Monday and yesterday is Tuesday"; - int expr_table_id = maat_table_get_id(g_maat_instance, expr_table_name); - int expr_plus_table_id = maat_table_get_id(g_maat_instance, expr_plus_table_name); + int expr_table_id = maat_get_table_id(g_maat_instance, expr_table_name); + int expr_plus_table_id = maat_get_table_id(g_maat_instance, expr_plus_table_name); int ret = maat_scan_string(g_maat_instance, expr_plus_table_id, 0, scan_data, strlen(scan_data), results, ARRAY_SIZE, &n_hit_result, &state); @@ -314,7 +349,7 @@ TEST_F(MaatStringScan, StreamInput) { const char *scan_data = "http://www.cyberessays.com/search_results.php?action=search&query=yulingjing,abckkk,1234567"; const char *table_name = "HTTP_URL_STREAM"; - int table_id = maat_table_get_id(g_maat_instance, table_name); + int table_id = maat_get_table_id(g_maat_instance, table_name); ASSERT_GT(table_id, 0); struct maat_stream *sp = maat_scan_stream_open(g_maat_instance, table_id, 0); @@ -354,7 +389,7 @@ TEST_F(MaatStringScan, ShouldNotHitExprPlus) { 0xf6, 0x30, 0x81, 0xf4, 0x3f, 0xc2, 0x19, 0x6a, 0x68, 0x61, 0x93, 0x08, 0xc0, 0x0a, 0xab, 0x00 }; - int table_id = maat_table_get_id(g_maat_instance, "APP_PAYLOAD"); + int table_id = maat_get_table_id(g_maat_instance, "APP_PAYLOAD"); ASSERT_GT(table_id, 0); int ret = maat_state_set_scan_district(g_maat_instance, &state, region_name, strlen(region_name)); @@ -375,7 +410,7 @@ TEST_F(MaatStringScan, ExprPlusWithHex) { const char *region_name1 = "Content-Type"; const char *region_name2 = "User-Agent"; - int table_id = maat_table_get_id(g_maat_instance, "HTTP_SIGNATURE"); + int table_id = maat_get_table_id(g_maat_instance, "HTTP_SIGNATURE"); ASSERT_GT(table_id, 0); int ret = maat_state_set_scan_district(g_maat_instance, &state, region_name1, strlen(region_name1)); @@ -391,7 +426,7 @@ TEST_F(MaatStringScan, ExprPlusWithHex) { results, ARRAY_SIZE, &n_hit_result, &state); EXPECT_EQ(ret, MAAT_SCAN_HALF_HIT); - table_id = maat_table_get_id(g_maat_instance, "KEYWORDS_TABLE"); + table_id = maat_get_table_id(g_maat_instance, "KEYWORDS_TABLE"); ret = maat_scan_string(g_maat_instance, table_id, 0, scan_data2, strlen(scan_data2), results, ARRAY_SIZE, &n_hit_result, &state); EXPECT_EQ(ret, MAAT_SCAN_HIT); @@ -432,7 +467,7 @@ TEST_F(MaatStringScan, ExprPlusWithOffset) 0xf6, 0x30, 0x81, 0xf4, 0x3f, 0xc2, 0x19, 0x6a, 0x68, 0x61, 0x93, 0x08, 0xc0, 0x0a }; - int table_id = maat_table_get_id(g_maat_instance, "APP_PAYLOAD"); + int table_id = maat_get_table_id(g_maat_instance, "APP_PAYLOAD"); ASSERT_GT(table_id, 0); int ret = maat_state_set_scan_district(g_maat_instance, &state, region_name, strlen(region_name)); @@ -452,7 +487,7 @@ TEST_F(MaatStringScan, ExprPlusWithOffset) #endif TEST_F(MaatStringScan, dynamic_config) { const char *table_name = "HTTP_URL_LITERAL"; - int table_id = maat_table_get_id(g_maat_instance, table_name); + int table_id = maat_get_table_id(g_maat_instance, table_name); char data[128] = "hello world"; long long results[ARRAY_SIZE] = {0}; size_t n_hit_result = 0; @@ -551,7 +586,7 @@ protected: TEST_F(MaatIPScan, IPv4) { const char *table_name = "IP_PLUS_CONFIG"; - int table_id = maat_table_get_id(g_maat_instance, table_name); + int table_id = maat_get_table_id(g_maat_instance, table_name); ASSERT_GT(table_id, 0); char ip_str[32] = "10.0.7.100"; @@ -573,7 +608,7 @@ TEST_F(MaatIPScan, IPv4) { TEST_F(MaatIPScan, IPv6) { const char *table_name = "IP_PLUS_CONFIG"; - int table_id = maat_table_get_id(g_maat_instance, table_name); + int table_id = maat_get_table_id(g_maat_instance, table_name); char ip_str[32] = "1001:da8:205:1::101"; uint8_t sip[16]; int ret = inet_pton(AF_INET6, ip_str, &sip); @@ -593,7 +628,7 @@ TEST_F(MaatIPScan, IPv6) { TEST_F(MaatIPScan, dynamic_config) { const char *table_name = "IP_PLUS_CONFIG"; - int table_id = maat_table_get_id(g_maat_instance, table_name); + int table_id = maat_get_table_id(g_maat_instance, table_name); char ip_str[32] = "100.100.100.100"; uint32_t sip; int ret = inet_pton(AF_INET, ip_str, &sip); @@ -693,7 +728,7 @@ TEST_F(MaatIntervalScan, Pure) { size_t n_hit_result = 0; struct maat_state *state = NULL; const char *table_name = "CONTENT_SIZE"; - int table_id = maat_table_get_id(g_maat_instance, table_name); + int table_id = maat_get_table_id(g_maat_instance, table_name); unsigned int scan_data1 = 2015; int ret = maat_scan_integer(g_maat_instance, table_id, 0, scan_data1, results, ARRAY_SIZE, @@ -714,7 +749,7 @@ TEST_F(MaatIntervalScan, IntervalPlus) { size_t n_hit_result = 0; struct maat_state *state = NULL; const char *table_name = "INTERGER_PLUS"; - int table_id = maat_table_get_id(g_maat_instance, table_name); + int table_id = maat_get_table_id(g_maat_instance, table_name); const char *district_str = "interval.plus"; int ret = maat_state_set_scan_district(g_maat_instance, &state, district_str, strlen(district_str)); @@ -750,8 +785,8 @@ TEST_F(NOTLogic, ScanNotAtLast) { struct maat_state *state = NULL; const char *hit_table_name = "HTTP_URL_LITERAL"; const char *not_hit_table_name = "KEYWORDS_TABLE"; - int hit_table_id = maat_table_get_id(g_maat_instance, hit_table_name); - int not_hit_table_id = maat_table_get_id(g_maat_instance, not_hit_table_name); + int hit_table_id = maat_get_table_id(g_maat_instance, hit_table_name); + int not_hit_table_id = maat_get_table_id(g_maat_instance, not_hit_table_name); int ret = maat_scan_string(g_maat_instance, hit_table_id, 0, string_should_hit, strlen(string_should_hit), results, ARRAY_SIZE, &n_hit_result, &state); @@ -818,7 +853,7 @@ protected: TEST_F(PluginTable, Callback) { const char *table_name = "QD_ENTRY_INFO"; - int table_id = maat_table_get_id(g_maat_instance, table_name); + int table_id = maat_get_table_id(g_maat_instance, table_name); int ret = maat_table_callback_register(g_maat_instance, table_id, maat_read_entry_start_cb, @@ -887,7 +922,7 @@ void ip_plugin_EX_dup_cb(int table_id, void **to, void **from, long argl, void * TEST_F(IPPluginTable, EX_DATA) { int ip_plugin_ex_data_counter = 0; const char *table_name = "TEST_IP_PLUGIN_WITH_EXDATA"; - int table_id = maat_table_get_id(g_maat_instance, table_name); + int table_id = maat_get_table_id(g_maat_instance, table_name); int ret = maat_plugin_table_ex_schema_register(g_maat_instance, table_id, ip_plugin_EX_new_cb, @@ -993,7 +1028,7 @@ void fqdn_plugin_ex_dup_cb(int table_id, void **to, void **from, long argl, void TEST_F(FQDNPluginTable, EX_DATA) { const char *table_name = "TEST_FQDN_PLUGIN_WITH_EXDATA"; - int table_id = maat_table_get_id(g_maat_instance, table_name); + int table_id = maat_get_table_id(g_maat_instance, table_name); ASSERT_GT(table_id, 0); int fqdn_plugin_ex_data_counter = 0; @@ -1086,7 +1121,7 @@ TEST_F(BoolPluginTable, EX_DATA) { int ex_data_counter = 0, i = 0; const char *table_name = "TEST_BOOL_PLUGIN_WITH_EXDATA"; - int table_id = maat_table_get_id(g_maat_instance, table_name); + int table_id = maat_get_table_id(g_maat_instance, table_name); ASSERT_GT(table_id, 0); int ret = maat_plugin_table_ex_schema_register(g_maat_instance, table_id, @@ -1146,7 +1181,7 @@ TEST_F(VirtualTable, basic) { size_t n_hit_result = 0; struct maat_state *state = NULL; const char *table_name = "HTTP_RESPONSE_KEYWORDS"; - int table_id = maat_table_get_id(g_maat_instance, table_name); + int table_id = maat_get_table_id(g_maat_instance, table_name); char scan_data[128] = "string1, string2, string3, string4, string5, string6, string7, string8"; int ret = maat_scan_string(g_maat_instance, table_id, 0, scan_data, strlen(scan_data), @@ -1224,17 +1259,17 @@ void compile_ex_param_dup(int table_id, void **to, void **from, long argl, void pthread_mutex_unlock(&(from_param->lock)); *((struct rule_ex_param**)to) = from_param; } - +#if 0 TEST_F(CompileTable, CompileEXData) { long long results[ARRAY_SIZE] = {0}; size_t n_hit_result = 0; struct maat_state *state = NULL; const char *url = "i.ytimg.com/vi/OtCNcustg_I/hqdefault.jpg?sqp=-oaymwEZCNACELwBSFXyq4qpAwsIARUAAIhCGAFwAQ==&rs=AOn4CLDOp_5fHMaCA9XZuJdCRv4DNDorMg"; const char *table_name = "HTTP_URL_LITERAL"; - const char *compile_table_name = "COMPILE"; + const char *compile_table_name = "COMPILE_ALIAS"; const char *expect_name = "I have a name"; - int table_id = maat_table_get_id(g_maat_instance, table_name); - int compile_table_id = maat_table_get_id(g_maat_instance, compile_table_name); + int table_id = maat_get_table_id(g_maat_instance, table_name); + int compile_table_id = maat_get_table_id(g_maat_instance, compile_table_name); int ex_data_counter = 0; int ret = maat_plugin_table_ex_schema_register(g_maat_instance, compile_table_id, @@ -1243,6 +1278,10 @@ TEST_F(CompileTable, CompileEXData) { compile_ex_param_dup, 0, &ex_data_counter); ASSERT_TRUE(ret == 0); + EXPECT_EQ(ex_data_counter, 1); + + ret = maat_state_set_scan_compile_tables(g_maat_instance, &state, &compile_table_name, 1); + EXPECT_EQ(ret, 0); ret = maat_scan_string(g_maat_instance, table_id, 0, url, strlen(url), results, ARRAY_SIZE, &n_hit_result, &state); @@ -1262,7 +1301,7 @@ TEST_F(CompileTable, CompileEXData) { maat_state_free(&state); } - +#endif TEST_F(CompileTable, CompileRuleUpdate) { //9999 0 0 0 0 0 anything 1 1 0.0 const char *compile_table_name = "COMPILE"; @@ -1328,7 +1367,7 @@ void accept_tags_entry_cb(int table_id, const char *table_line, void *u_para) TEST_F(Policy, PluginRuleTags1) { const char *table_name = "TEST_EFFECTIVE_RANGE_TABLE"; - int table_id = maat_table_get_id(g_maat_instance, table_name); + int table_id = maat_get_table_id(g_maat_instance, table_name); ASSERT_GT(table_id, 0); int callback_times=0; @@ -1349,7 +1388,7 @@ void accept_tags_entry2_cb(int table_id, const char *table_line, void *u_para) TEST_F(Policy, PluginRuleTags2) { const char *table_name = "IR_INTERCEPT_IP"; - int table_id = maat_table_get_id(g_maat_instance, table_name); + int table_id = maat_get_table_id(g_maat_instance, table_name); ASSERT_GT(table_id, 0); int callback_times = 0; @@ -1370,7 +1409,7 @@ TEST_F(Policy, CompileRuleTags) { const char *should_not_hit = "string aaa should not hit"; const char *table_name = "HTTP_URL_LITERAL"; - int table_id = maat_table_get_id(g_maat_instance, table_name); + int table_id = maat_get_table_id(g_maat_instance, table_name); ASSERT_GT(table_id, 0); int ret = maat_scan_string(g_maat_instance, table_id, 0, should_not_hit, @@ -1394,11 +1433,11 @@ TEST_F(Policy, CompileEXData) { const char *table_name = "HTTP_URL_LITERAL"; const char *expect_name = "I have a name"; - int table_id = maat_table_get_id(g_maat_instance, table_name); + int table_id = maat_get_table_id(g_maat_instance, table_name); ASSERT_GT(table_id, 0); int ex_data_counter = 0; - int compile_table_id = maat_table_get_id(g_maat_instance, "COMPILE"); + int compile_table_id = maat_get_table_id(g_maat_instance, "COMPILE"); int ret = maat_plugin_table_ex_schema_register(g_maat_instance, compile_table_id, compile_ex_param_new, compile_ex_param_free, @@ -1436,14 +1475,14 @@ TEST_F(Policy, SubGroup) { uint32_t sip; inet_pton(AF_INET,"10.0.6.205", &sip); - int table_id = maat_table_get_id(g_maat_instance, "MAIL_ADDR"); + int table_id = maat_get_table_id(g_maat_instance, "MAIL_ADDR"); ASSERT_GT(table_id, 0); int ret = maat_scan_string(g_maat_instance, table_id, 0, scan_data, strlen(scan_data), results, ARRAY_SIZE, &n_hit_result, &state); EXPECT_EQ(ret, MAAT_SCAN_HALF_HIT); - table_id = maat_table_get_id(g_maat_instance, "IP_CONFIG"); + table_id = maat_get_table_id(g_maat_instance, "IP_CONFIG"); ASSERT_GT(table_id, 0); ret = maat_scan_ipv4(g_maat_instance, table_id, 0, sip, results, ARRAY_SIZE, @@ -1511,7 +1550,7 @@ TEST_F(MaatCmdTest, PluginEXData) { "2\t192.168.0.2\tliuqiangdong\t1\t0", "3\t192.168.0.3\tmayun\t1\t0", "4\t192.168.0.4\tliyanhong\t1\t0"}; - int table_id = maat_table_get_id(g_maat_instance, table_name); + int table_id = maat_get_table_id(g_maat_instance, table_name); ASSERT_GT(table_id, 0); /* 1st line */ diff --git a/test/maat_input_mode_gtest.cpp b/test/maat_input_mode_gtest.cpp index 4ab1a93..43a76e4 100644 --- a/test/maat_input_mode_gtest.cpp +++ b/test/maat_input_mode_gtest.cpp @@ -37,7 +37,7 @@ TEST(json_mode, maat_scan_string) { EXPECT_TRUE(maat_instance != NULL); const char *table_name = "KEYWORDS_TABLE"; - int table_id = maat_table_get_id(maat_instance, table_name); + int table_id = maat_get_table_id(maat_instance, table_name); char scan_data[128] = "string1, string2, string3, string4, string5, string6, string7, string8"; long long results[5] = {0}; size_t n_hit_result = 0; @@ -82,7 +82,7 @@ TEST(iris_mode, maat_scan_string) { EXPECT_TRUE(maat_instance != NULL); const char *table_name = "KEYWORDS_TABLE"; - int table_id = maat_table_get_id(maat_instance, table_name); + int table_id = maat_get_table_id(maat_instance, table_name); char scan_data[128] = "string1, string2, string3, string4, string5, string6, string7, string8"; long long results[5] = {0}; size_t n_hit_result = 0; @@ -204,7 +204,7 @@ TEST(redis_mode, maat_scan_string) { struct maat *maat_instance = maat_new(opts, table_info_path); const char *table_name = "KEYWORDS_TABLE"; - int table_id = maat_table_get_id(maat_instance, table_name); + int table_id = maat_get_table_id(maat_instance, table_name); char scan_data[128] = "string1, string2, string3, string4, string5, string6, string7, string8"; long long results[5] = {0}; size_t n_hit_result = 0; diff --git a/test/maat_json.json b/test/maat_json.json index 2791de8..546d58d 100644 --- a/test/maat_json.json +++ b/test/maat_json.json @@ -322,31 +322,6 @@ } ] }, - { - "compile_id": 127, - "service": 1, - "action": 1, - "do_blacklist": 1, - "do_log": 1, - "user_region": "anything", - "is_valid": "yes", - "groups": [ - { - "group_name": "Untitled", - "regions": [ - { - "table_name": "FILE_DIGEST", - "table_type": "digest", - "table_content": { - "raw_len": 1160164, - "digest": "3072:Xk/maCm4yLYtRIFDFnVfHH+CAQI6VD5mekDmaa/4qCuFnqak1s3/+Gn1IJHa/AvybUsbGWcIAy9grTp2s5bbj/TaKxONfb[0:1160163]#12288:UChtbFS6pypdTy4m2[0:1160163]", - "cfds_level": 70 - } - } - ] - } - ] - }, { "compile_id": 128, "service": 1, @@ -530,30 +505,6 @@ } ] }, - { - "compile_id": 135, - "service": 1, - "action": 1, - "do_blacklist": 1, - "do_log": 1, - "user_region": "anything", - "is_valid": "yes", - "groups": [ - { - "group_name": "Untitled", - "regions": [ - { - "table_name": "SIM_URL", - "table_type": "similar", - "table_content": { - "target": "mwss.xiu.youku.com/live/hls/v1/0000000000000000000000001526a0a8/709.ts?&token=98765", - "threshold": 90 - } - } - ] - } - ] - }, { "compile_id": 136, "service": 1, @@ -693,10 +644,12 @@ "do_blacklist": 1, "do_log": 1, "user_region": "Something:I\\bhave\\ba\\bname,7799", + "compile_table_name": "COMPILE_ALIAS", "is_valid": "yes", "groups": [ { "group_name": "Untitled", + "g2c_table_name": "GROUP2COMPILE_ALIAS", "regions": [ { "table_name": "HTTP_URL_LITERAL", @@ -2457,6 +2410,30 @@ ] } ] + }, + { + "compile_id": 196, + "service": 0, + "action": 0, + "do_blacklist": 0, + "do_log": 0, + "user_region": "anything", + "is_valid": "yes", + "groups": [ + { + "regions": [ + { + "table_type": "flag_plus", + "table_name": "FLAG_PLUS_CONFIG", + "table_content": { + "district": "I love\\bChina", + "flag": 30, + "flag_mask": 14 + } + } + ] + } + ] } ], "plugin_table": [ diff --git a/test/table_info.conf b/test/table_info.conf index d243da4..4bf2522 100644 --- a/test/table_info.conf +++ b/test/table_info.conf @@ -27,170 +27,6 @@ }, { "table_id":2, - "table_name":"GROUP2GROUP", - "table_type":"group2group", - "valid_column":3, - "custom": { - "group_id":1, - "super_group_id":2 - } - }, - { - "table_id":3, - "table_name":"HTTP_URL_LITERAL", - "table_type":"expr", - "valid_column":7, - "custom": { - "scan_mode":"block", - "pattern_type":"literal", - "item_id":1, - "group_id":2, - "keywords":3, - "expr_type":4, - "match_method":5, - "is_hexbin":6 - } - }, - { - "table_id":4, - "table_name":"HTTP_URL_REGEX", - "table_type":"expr", - "valid_column":7, - "custom": { - "scan_mode":"block", - "pattern_type":"regex", - "item_id":1, - "group_id":2, - "keywords":3, - "expr_type":4, - "match_method":5, - "is_hexbin":6 - } - }, - { - "table_id":5, - "table_name":"HTTP_HOST", - "table_type":"expr", - "valid_column":7, - "custom": { - "scan_mode":"block", - "pattern_type":"literal", - "item_id":1, - "group_id":2, - "keywords":3, - "expr_type":4, - "match_method":5, - "is_hexbin":6 - } - }, - { - "table_id":6, - "table_name":"KEYWORDS_TABLE", - "table_type":"expr", - "valid_column":7, - "custom": { - "scan_mode":"block", - "pattern_type":"literal", - "item_id":1, - "group_id":2, - "keywords":3, - "expr_type":4, - "match_method":5, - "is_hexbin":6 - } - }, - { - "table_id":7, - "table_name":"IP_CONFIG", - "table_type":"ip_plus", - "valid_column":18, - "custom": { - "item_id":1, - "group_id":2, - "addr_type":3, - "saddr_format":4, - "sip1":5, - "sip2":6 - } - }, - { - "table_id":8, - "table_name":"CONTENT_SIZE", - "table_type":"intval", - "valid_column":5, - "custom": { - "item_id":1, - "group_id":2, - "low_bound":3, - "up_bound":4 - } - }, - { - "table_id":9, - "table_name":"QD_ENTRY_INFO", - "table_type":"plugin", - "valid_column":4, - "custom": { - "key":1, - "tag":3 - } - }, - { - "table_id":10, - "table_name":"HTTP_SIGNATURE", - "table_type":"expr_plus", - "valid_column":8, - "custom": { - "scan_mode":"block", - "pattern_type":"literal", - "item_id":1, - "group_id":2, - "district":3, - "keywords":4, - "expr_type":5, - "match_method":6, - "is_hexbin":7 - } - }, - { - "table_id":11, - "table_name":"IMAGE_FP", - "table_type":"expr", - "valid_column":7, - "custom": { - "scan_mode":"block", - "pattern_type":"literal", - "item_id":1, - "group_id":2, - "keywords":3, - "expr_type":4, - "match_method":5, - "is_hexbin":6 - } - }, - { - "table_id":12, - "table_name":"TEST_EFFECTIVE_RANGE_TABLE", - "table_type":"plugin", - "valid_column":4, - "custom": { - "key":1, - "tag":5 - } - }, - { - "table_id":13, - "table_name":"TEST_FOREIGN_KEY", - "table_type":"plugin", - "valid_column":4, - "custom": { - "key":2, - "tag":3, - "foreign": [6,8] - } - }, - { - "table_id":14, "table_name":"COMPILE_ALIAS", "table_type":"compile", "user_region_encoded":"escape", @@ -207,8 +43,186 @@ "evaluation_order":10 } }, + { + "table_id":3, + "table_name":"GROUP2COMPILE_ALIAS", + "table_type":"group2compile", + "associated_compile_table_id":0, + "valid_column":3, + "custom": { + "group_id":1, + "compile_id":2, + "not_flag":4, + "virtual_table_name":5, + "clause_index":6 + } + }, + { + "table_id":4, + "table_name":"GROUP2GROUP", + "table_type":"group2group", + "valid_column":3, + "custom": { + "group_id":1, + "super_group_id":2 + } + }, + { + "table_id":5, + "table_name":"HTTP_URL_LITERAL", + "table_type":"expr", + "valid_column":7, + "custom": { + "scan_mode":"block", + "pattern_type":"literal", + "item_id":1, + "group_id":2, + "keywords":3, + "expr_type":4, + "match_method":5, + "is_hexbin":6 + } + }, + { + "table_id":6, + "table_name":"HTTP_URL_REGEX", + "table_type":"expr", + "valid_column":7, + "custom": { + "scan_mode":"block", + "pattern_type":"regex", + "item_id":1, + "group_id":2, + "keywords":3, + "expr_type":4, + "match_method":5, + "is_hexbin":6 + } + }, + { + "table_id":7, + "table_name":"HTTP_HOST", + "table_type":"expr", + "valid_column":7, + "custom": { + "scan_mode":"block", + "pattern_type":"literal", + "item_id":1, + "group_id":2, + "keywords":3, + "expr_type":4, + "match_method":5, + "is_hexbin":6 + } + }, + { + "table_id":8, + "table_name":"KEYWORDS_TABLE", + "table_type":"expr", + "valid_column":7, + "custom": { + "scan_mode":"block", + "pattern_type":"literal", + "item_id":1, + "group_id":2, + "keywords":3, + "expr_type":4, + "match_method":5, + "is_hexbin":6 + } + }, + { + "table_id":9, + "table_name":"IP_CONFIG", + "table_type":"ip_plus", + "valid_column":18, + "custom": { + "item_id":1, + "group_id":2, + "addr_type":3, + "saddr_format":4, + "sip1":5, + "sip2":6 + } + }, + { + "table_id":10, + "table_name":"CONTENT_SIZE", + "table_type":"intval", + "valid_column":5, + "custom": { + "item_id":1, + "group_id":2, + "low_bound":3, + "up_bound":4 + } + }, + { + "table_id":11, + "table_name":"QD_ENTRY_INFO", + "table_type":"plugin", + "valid_column":4, + "custom": { + "key":1, + "tag":3 + } + }, + { + "table_id":12, + "table_name":"HTTP_SIGNATURE", + "table_type":"expr_plus", + "valid_column":8, + "custom": { + "scan_mode":"block", + "pattern_type":"literal", + "item_id":1, + "group_id":2, + "district":3, + "keywords":4, + "expr_type":5, + "match_method":6, + "is_hexbin":7 + } + }, + { + "table_id":13, + "table_name":"IMAGE_FP", + "table_type":"expr", + "valid_column":7, + "custom": { + "scan_mode":"block", + "pattern_type":"literal", + "item_id":1, + "group_id":2, + "keywords":3, + "expr_type":4, + "match_method":5, + "is_hexbin":6 + } + }, + { + "table_id":14, + "table_name":"TEST_EFFECTIVE_RANGE_TABLE", + "table_type":"plugin", + "valid_column":4, + "custom": { + "key":1, + "tag":5 + } + }, { "table_id":15, + "table_name":"TEST_FOREIGN_KEY", + "table_type":"plugin", + "valid_column":4, + "custom": { + "key":2, + "tag":3, + "foreign": [6,8] + } + }, + { + "table_id":16, "table_name":"TEST_PLUGIN_EXDATA_TABLE", "table_type":"plugin", "valid_column":4, @@ -219,7 +233,7 @@ } }, { - "table_id":16, + "table_id":17, "table_name":"IR_INTERCEPT_IP", "table_type":"plugin", "valid_column":14, @@ -229,7 +243,7 @@ } }, { - "table_id":17, + "table_id":18, "table_name":"APP_PAYLOAD", "table_type":"expr_plus", "valid_column":8, @@ -246,7 +260,7 @@ } }, { - "table_id":18, + "table_id":19, "table_name":"TROJAN_PAYLOAD", "table_type":"expr", "valid_column":7, @@ -263,7 +277,7 @@ } }, { - "table_id":19, + "table_id":20, "table_name":"MAIL_ADDR", "table_type":"expr", "valid_column":7, @@ -279,7 +293,7 @@ } }, { - "table_id":20, + "table_id":21, "table_name":"IP_PLUS_CONFIG", "table_type":"ip_plus", "valid_column":18, @@ -293,43 +307,43 @@ } }, { - "table_id":21, + "table_id":22, "table_name":"HTTP_RESPONSE_KEYWORDS", "table_type":"virtual", "physical_table": ["KEYWORDS_TABLE"] }, { - "table_id":22, + "table_id":23, "table_name":"HTTP_REQUEST_HEADER", "table_type":"virtual", "physical_table": ["HTTP_SIGNATURE"] }, { - "table_id":23, + "table_id":24, "table_name":"HTTP_RESPONSE_HEADER", "table_type":"virtual", "physical_table": ["HTTP_SIGNATURE"] }, { - "table_id":24, + "table_id":25, "table_name":"VIRTUAL_IP_PLUS_TABLE", "table_type":"virtual", "physical_table": ["IP_PLUS_CONFIG"] }, { - "table_id":24, + "table_id":26, "table_name":"VIRTUAL_IP_PLUS_SOURCE", "table_type":"virtual", "physical_table": ["IP_PLUS_CONFIG"] }, { - "table_id":24, + "table_id":27, "table_name":"VIRTUAL_IP_PLUS_DESTINATION", "table_type":"virtual", "physical_table": ["IP_PLUS_CONFIG"] }, { - "table_id":29, + "table_id":28, "table_name":"TEST_IP_PLUGIN_WITH_EXDATA", "table_type":"ip_plugin", "valid_column":6, @@ -342,7 +356,7 @@ } }, { - "table_id":30, + "table_id":29, "table_name":"AS_NUMBER", "table_type":"expr", "valid_column":7, @@ -358,19 +372,19 @@ } }, { - "table_id":31, + "table_id":30, "table_name":"SOURCE_IP_ASN", "table_type":"virtual", "physical_table":["AS_NUMBER"] }, { - "table_id":32, + "table_id":31, "table_name":"DESTINATION_IP_ASN", "table_type":"virtual", "physical_table":["AS_NUMBER"] }, { - "table_id":33, + "table_id":32, "table_name":"GeoLocation", "table_type":"expr", "valid_column":7, @@ -386,13 +400,13 @@ } }, { - "table_id":34, + "table_id":33, "table_name":"SOURCE_IP_GEO", "table_type":"virtual", "physical_table":["GeoLocation"] }, { - "table_id":35, + "table_id":34, "table_name":"INTERGER_PLUS", "table_type":"intval_plus", "valid_column":6, @@ -405,7 +419,7 @@ } }, { - "table_id":36, + "table_id":35, "table_name":"TEST_FQDN_PLUGIN_WITH_EXDATA", "table_type":"fqdn_plugin", "valid_column":5, @@ -416,7 +430,7 @@ } }, { - "table_id":38, + "table_id":36, "table_name":"APP_ID", "table_type":"intval", "valid_column":5, @@ -428,7 +442,7 @@ } }, { - "table_id":39, + "table_id":37, "table_name":"EMPTY_KEYWORD", "table_type":"expr", "valid_column":7, @@ -444,7 +458,7 @@ } }, { - "table_id":40, + "table_id":38, "table_name":"EMPTY_INTERGER", "table_type":"intval", "valid_column":5, @@ -456,7 +470,7 @@ } }, { - "table_id":41, + "table_id":39, "table_name":"TEST_BOOL_PLUGIN_WITH_EXDATA", "table_type":"bool_plugin", "valid_column":4, @@ -466,7 +480,7 @@ } }, { - "table_id":42, + "table_id":40, "table_name":"FLAG_CONFIG", "table_type":"flag", "valid_column":5, @@ -478,9 +492,22 @@ } }, { - "table_id":43, - "table_name":"PORT_PROTO_CONFIG", - "table_type":"port_proto", + "table_id":41, + "table_name":"FLAG_PLUS_CONFIG", + "table_type":"flag_plus", + "valid_column":6, + "custom": { + "item_id":1, + "group_id":2, + "district":3, + "flag":4, + "flag_mask":5 + } + }, + { + "table_id":42, + "table_name":"PORT_CONFIG", + "table_type":"port", "valid_column":7, "custom": { "item_id":1, @@ -490,5 +517,28 @@ "port2":5, "proto":6 } + }, + { + "table_id":43, + "table_name":"VIRTUAL_PORT_SOURCE", + "table_type":"virtual", + "physical_table": ["PORT_CONFIG"] + }, + { + "table_id":44, + "table_name":"VIRTUAL_PORT_DESTINATION", + "table_type":"virtual", + "physical_table": ["PORT_CONFIG"] + }, + { + "table_id":45, + "table_name":"COMPOSITION_CONFIG", + "table_type":"composition", + "custom": { + "source_ip":"VIRTUAL_IP_PLUS_SOURCE", + "dest_ip":"VIRTUAL_IP_PLUS_DESTINATION", + "source_port":"VIRTUAL_PORT_SOURCE", + "dest_port":"VIRTUAL_PORT_DESTINATION" + } } ] \ No newline at end of file diff --git a/tools/maat_redis_tool.cpp b/tools/maat_redis_tool.cpp index e8969a9..7e7dd35 100644 --- a/tools/maat_redis_tool.cpp +++ b/tools/maat_redis_tool.cpp @@ -162,7 +162,7 @@ void read_rule_from_redis(redisContext *c, long long desire_version, const char cur_table = rule_list[i].table_name; } - fprintf(table_fp, "%s\tkey=%ld\n", rule_list[i].table_line, rule_list[i].rule_id); + fprintf(table_fp, "%s\tkey=%lld\n", rule_list[i].table_line, rule_list[i].rule_id); line_count++; }