diff --git a/CMakeLists.txt b/CMakeLists.txt index 3966239..a449375 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -14,4 +14,5 @@ enable_testing() add_subdirectory(vendor) add_subdirectory(src) add_subdirectory(test) +add_subdirectory(tools) add_subdirectory(scanner) \ No newline at end of file diff --git a/include/maat/maat.h b/include/maat/maat.h index 6ad2167..c8682e3 100644 --- a/include/maat/maat.h +++ b/include/maat/maat.h @@ -66,6 +66,10 @@ int maat_options_set_instance_name(struct maat_options *opts, const char *instan int maat_options_set_deferred_load_on(struct maat_options *opts); int maat_options_set_iris_full_dir(struct maat_options *opts, const char *full_dir); int maat_options_set_iris_inc_dir(struct maat_options *opts, const char *inc_dir); +int maat_options_set_json_file(struct maat_options *opts, const char *json_filename); +int maat_options_set_redis_ip(struct maat_options *opts, const char *redis_ip); +int maat_options_set_redis_port(struct maat_options *opts, uint16_t redis_port); +int maat_options_set_redis_db_index(struct maat_options *opts, int db_index); /* maat_instance API */ struct maat *maat_new(struct maat_options *opts, const char *table_info_path); diff --git a/include/utils.h b/include/utils.h index 295ce5d..9981a8f 100644 --- a/include/utils.h +++ b/include/utils.h @@ -20,6 +20,15 @@ extern "C" #define ALLOC(type, number) ((type *)calloc(sizeof(type), number)) +#define FREE(ptr) \ + { \ + if (ptr) \ + { \ + free(ptr); \ + ptr = NULL; \ + } \ + } + #ifdef __cpluscplus } #endif diff --git a/scanner/IPMatcher.h b/scanner/IPMatcher.h deleted file mode 100644 index 0e387f3..0000000 --- a/scanner/IPMatcher.h +++ /dev/null @@ -1,117 +0,0 @@ -/* - * - * Copyright (c) 2020 - * String Algorithms Research Group - * Institute of Information Engineering, Chinese Academy of Sciences (IIE-CAS) - * National Engineering Laboratory for Information Security Technologies (NELIST) - * All rights reserved - * - * Written by: LU YUHAI (luyuhai@iie.ac.cn) - * Last modification: 2020-04-20 - * - * This code is the exclusive and proprietary property of IIE-CAS and NELIST. - * Usage for direct or indirect commercial advantage is not allowed without - * written permission from the authors. - * - */ - -#ifndef H_IP_MATCHER_H -#define H_IP_MATCHER_H -#include - -#ifdef __cplusplus -extern "C" -{ -#endif - - enum IP_TYPE - { - IPv4, - IPv6 - }; - - /* 带掩码的单点IPv4规则 */ - struct ipv4_range - { - unsigned int start_ip; /* IP范围下界 */ - unsigned int end_ip; /* IP范围上界 */ - }; - - /* 带掩码的单点IPv6规则 */ - struct ipv6_range - { - unsigned int start_ip[4]; /* IP范围下界 */ - unsigned int end_ip[4]; /* IP范围上界 */ - }; - - /* 通用的ip规则类型 */ - struct ip_rule - { - enum IP_TYPE type; /* 规则类型，ipv4或ipv6 */ - unsigned int rule_id; /* 规则ID */ - void* user_tag; /* 用户自定义数据，命中时随匹配结果返回 */ - union - { - struct ipv4_range ipv4_rule; /*带掩码的单点IPv4规则*/ - struct ipv6_range ipv6_rule; /*带掩码的单点IPv6规则*/ - }; - }; - - /* 通用的待扫描数据类型 */ - struct ip_data - { - enum IP_TYPE type; /* 规则类型，ipv4或ipv6 */ - union /* 根据rule_type决定数据负载是ipv4还是ipv6 */ - { - unsigned int ipv4; /* ipv4数据*/ - unsigned int ipv6[4]; /* ipv6数据*/ - }; - }; - - - /* 布尔表达式的扫描结果类型 */ - struct scan_result - { - unsigned int rule_id; /* 规则的ID */ - void * tag; /* 用户自定义数据，命中时随匹配结果返回 */ - }; - - - struct ip_matcher; - - /* - 功能：根据输入的规则生成扫描器 - 参数： - rules[in]：一组ip规则 - rule_num[in]：输入的规则数量 - mem_use[out]：内存消耗量 - 返回值： - ip扫描器,返回空指针生成扫描器失败 - */ - struct ip_matcher* ip_matcher_new(struct ip_rule * rules, size_t rule_num, size_t * mem_use); - - /* - 功能：调用ip扫描器对输入的ip数据进行扫描 - 参数： - matcher[in]：ip扫描器 - data[in]：输入的待扫描ip数据 - result[in]：返回结果存储数组 - size[in]：结果数组的大小 - 返回值： - 命中结果的数量（<=size）；返回值为-1表示出错。 - - */ - int ip_matcher_match(struct ip_matcher* matcher, struct ip_data * data, struct scan_result* result, size_t size); - - /* - 功能：销毁一个ip扫描器 - 参数： - matcher[in]：待销毁的ip扫描器指针 - */ - void ip_matcher_free(struct ip_matcher* matcher); - -#ifdef __cplusplus -} -#endif - -#endif /* !defined(H_IP_MATCHER_H) */ diff --git a/scanner/adapter_hs.cpp b/scanner/adapter_hs.cpp index 2ff4a5f..c852325 100644 --- a/scanner/adapter_hs.cpp +++ b/scanner/adapter_hs.cpp @@ -184,7 +184,7 @@ void adpt_hs_compile_data_free(struct adpt_hs_compile_data *hs_cd, size_t n_patt struct adapter_hs *adapter_hs_initialize(int scan_mode, size_t nr_worker_threads, and_expr_t *expr_array, size_t n_expr_array) { - if ((scan_mode != SCAN_MODE_BLOCK && scan_mode != SCAN_MODE_STREAM) || + if ((scan_mode != HS_SCAN_MODE_BLOCK && scan_mode != HS_SCAN_MODE_STREAM) || 0 == nr_worker_threads || NULL == expr_array || 0 == n_expr_array) { fprintf(stderr, "%s input parameters illegal!", __func__); return NULL; @@ -407,11 +407,12 @@ int adapter_hs_scan(struct adapter_hs *hs_instance, int thread_id, const char *d utarray_new(pattern_id_set, &ut_pattern_id_icd); utarray_reserve(pattern_id_set, hs_instance->n_patterns); + int err_count = 0; if (hs_rt->literal_db != NULL) { err = hs_scan(hs_rt->literal_db, data, data_len, 0, scratch, matched_event_cb, pattern_id_set); if (err != HS_SUCCESS) { //log_error() - return -1; + err_count++; } } @@ -419,10 +420,14 @@ int adapter_hs_scan(struct adapter_hs *hs_instance, int thread_id, const char *d err = hs_scan(hs_rt->regex_db, data, data_len, 0, scratch, matched_event_cb, pattern_id_set); if (err != HS_SUCCESS) { //log_error() - return -1; + err_count++; } } + if (2 == err_count) { + return -1; + } + size_t pattern_set_size = utarray_len(pattern_id_set); unsigned long long items[pattern_set_size]; memset(items, 0, sizeof(unsigned long long) * pattern_set_size); diff --git a/scanner/adapter_hs.h b/scanner/adapter_hs.h index 3d6f9f3..1679b61 100644 --- a/scanner/adapter_hs.h +++ b/scanner/adapter_hs.h @@ -24,10 +24,10 @@ extern "C" struct adapter_hs; /* scan mode */ -enum scan_mode { - SCAN_MODE_BLOCK = 1, - SCAN_MODE_STREAM, - SCAN_MODE_MAX +enum hs_scan_mode { + HS_SCAN_MODE_BLOCK = 1, + HS_SCAN_MODE_STREAM, + HS_SCAN_MODE_MAX }; /* pattern type: PATTERN_TYPE_STR(pure literal string) or PATTERN_TYPE_REG(regex expression) */ diff --git a/scanner/adapter_hs_gtest.cpp b/scanner/adapter_hs_gtest.cpp index 1031f53..ef93983 100644 --- a/scanner/adapter_hs_gtest.cpp +++ b/scanner/adapter_hs_gtest.cpp @@ -66,11 +66,11 @@ TEST(block_mode_initialize, invalid_input_parameter) EXPECT_EQ(hs_instance, nullptr); /* case2: invalid expr parameter */ - hs_instance = adapter_hs_initialize(SCAN_MODE_BLOCK, 1, nullptr, 1); + hs_instance = adapter_hs_initialize(HS_SCAN_MODE_BLOCK, 1, nullptr, 1); EXPECT_EQ(hs_instance, nullptr); /* case3: invalid expr num */ - hs_instance = adapter_hs_initialize(SCAN_MODE_BLOCK, 1, exprs, 0); + hs_instance = adapter_hs_initialize(HS_SCAN_MODE_BLOCK, 1, exprs, 0); EXPECT_EQ(hs_instance, nullptr); } @@ -79,7 +79,7 @@ TEST(block_mode_scan, invalid_input_parameter) and_expr_t expr_array[64]; size_t n_expr_array = 0; - struct adapter_hs *hs_instance = adapter_hs_initialize(SCAN_MODE_BLOCK, 1, nullptr, 0); + struct adapter_hs *hs_instance = adapter_hs_initialize(HS_SCAN_MODE_BLOCK, 1, nullptr, 0); EXPECT_EQ(hs_instance, nullptr); hs_instance = adapter_hs_initialize(0, 1, expr_array, n_expr_array); @@ -88,7 +88,7 @@ TEST(block_mode_scan, invalid_input_parameter) n_expr_array = 1; expr_array[0].expr_id = 101; expr_array[0].n_patterns = 10; - hs_instance = adapter_hs_initialize(SCAN_MODE_BLOCK, 1, expr_array, n_expr_array); + hs_instance = adapter_hs_initialize(HS_SCAN_MODE_BLOCK, 1, expr_array, n_expr_array); EXPECT_EQ(hs_instance, nullptr); memset(expr_array, 0, sizeof(expr_array)); @@ -96,7 +96,7 @@ TEST(block_mode_scan, invalid_input_parameter) expr_array[0].expr_id = 101; expr_array[0].n_patterns = 1; expr_array[0].patterns[0].type = 0; - hs_instance = adapter_hs_initialize(SCAN_MODE_BLOCK, 1, expr_array, n_expr_array); + hs_instance = adapter_hs_initialize(HS_SCAN_MODE_BLOCK, 1, expr_array, n_expr_array); EXPECT_EQ(hs_instance, nullptr); } @@ -109,7 +109,7 @@ TEST(block_mode_scan, hit_one_expr) EXPECT_EQ(ret, 0); EXPECT_EQ(n_expr_array, 6); - struct adapter_hs *hs_instance = adapter_hs_initialize(SCAN_MODE_BLOCK, 1, expr_array, n_expr_array); + struct adapter_hs *hs_instance = adapter_hs_initialize(HS_SCAN_MODE_BLOCK, 1, expr_array, n_expr_array); EXPECT_NE(hs_instance, nullptr); expr_array_free(expr_array, n_expr_array); @@ -149,7 +149,7 @@ TEST(block_mode_scan, hit_two_expr) EXPECT_EQ(ret, 0); EXPECT_EQ(n_expr_array, 6); - struct adapter_hs *hs_instance = adapter_hs_initialize(SCAN_MODE_BLOCK, 1, expr_array, n_expr_array); + struct adapter_hs *hs_instance = adapter_hs_initialize(HS_SCAN_MODE_BLOCK, 1, expr_array, n_expr_array); EXPECT_NE(hs_instance, nullptr); expr_array_free(expr_array, n_expr_array); @@ -184,7 +184,7 @@ TEST(block_mode_scan, hit_three_expr) EXPECT_EQ(ret, 0); EXPECT_EQ(n_expr_array, 6); - struct adapter_hs *hs_instance = adapter_hs_initialize(SCAN_MODE_BLOCK, 1, expr_array, n_expr_array); + struct adapter_hs *hs_instance = adapter_hs_initialize(HS_SCAN_MODE_BLOCK, 1, expr_array, n_expr_array); EXPECT_NE(hs_instance, nullptr); expr_array_free(expr_array, n_expr_array); @@ -221,7 +221,7 @@ TEST(block_mode_scan, hit_four_expr) EXPECT_EQ(ret, 0); EXPECT_EQ(n_expr_array, 6); - struct adapter_hs *hs_instance = adapter_hs_initialize(SCAN_MODE_BLOCK, 1, expr_array, n_expr_array); + struct adapter_hs *hs_instance = adapter_hs_initialize(HS_SCAN_MODE_BLOCK, 1, expr_array, n_expr_array); EXPECT_NE(hs_instance, nullptr); expr_array_free(expr_array, n_expr_array); @@ -260,7 +260,7 @@ TEST(block_mode_scan, hit_five_expr) EXPECT_EQ(ret, 0); EXPECT_EQ(n_expr_array, 6); - struct adapter_hs *hs_instance = adapter_hs_initialize(SCAN_MODE_BLOCK, 1, expr_array, n_expr_array); + struct adapter_hs *hs_instance = adapter_hs_initialize(HS_SCAN_MODE_BLOCK, 1, expr_array, n_expr_array); EXPECT_NE(hs_instance, nullptr); expr_array_free(expr_array, n_expr_array); diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 1e1b481..a6c25d3 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -8,8 +8,9 @@ set(MAAT_FRAME_VERSION ${MAAT_FRAME_MAJOR_VERSION}.${MAAT_FRAME_MINOR_VERSION}.$ message(STATUS "Maat Frame, Version: ${MAAT_FRAME_VERSION}") add_definitions(-fPIC) -set(MAAT_SRC maat_api.cpp rcu_hash.cpp maat_garbage_collection.cpp maat_config_monitor.cpp maat_rule.cpp - maat_kv.cpp maat_ex_data.cpp maat_table_schema.cpp maat_table_runtime.cpp maat_utils.cpp) +set(MAAT_SRC json2iris.cpp maat_api.cpp rcu_hash.cpp maat_garbage_collection.cpp maat_config_monitor.cpp + maat_rule.cpp maat_kv.cpp maat_ex_data.cpp maat_table_schema.cpp maat_table_runtime.cpp maat_utils.cpp + maat_command.cpp maat_redis_monitor.cpp) set(LIB_SOURCE_FILES ${PROJECT_SOURCE_DIR}/deps/cJSON/cJSON.c) @@ -25,7 +26,7 @@ set_target_properties(maat_frame_static PROPERTIES LINKER_LANGUAGE CXX) set_target_properties(maat_frame_static PROPERTIES OUTPUT_NAME maatframe) set_target_properties(maat_frame_static PROPERTIES CLEAN_DIRECT_OUTPUT 1) -target_link_libraries(maat_frame_static adapter-static pthread) +target_link_libraries(maat_frame_static adapter-static hiredis-static pthread crypto z) # Shared Library Output #add_library(maat_frame_shared SHARED ${MAAT_SRC} ${LIB_SOURCE_FILES}) diff --git a/src/inc_internal/json2iris.h b/src/inc_internal/json2iris.h new file mode 100644 index 0000000..bfeca07 --- /dev/null +++ b/src/inc_internal/json2iris.h @@ -0,0 +1,29 @@ +/* +********************************************************************************************** +* File: json2iris.h +* Description: rule for transform json2iris +* Authors: Liu WenTan +* Date: 2022-10-31 +* Copyright: (c) 2018-2022 Geedge Networks, Inc. All rights reserved. +*********************************************************************************************** +*/ + +#ifndef _JSON2IRIS_H_ +#define _JSON2IRIS_H_ + +#ifdef __cpluscplus +extern "C" +{ +#endif + +#include "hiredis/hiredis.h" + +int json2iris(const char* json_buff, const char* json_filename, const char*compile_tn, + const char* group2compile_tn, const char* group2group_tn, redisContext *redis_write_ctx, + char* iris_dir_buf, int buf_len, char* encrypt_key, char* encrypt_algo); + +#ifdef __cpluscplus +} +#endif + +#endif \ No newline at end of file diff --git a/src/inc_internal/maat_command.h b/src/inc_internal/maat_command.h new file mode 100644 index 0000000..66a0636 --- /dev/null +++ b/src/inc_internal/maat_command.h @@ -0,0 +1,39 @@ +/* +********************************************************************************************** +* File: maat_command.h +* Description: +* Authors: Liu WenTan +* Date: 2022-10-31 +* Copyright: (c) 2018-2022 Geedge Networks, Inc. All rights reserved. +*********************************************************************************************** +*/ + +#ifndef _MAAT_COMMAND_H_ +#define _MAAT_COMMAND_H_ + +#ifdef __cpluscplus +extern "C" +{ +#endif + +enum maat_operation { + MAAT_OP_DEL = 0, + MAAT_OP_ADD, + MAAT_OP_RENEW_TIMEOUT //Rule expire time is changed to now+cmd->expire_after +}; + +struct maat_cmd_line +{ + const char *table_name; + const char *table_line; + int rule_id; // for MAAT_OP_DEL, only rule_id and table_name are necessary. + int expire_after; //expired after $timeout$ seconds, set to 0 for never timeout. +}; + +int maat_cmd_set_line(struct maat *maat_instance, const struct maat_cmd_line *line_rule); + +#ifdef __cpluscplus +} +#endif + +#endif \ No newline at end of file diff --git a/src/inc_internal/maat_common.h b/src/inc_internal/maat_common.h index 872120d..61aa499 100644 --- a/src/inc_internal/maat_common.h +++ b/src/inc_internal/maat_common.h @@ -18,6 +18,8 @@ extern "C" #include +#include "maat_rule.h" + struct maat_options { size_t nr_worker_threads; int rule_effect_interval_ms; @@ -25,8 +27,11 @@ struct maat_options { int gc_timeout_ms; int deferred_load_on; enum data_source input_mode; - char iris_full_dir[NAME_MAX]; - char iris_inc_dir[NAME_MAX]; + union { + struct source_iris_ctx iris_ctx; + struct source_json_ctx json_ctx; + struct source_redis_ctx redis_ctx; + }; }; #ifdef __cpluscplus diff --git a/src/inc_internal/maat_config_monitor.h b/src/inc_internal/maat_config_monitor.h index 46878ac..eb3de29 100644 --- a/src/inc_internal/maat_config_monitor.h +++ b/src/inc_internal/maat_config_monitor.h @@ -28,6 +28,8 @@ void config_monitor_traverse(long long version, const char *idx_dir, void (*finish_fn)(void *), void *u_param); +int load_maat_json_file(struct maat *maat_instance, const char *json_filename, char *err_str, size_t err_str_sz); + #ifdef __cpluscplus } #endif diff --git a/src/inc_internal/maat_redis_monitor.h b/src/inc_internal/maat_redis_monitor.h new file mode 100644 index 0000000..7bb4800 --- /dev/null +++ b/src/inc_internal/maat_redis_monitor.h @@ -0,0 +1,33 @@ +/* +********************************************************************************************** +* File: maat_redis_monitor.h +* Description: maat redis monitor api +* Authors: Liu WenTan +* Date: 2022-11-29 +* Copyright: (c) 2018-2022 Geedge Networks, Inc. All rights reserved. +*********************************************************************************************** +*/ + +#ifndef _MAAT_REDIS_MONITOR_H_ +#define _MAAT_REDIS_MONITOR_H_ + +#ifdef __cpluscplus +extern "C" +{ +#endif + +#include "maat_rule.h" + +#include + +void redis_monitor_traverse(long long version, struct source_redis_ctx* mr_ctx, + void (*start_fn)(long long, int, void *), + int (*update_fn)(const char *, const char *, void *), + void (*finish_fn)(void *), + void *u_param); + +#ifdef __cpluscplus +} +#endif + +#endif \ No newline at end of file diff --git a/src/inc_internal/maat_rule.h b/src/inc_internal/maat_rule.h index 808d95d..6e7cec6 100644 --- a/src/inc_internal/maat_rule.h +++ b/src/inc_internal/maat_rule.h @@ -21,6 +21,13 @@ extern "C" #include #include #include +#include +#include + +#include "hiredis/hiredis.h" +#include "uthash/uthash.h" +#include "maat_table_schema.h" +#include "maat_command.h" struct maat_runtime { /* maat_runtime can be created and destroy dynamic, so need version info */ @@ -37,7 +44,9 @@ struct maat_runtime { enum data_source { DATA_SOURCE_NONE = 0, - DATA_SOURCE_IRIS_FILE + DATA_SOURCE_REDIS, + DATA_SOURCE_IRIS_FILE, + DATA_SOURCE_JSON_FILE }; struct source_iris_ctx { @@ -45,6 +54,52 @@ struct source_iris_ctx { char full_dir[NAME_MAX]; }; +struct source_json_ctx +{ + char json_file[NAME_MAX]; + char iris_file[NAME_MAX]; + char effective_json_md5[MD5_DIGEST_LENGTH*2+1]; + struct timespec last_md5_time; +}; + +struct source_redis_ctx +{ + redisContext *read_ctx; + redisContext *write_ctx; + char redis_ip[64]; + uint16_t redis_port; + int redis_db; + time_t last_reconnect_time; +}; + +struct foreign_key { + int column; + char *key; + size_t key_len; + char *filename; +}; + +//rm= Redis Maat +struct serial_rule { + enum maat_operation op;//0: delete, 1: add. + unsigned long rule_id; + int label_id; + long long timeout; // absolute unix time. + char table_name[NAME_MAX]; + char *table_line; + int n_foreign; + struct foreign_key *f_keys; + TAILQ_ENTRY(serial_rule) entries; + UT_hash_handle hh; +}; + +#define POSSIBLE_REDIS_REPLY_SIZE 2 +struct expected_reply { + int s_rule_seq; + int possible_reply_num; + redisReply possible_replies[POSSIBLE_REDIS_REPLY_SIZE]; +}; + struct maat { char instance_name[NAME_MAX]; @@ -56,6 +111,8 @@ struct maat { enum data_source input_mode; union { struct source_iris_ctx iris_ctx; + struct source_json_ctx json_ctx; + struct source_redis_ctx mr_ctx; }; int deferred_load; @@ -72,7 +129,24 @@ struct maat { int rule_update_checking_interval_ms; int gc_timeout_ms; //garbage collection timeout_ms; + int cumulative_update_off; //Default: cumulative update on + struct maat_garbage_bin *garbage_bin; + + char compile_tn[NAME_MAX]; + char group_tn[NAME_MAX]; + char group2compile_tn[NAME_MAX]; + char group2group_tn[NAME_MAX]; + + char decrypt_key[NAME_MAX]; + char decrypt_algo[NAME_MAX]; + int maat_json_is_gzipped; + + long long load_specific_version; //Default: Load the Latest. Only valid in redis mode, and maybe failed for too old + char foreign_cont_dir[NAME_MAX]; + + /* statistics */ + long long line_cmd_acc_num; }; void maat_start_cb(long long new_version, int update_type, void *u_para); @@ -85,6 +159,40 @@ void *rule_monitor_loop(void *arg); void maat_read_full_config(struct maat *maat_instance); +/* maat command API for internal */ +redisContext *maat_cmd_connect_redis(const char *redis_ip, int redis_port, int redis_db); + +redisReply *maat_cmd_wrap_redis_command(redisContext *c, const char *format, ...); + +int maat_cmd_wrap_redis_get_reply(redisContext *c, redisReply **reply); + +long long maat_cmd_redis_server_time_s(redisContext *c); + +long long maat_cmd_read_redis_integer(const redisReply *reply); + +int maat_cmd_get_valid_flag_offset(const char *line, enum table_type table_type, int valid_column_seq); + +const char *maat_cmd_find_Nth_column(const char *line, int Nth, int *column_len); + +int maat_cmd_exec_serial_rule(redisContext *c, struct serial_rule *s_rule, size_t serial_rule_num, long long server_time); + +void maat_cmd_empty_serial_rule(struct serial_rule *s_rule); + +int maat_cmd_get_rm_key_list(redisContext *c, long long instance_version, long long desired_version, + long long *new_version, struct table_schema_manager* table_schema_mgr, + struct serial_rule **list, int *update_type, int cumulative_off); + +int maat_cmd_get_redis_value(redisContext *c, struct serial_rule *rule_list, int rule_num, int print_process); + +int maat_cmd_get_foreign_keys_by_prefix(redisContext *ctx, struct serial_rule *rule_list, int rule_num, const char* dir); + +void maat_cmd_get_foreign_conts(redisContext *ctx, struct serial_rule *rule_list, int rule_num, int print_fn); + +void maat_cmd_rewrite_table_line_with_foreign(struct serial_rule *s_rule); + +void maat_cmd_set_serial_rule(struct serial_rule *rule, enum maat_operation op, unsigned long rule_id, + const char *table_name, const char *line, long long timeout); + #ifdef __cpluscplus } #endif diff --git a/src/inc_internal/maat_table_schema.h b/src/inc_internal/maat_table_schema.h index 570b591..739cef6 100644 --- a/src/inc_internal/maat_table_schema.h +++ b/src/inc_internal/maat_table_schema.h @@ -25,13 +25,28 @@ extern "C" #define MAX_DISTRICT_STR 128 #define MAX_IP_STR 128 #define MAX_KEYWORDS_STR 1024 +#define MAX_FOREIGN_CLMN_NUM 8 enum table_type { TABLE_TYPE_EXPR = 0, TABLE_TYPE_EXPR_PLUS, TABLE_TYPE_IP, + TABLE_TYPE_IP_PLUS, + TABLE_TYPE_INTERVAL, + TABLE_TYPE_INTERVAL_PLUS, + TABLE_TYPE_DIGEST, + TABLE_TYPE_SIMILARITY, TABLE_TYPE_PLUGIN, TABLE_TYPE_IP_PLUGIN, + TABLE_TYPE_FQDN_PLUGIN, + TABLE_TYPE_BOOL_PLUGIN, + //above are physical table + TABLE_TYPE_VIRTUAL, + TABLE_TYPE_COMPOSITION, + TABLE_TYPE_COMPILE, + TABLE_TYPE_GROUP, + TABLE_TYPE_GROUP2GROUP, + TABLE_TYPE_GROUP2COMPILE, TABLE_TYPE_MAX }; @@ -42,8 +57,21 @@ enum expr_type { EXPR_TYPE_MAX }; +enum scan_type { + SCAN_TYPE_INVALID = -1, + SCAN_TYPE_NONE = 0, + SCAN_TYPE_PLUGIN, + SCAN_TYPE_IP_PLUGIN, + SCAN_TYPE_FQDN_PLUGIN, + SCAN_TYPE_BOOL_PLUGIN, + SCAN_TYPE_IP, + SCAN_TYPE_INTERVAL, + SCAN_TYPE_STRING, + SCAN_TYPE_MAX +}; + enum match_method { - MATCH_METHOD_SUB=0, + MATCH_METHOD_SUB = 0, MATCH_METHOD_RIGHT, MATCH_METHOD_LEFT, MATCH_METHOD_COMPLETE, @@ -60,10 +88,6 @@ struct expr_item { int is_hexbin; int is_case_sensitive; int is_valid; - - //rule_tag; 鍙瓨鍦╯chema閲 - //int have_exdata; - //struct ex_data *ex_data; //hash琛 }; struct plugin_item { @@ -129,14 +153,21 @@ void table_schema_manager_all_plugin_cb_finish(struct table_schema_manager* tabl /* table schema generic API */ struct table_schema *table_schema_get(struct table_schema_manager *table_schema_mgr, int table_id); +struct table_schema *table_schema_get_by_scan_type(struct table_schema_manager *table_schema_mgr, + int table_id, enum scan_type type, int *virtual_table_id); + enum table_type table_schema_get_table_type(struct table_schema *table_schema); int table_schema_get_table_id(struct table_schema *table_schema); +enum scan_type table_schema_get_scan_type(struct table_schema *table_schema); + struct table_item *table_schema_line_to_item(const char *line, struct table_schema *table_schema); +int table_schema_get_valid_flag_column(struct table_schema *table_schema); + /* expr table schema API */ -enum scan_mode expr_table_schema_get_scan_mode(struct table_schema *table_schema); +enum hs_scan_mode expr_table_schema_get_scan_mode(struct table_schema *table_schema); /* plugin table schema API */ int plugin_table_schema_set_ex_data_schema(struct table_schema *table_schema, @@ -165,6 +196,8 @@ size_t plugin_table_schema_callback_count(struct table_schema *table_schema); void plugin_table_schema_all_cb_update(struct table_schema *table_schema, const char *row); +int plugin_table_schema_get_foreign_column(struct table_schema *table_schema, int *foreign_columns); + #ifdef __cpluscplus } #endif diff --git a/src/inc_internal/maat_utils.h b/src/inc_internal/maat_utils.h index 3d26770..42f0824 100644 --- a/src/inc_internal/maat_utils.h +++ b/src/inc_internal/maat_utils.h @@ -30,6 +30,8 @@ extern "C" #define MIN(a, b) (((a) < (b)) ? (a) : (b)) #endif +#define UNUSED __attribute__((unused)) + char *maat_strdup(const char *s); int get_column_pos(const char *line, int column_seq, size_t *offset, size_t *len); @@ -42,6 +44,19 @@ char *str_unescape_and(char *s); char *str_unescape(char *s); +char *md5_file(const char *filename, char *md5string); + +int decrypt_open(const char* file_name, const char* key, const char* algorithm, + unsigned char**pp_out, size_t *out_sz, char* err_str, size_t err_str_sz); + +int crypt_memory(const unsigned char *inbuf, size_t inlen, unsigned char **pp_out, size_t *out_sz, + const char *key, const char *algorithm, int do_encrypt, char *err_str, size_t err_str_sz); + +int gzip_uncompress(const unsigned char *in_compressed_data, size_t in_compressed_sz, + unsigned char **out_uncompressed_data, size_t *out_uncompressed_sz); + +size_t memcat(void **dest, size_t offset, size_t *n_dest, const void *src, size_t n_src); + /* system cmd wrapper */ int system_cmd_mkdir(const char* path); diff --git a/src/json2iris.cpp b/src/json2iris.cpp new file mode 100644 index 0000000..46cc7a6 --- /dev/null +++ b/src/json2iris.cpp @@ -0,0 +1,1207 @@ +/* +********************************************************************************************** +* File: json2iris.h +* Description: rule for transform json2iris +* Authors: Liu WenTan +* Date: 2022-10-31 +* Copyright: (c) 2018-2022 Geedge Networks, Inc. All rights reserved. +*********************************************************************************************** +*/ + +#include +#include +#include +#include +#include +#include + +#include "utils.h" +#include "json2iris.h" +#include "cJSON/cJSON.h" +#include "maat_kv.h" +#include "maat_utils.h" +#include "maat_table_schema.h" +#include "maat_rule.h" +#include "uthash/uthash.h" + +#define MAX_COLUMN_NUM 32 + +#define mr_region_id_var "SEQUENCE_REGION" +#define mr_group_id_var "SEQUENCE_GROUP" + +const int json_version = 1; +const char *untitled_group_name="Untitled"; + +enum maat_group_relation { + PARENT_TYPE_COMPILE = 0, + PARENT_TYPE_GROUP +}; + +struct group_info { + int group_id; + char group_name[PATH_MAX]; + UT_hash_handle hh; +}; + +struct iris_table { + char table_name[PATH_MAX]; + char table_path[PATH_MAX]; + int line_count; + enum table_type table_type; + void *buff; + size_t write_pos; + size_t buff_sz; + UT_hash_handle hh; +}; + +struct iris_description { + int group_cnt; + int region_cnt; + + char tmp_iris_dir[PATH_MAX]; + char tmp_iris_index_dir[PATH_MAX]; + char index_path[PATH_MAX]; + + struct iris_table *group_table; + struct iris_table *group2group_table; + struct iris_table *group2compile_table; + struct iris_table *compile_table; + + struct group_info *group_name_map; + struct iris_table *iris_table_map; + + struct maat_kv_store *str2int_map; + redisContext *redis_write_ctx; + + char *encrypt_key; + char *encrypt_algo; + FILE *idx_fp; +}; + +struct translate_command { + const char *json_string; + char *json_value; + int json_type; + int str2int_flag; + int empty_allowed; + const char *default_string; + int default_int; +}; + +struct iris_table *query_table_info(struct iris_description *p_iris, const char *table_name, enum table_type table_type) +{ + struct iris_table *table_info = NULL; + HASH_FIND(hh, p_iris->iris_table_map, table_name, strlen(table_name), table_info); + if (NULL == table_info) { + table_info = ALLOC(struct iris_table, 1); + table_info->line_count = 0; + table_info->table_type = table_type; + memcpy(table_info->table_name, table_name, MIN(sizeof(table_info->table_name) - 1, strlen(table_name))); + snprintf(table_info->table_path, sizeof(table_info->table_path), "%s/%s.local", p_iris->tmp_iris_dir, table_info->table_name); + HASH_ADD_KEYPTR(hh, p_iris->iris_table_map, table_name, strlen(table_name), table_info); + } + + return table_info; +} + +void free_iris_table_info(void *p) +{ + struct iris_table *table = (struct iris_table *)p; + if (table != NULL) { + FREE(table->buff); + } + FREE(table); +} + +int set_iris_descriptor(const char *json_file, cJSON *json, const char *encrypt_key, const char *encrypt_algo, + const char *compile_tn, const char *group2compile_tn, const char* group2group_tn, + redisContext *redis_write_ctx, struct iris_description *iris_cfg) +{ + memset(iris_cfg, 0, sizeof(struct iris_description)); + snprintf(iris_cfg->tmp_iris_dir, sizeof(iris_cfg->tmp_iris_dir), "%s_iris_tmp", json_file); + snprintf(iris_cfg->tmp_iris_index_dir, sizeof(iris_cfg->tmp_iris_index_dir), "%s_iris_tmp/index", json_file); + snprintf(iris_cfg->index_path, sizeof(iris_cfg->index_path), "%s/full_config_index.%010d", iris_cfg->tmp_iris_index_dir, json_version); + + iris_cfg->redis_write_ctx = redis_write_ctx; + iris_cfg->str2int_map = maat_kv_store_new(); + + maat_kv_register(iris_cfg->str2int_map, "yes", 1); + maat_kv_register(iris_cfg->str2int_map, "no", 0); + + maat_kv_register(iris_cfg->str2int_map, "ip", TABLE_TYPE_IP); + maat_kv_register(iris_cfg->str2int_map, "ip_plus", TABLE_TYPE_IP_PLUS); + maat_kv_register(iris_cfg->str2int_map, "string", TABLE_TYPE_EXPR); + maat_kv_register(iris_cfg->str2int_map, "expr", TABLE_TYPE_EXPR); + maat_kv_register(iris_cfg->str2int_map, "expr_plus", TABLE_TYPE_EXPR_PLUS); + maat_kv_register(iris_cfg->str2int_map, "intval", TABLE_TYPE_INTERVAL); + maat_kv_register(iris_cfg->str2int_map, "interval", TABLE_TYPE_INTERVAL); + maat_kv_register(iris_cfg->str2int_map, "intval_plus", TABLE_TYPE_INTERVAL_PLUS); + maat_kv_register(iris_cfg->str2int_map, "interval_plus", TABLE_TYPE_INTERVAL_PLUS); + maat_kv_register(iris_cfg->str2int_map, "digest", TABLE_TYPE_DIGEST); + maat_kv_register(iris_cfg->str2int_map, "similar", TABLE_TYPE_SIMILARITY); + + maat_kv_register(iris_cfg->str2int_map, "ipv4", 4); + maat_kv_register(iris_cfg->str2int_map, "ipv6", 6); + + maat_kv_register(iris_cfg->str2int_map, "double", 0); + maat_kv_register(iris_cfg->str2int_map, "single", 1); + + maat_kv_register(iris_cfg->str2int_map, "none", 0); + maat_kv_register(iris_cfg->str2int_map, "and", 1); + maat_kv_register(iris_cfg->str2int_map, "regex", 2); + maat_kv_register(iris_cfg->str2int_map, "offset", 3); + + maat_kv_register(iris_cfg->str2int_map, "sub", 0); + maat_kv_register(iris_cfg->str2int_map, "right", 1); + maat_kv_register(iris_cfg->str2int_map, "suffix", 1); + maat_kv_register(iris_cfg->str2int_map, "left", 2); + maat_kv_register(iris_cfg->str2int_map, "prefix", 2); + maat_kv_register(iris_cfg->str2int_map, "complete", 3); + maat_kv_register(iris_cfg->str2int_map, "exact", 3); + + maat_kv_register(iris_cfg->str2int_map, "uncase plain", 0); + maat_kv_register(iris_cfg->str2int_map, "hexbin", 1); + maat_kv_register(iris_cfg->str2int_map, "case plain", 2); + + iris_cfg->compile_table = query_table_info(iris_cfg, compile_tn, TABLE_TYPE_COMPILE); + iris_cfg->group2compile_table = query_table_info(iris_cfg, group2compile_tn, TABLE_TYPE_GROUP2COMPILE); + iris_cfg->group2group_table = query_table_info(iris_cfg, group2group_tn, TABLE_TYPE_GROUP2GROUP); + + if (encrypt_key && encrypt_algo) { + iris_cfg->encrypt_key = maat_strdup(encrypt_key); + iris_cfg->encrypt_algo = maat_strdup(encrypt_algo); + } + + return 0; +} + +void clear_iris_descriptor(struct iris_description *iris_cfg) +{ + if (iris_cfg->group_name_map != NULL) { + struct group_info *node = NULL; + struct group_info *tmp = NULL; + HASH_ITER(hh, iris_cfg->group_name_map, node, tmp) { + HASH_DELETE(hh, iris_cfg->group_name_map, node); + FREE(node); + } + } + + if(iris_cfg->iris_table_map != NULL) { + struct iris_table *node = NULL; + struct iris_table *tmp = NULL; + HASH_ITER(hh, iris_cfg->iris_table_map, node, tmp) { + HASH_DELETE(hh, iris_cfg->iris_table_map, node); + free_iris_table_info(node); + } + } + + maat_kv_store_free(iris_cfg->str2int_map); + FREE(iris_cfg->encrypt_algo); + FREE(iris_cfg->encrypt_key); + + return; +} + +int create_tmp_dir(struct iris_description *p) +{ + if ((access(p->tmp_iris_dir,F_OK)) < 0) { + if ((mkdir(p->tmp_iris_dir, 0777)) < 0) { + return -1; + } + } + + if ((access(p->tmp_iris_index_dir,F_OK)) < 0) { + if ((mkdir(p->tmp_iris_index_dir, 0777)) < 0) { + return -1; + } + } + + return 0; +} + +int write_plugin_line(cJSON *plug_table_json, int sequence, struct iris_description *p_iris) +{ + cJSON *item = cJSON_GetObjectItem(plug_table_json, "table_name"); + if (NULL == item || item->type != cJSON_String) { + fprintf(stderr, "The %d plugin_table's table_name not defined or format error.", sequence); + return -1; + } + const char *table_name = item->valuestring; + + cJSON *table_content = cJSON_GetObjectItem(plug_table_json, "table_content"); + if (NULL == table_content || table_content->type != cJSON_Array) { + fprintf(stderr, "%d plugin_table's table_content not defined or format error.", sequence); + return -1; + } + int line_cnt = cJSON_GetArraySize(table_content); + + struct iris_table *table_info = query_table_info(p_iris, table_name, TABLE_TYPE_PLUGIN); + + cJSON *each_line = NULL; + const char *line_content = NULL; + for (int i = 0; i < line_cnt; i++) { + each_line = cJSON_GetArrayItem(table_content, i); + if (NULL == each_line || each_line->type != cJSON_String) { + fprintf(stderr, "plugin_table %s's line %d format error.", table_info->table_name, i+1); + continue; + } + + line_content = each_line->valuestring; + table_info->write_pos += memcat(&(table_info->buff), table_info->write_pos, &(table_info->buff_sz), line_content, strlen(line_content)); + table_info->write_pos += memcat(&(table_info->buff), table_info->write_pos, &(table_info->buff_sz), "\n", 1); + table_info->line_count++; + } + + return 0; +} + +static struct group_info *group_info_read(struct group_info *group_name_map, const char *group_name) +{ + struct group_info *node = NULL; + HASH_FIND(hh, group_name_map, group_name, strlen(group_name), node); + + return node; +} + +static int get_group_seq(struct iris_description *iris_cfg) +{ + redisReply *data_reply=NULL; + int sequence = 0; + + if (NULL == iris_cfg->redis_write_ctx) { + sequence = iris_cfg->group_cnt; + } else { + data_reply = maat_cmd_wrap_redis_command(iris_cfg->redis_write_ctx, "INCRBY %s 1", mr_group_id_var); + sequence = (int)data_reply->integer - 1; + freeReplyObject(data_reply); + data_reply = NULL; + } + iris_cfg->group_cnt++; + + return sequence; +} + +static struct group_info *group_info_add_unsafe(struct iris_description *p_iris, const char *group_name) +{ + static struct group_info untitled_group; + struct group_info *group_info = NULL; + + if (0 == strncasecmp(group_name, untitled_group_name, strlen(untitled_group_name))) { + group_info = &untitled_group; + group_info->group_id = get_group_seq(p_iris); + } else { + group_info = ALLOC(struct group_info, 1); + group_info->group_id = get_group_seq(p_iris); + strncpy(group_info->group_name, group_name, sizeof(group_info->group_name)); + HASH_ADD_KEYPTR(hh, p_iris->group_name_map, group_name, strlen(group_name), group_info); + } + return group_info; +} + +static int get_region_seq(struct iris_description *iris_cfg) +{ + int sequence = 0; + + if (NULL == iris_cfg->redis_write_ctx) { + sequence = iris_cfg->region_cnt; + } else { + redisReply *data_reply = maat_cmd_wrap_redis_command(iris_cfg->redis_write_ctx, "INCRBY %s 1", mr_region_id_var); + sequence = (int)data_reply->integer - 1; + freeReplyObject(data_reply); + } + iris_cfg->region_cnt++; + + return sequence; +} + +int direct_write_rule(cJSON *json, struct maat_kv_store *str2int, struct translate_command *cmd, int cmd_cnt, struct iris_table *table) +{ + int i = 0; + int ret = -1; + + for (i = 0; i < cmd_cnt; i++) { + cJSON *item = cJSON_GetObjectItem(json, cmd[i].json_string); + if (NULL == item && (1 == cmd[i].empty_allowed)) { + cJSON dummy; + dummy.valuestring = (char *)cmd[i].default_string; + dummy.valueint = cmd[i].default_int; + dummy.valuedouble = cmd[i].default_int; + dummy.type = cmd[i].json_type; + item = &dummy; + } + + if (NULL == item || item->type != cmd[i].json_type) { + fprintf(stderr, "%s not defined or wrong format.", cmd[i].json_string); + ret = -1; + goto error_out; + } + + if (1 == cmd[i].str2int_flag) { + int int_value = 0; + char *p = item->valuestring; + ret = maat_kv_read(str2int, p, &int_value); + if (ret < 0) { + fprintf(stderr, "%s's value %s is not valid format.", cmd[i].json_string, p); + FREE(p); + ret = -1; + goto error_out; + } + cmd[i].json_value = (char *)malloc(21);/* 2^64+1 can be represented in 21 chars. */ + snprintf(cmd[i].json_value, 21, "%d", int_value); + } else { + switch (item->type) { + case cJSON_Number: + cmd[i].json_value = cJSON_Print(item); + break; + case cJSON_String: + cmd[i].json_value = ALLOC(char, strlen(item->valuestring) + 1); + memcpy(cmd[i].json_value, item->valuestring, strlen(item->valuestring)); + break; + default://impossible ,already checked + assert(0); + break; + } + } + } + + for (i = 0; i < cmd_cnt; i++) { + table->write_pos += memcat(&(table->buff), table->write_pos, &table->buff_sz, cmd[i].json_value, strlen(cmd[i].json_value)); + table->write_pos += memcat(&(table->buff), table->write_pos, &table->buff_sz, "\t", 1); + } + table->write_pos += memcat(&(table->buff), table->write_pos, &table->buff_sz, "\n", 1); + table->line_count++; + ret = 0; +error_out: + for (i = 0; i < cmd_cnt; i++) { + if (cmd[i].json_value != NULL) { + FREE(cmd[i].json_value); + } + } + return ret; +} + +int write_expr_line(cJSON *region_json, struct iris_description *p_iris, struct iris_table *table) +{ + struct translate_command json_cmd[MAX_COLUMN_NUM]; + int cmd_cnt = 0; + memset(json_cmd, 0, sizeof(json_cmd)); + + json_cmd[cmd_cnt].json_string = "region_id"; + json_cmd[cmd_cnt].json_type = cJSON_Number; + cmd_cnt++; + + json_cmd[cmd_cnt].json_string = "group_id"; + json_cmd[cmd_cnt].json_type = cJSON_Number; + cmd_cnt++; + + if (table->table_type==TABLE_TYPE_EXPR_PLUS) { + json_cmd[cmd_cnt].json_string = "district"; + json_cmd[cmd_cnt].json_type = cJSON_String; + cmd_cnt++; + } + + json_cmd[cmd_cnt].json_string = "keywords"; + json_cmd[cmd_cnt].json_type = cJSON_String; + cmd_cnt++; + + json_cmd[cmd_cnt].json_string = "expr_type"; + json_cmd[cmd_cnt].json_type = cJSON_String; + json_cmd[cmd_cnt].str2int_flag = 1; + cmd_cnt++; + + json_cmd[cmd_cnt].json_string = "match_method"; + json_cmd[cmd_cnt].json_type = cJSON_String; + json_cmd[cmd_cnt].str2int_flag = 1; + cmd_cnt++; + + json_cmd[cmd_cnt].json_string = "format"; + json_cmd[cmd_cnt].json_type = cJSON_String; + json_cmd[cmd_cnt].str2int_flag = 1; + cmd_cnt++; + + json_cmd[cmd_cnt].json_string = "is_valid"; + json_cmd[cmd_cnt].json_type = cJSON_Number; + cmd_cnt++; + + return direct_write_rule(region_json, p_iris->str2int_map, json_cmd, cmd_cnt, table); +} + +int write_ip_line(cJSON *region_json, struct iris_description *p_iris, struct iris_table *table) +{ + struct translate_command json_cmd[MAX_COLUMN_NUM]; + int cmd_cnt = 0; + memset(json_cmd, 0, sizeof(json_cmd)); + + json_cmd[cmd_cnt].json_string = "region_id"; + json_cmd[cmd_cnt].json_type = cJSON_Number; + cmd_cnt++; + + json_cmd[cmd_cnt].json_string = "group_id"; + json_cmd[cmd_cnt].json_type = cJSON_Number; + cmd_cnt++; + + json_cmd[cmd_cnt].json_string = "addr_type"; + json_cmd[cmd_cnt].json_type = cJSON_String; + json_cmd[cmd_cnt].str2int_flag = 1; + cmd_cnt++; + + json_cmd[cmd_cnt].json_string = "src_ip"; + json_cmd[cmd_cnt].json_type = cJSON_String; + json_cmd[cmd_cnt].empty_allowed = 1; + json_cmd[cmd_cnt].default_string = "0.0.0.0"; + cmd_cnt++; + + json_cmd[cmd_cnt].json_string = "mask_src_ip"; + json_cmd[cmd_cnt].json_type = cJSON_String; + json_cmd[cmd_cnt].empty_allowed = 1; + json_cmd[cmd_cnt].default_string = "255.255.255.255"; + cmd_cnt++; + + json_cmd[cmd_cnt].json_string = "src_port"; + json_cmd[cmd_cnt].json_type = cJSON_String; + json_cmd[cmd_cnt].empty_allowed = 1; + json_cmd[cmd_cnt].default_string = "0"; + cmd_cnt++; + + json_cmd[cmd_cnt].json_string = "mask_src_port"; + json_cmd[cmd_cnt].json_type = cJSON_String; + json_cmd[cmd_cnt].empty_allowed = 1; + json_cmd[cmd_cnt].default_string = "65535"; + cmd_cnt++; + + json_cmd[cmd_cnt].json_string = "dst_ip"; + json_cmd[cmd_cnt].json_type = cJSON_String; + json_cmd[cmd_cnt].empty_allowed = 1; + json_cmd[cmd_cnt].default_string = "0.0.0.0"; + cmd_cnt++; + + json_cmd[cmd_cnt].json_string = "mask_dst_ip"; + json_cmd[cmd_cnt].json_type = cJSON_String; + json_cmd[cmd_cnt].empty_allowed = 1; + json_cmd[cmd_cnt].default_string = "255.255.255.255"; + cmd_cnt++; + + json_cmd[cmd_cnt].json_string = "dst_port"; + json_cmd[cmd_cnt].json_type = cJSON_String; + json_cmd[cmd_cnt].empty_allowed = 1; + json_cmd[cmd_cnt].default_string = "0"; + cmd_cnt++; + + json_cmd[cmd_cnt].json_string = "mask_dst_port"; + json_cmd[cmd_cnt].json_type = cJSON_String; + json_cmd[cmd_cnt].empty_allowed = 1; + json_cmd[cmd_cnt].default_string = "65535"; + cmd_cnt++; + + json_cmd[cmd_cnt].json_string = "protocol"; + json_cmd[cmd_cnt].json_type = cJSON_Number; + json_cmd[cmd_cnt].empty_allowed = 1; + json_cmd[cmd_cnt].default_int = 0; + cmd_cnt++; + + json_cmd[cmd_cnt].json_string = "direction"; + json_cmd[cmd_cnt].json_type = cJSON_String; + json_cmd[cmd_cnt].str2int_flag = 1; + json_cmd[cmd_cnt].empty_allowed = 1; + json_cmd[cmd_cnt].default_string = "double"; + cmd_cnt++; + + json_cmd[cmd_cnt].json_string = "is_valid"; + json_cmd[cmd_cnt].json_type = cJSON_Number; + cmd_cnt++; + + return direct_write_rule(region_json, p_iris->str2int_map, json_cmd, cmd_cnt, table); +} + +int write_ip_plus_line(cJSON *region_json, struct iris_description *p_iris, struct iris_table *table) +{ + struct translate_command json_cmd[MAX_COLUMN_NUM]; + int cmd_cnt = 0; + memset(json_cmd, 0, sizeof(json_cmd)); + + json_cmd[cmd_cnt].json_string = "region_id"; + json_cmd[cmd_cnt].json_type = cJSON_Number; + cmd_cnt++; + + json_cmd[cmd_cnt].json_string = "group_id"; + json_cmd[cmd_cnt].json_type = cJSON_Number; + cmd_cnt++; + + json_cmd[cmd_cnt].json_string = "addr_type"; + json_cmd[cmd_cnt].json_type = cJSON_String; + json_cmd[cmd_cnt].str2int_flag = 1; + cmd_cnt++; + + json_cmd[cmd_cnt].json_string = "saddr_format"; + json_cmd[cmd_cnt].json_type = cJSON_String; + json_cmd[cmd_cnt].empty_allowed = 1; + json_cmd[cmd_cnt].default_string = "mask"; + cmd_cnt++; + + json_cmd[cmd_cnt].json_string = "src_ip1"; + json_cmd[cmd_cnt].json_type = cJSON_String; + json_cmd[cmd_cnt].empty_allowed = 1; + json_cmd[cmd_cnt].default_string = "0.0.0.0"; + cmd_cnt++; + + json_cmd[cmd_cnt].json_string = "src_ip2"; + json_cmd[cmd_cnt].json_type = cJSON_String; + json_cmd[cmd_cnt].empty_allowed = 1; + json_cmd[cmd_cnt].default_string = "255.255.255.255"; + cmd_cnt++; + + json_cmd[cmd_cnt].json_string = "sport_format"; + json_cmd[cmd_cnt].json_type = cJSON_String; + json_cmd[cmd_cnt].empty_allowed = 1; + json_cmd[cmd_cnt].default_string = "mask"; + cmd_cnt++; + + json_cmd[cmd_cnt].json_string = "src_port1"; + json_cmd[cmd_cnt].json_type = cJSON_String; + json_cmd[cmd_cnt].empty_allowed = 1; + json_cmd[cmd_cnt].default_string = "0"; + cmd_cnt++; + + json_cmd[cmd_cnt].json_string = "src_port2"; + json_cmd[cmd_cnt].json_type = cJSON_String; + json_cmd[cmd_cnt].empty_allowed = 1; + json_cmd[cmd_cnt].default_string = "65535"; + cmd_cnt++; + + json_cmd[cmd_cnt].json_string = "daddr_format"; + json_cmd[cmd_cnt].json_type = cJSON_String; + json_cmd[cmd_cnt].empty_allowed = 1; + json_cmd[cmd_cnt].default_string = "mask"; + cmd_cnt++; + + json_cmd[cmd_cnt].json_string = "dst_ip1"; + json_cmd[cmd_cnt].json_type = cJSON_String; + json_cmd[cmd_cnt].empty_allowed = 1; + json_cmd[cmd_cnt].default_string = "0.0.0.0"; + cmd_cnt++; + + json_cmd[cmd_cnt].json_string = "dst_ip2"; + json_cmd[cmd_cnt].json_type = cJSON_String; + json_cmd[cmd_cnt].empty_allowed = 1; + json_cmd[cmd_cnt].default_string = "255.255.255.255"; + cmd_cnt++; + + json_cmd[cmd_cnt].json_string = "dport_format"; + json_cmd[cmd_cnt].json_type = cJSON_String; + json_cmd[cmd_cnt].empty_allowed = 1; + json_cmd[cmd_cnt].default_string = "mask"; + cmd_cnt++; + + json_cmd[cmd_cnt].json_string = "dst_port1"; + json_cmd[cmd_cnt].json_type = cJSON_String; + json_cmd[cmd_cnt].empty_allowed = 1; + json_cmd[cmd_cnt].default_string = "0"; + cmd_cnt++; + + json_cmd[cmd_cnt].json_string = "dst_port2"; + json_cmd[cmd_cnt].json_type = cJSON_String; + json_cmd[cmd_cnt].empty_allowed = 1; + json_cmd[cmd_cnt].default_string = "65535"; + cmd_cnt++; + + json_cmd[cmd_cnt].json_string = "protocol"; + json_cmd[cmd_cnt].json_type = cJSON_Number; + json_cmd[cmd_cnt].empty_allowed = 1; + json_cmd[cmd_cnt].default_int = 0; + cmd_cnt++; + + json_cmd[cmd_cnt].json_string = "direction"; + json_cmd[cmd_cnt].json_type = cJSON_String; + json_cmd[cmd_cnt].str2int_flag = 1; + json_cmd[cmd_cnt].empty_allowed = 1; + json_cmd[cmd_cnt].default_string = "double"; + cmd_cnt++; + + json_cmd[cmd_cnt].json_string = "is_valid"; + json_cmd[cmd_cnt].json_type = cJSON_Number; + cmd_cnt++; + + return direct_write_rule(region_json, p_iris->str2int_map, json_cmd, cmd_cnt, table); +} + +int write_intval_line(cJSON *region_json, struct iris_description *p_iris, struct iris_table *table) +{ + struct translate_command json_cmd[MAX_COLUMN_NUM]; + int cmd_cnt = 0; + memset(json_cmd, 0, sizeof(json_cmd)); + + json_cmd[cmd_cnt].json_string = "region_id"; + json_cmd[cmd_cnt].json_type = cJSON_Number; + cmd_cnt++; + + json_cmd[cmd_cnt].json_string = "group_id"; + json_cmd[cmd_cnt].json_type = cJSON_Number; + cmd_cnt++; + + if (table->table_type == TABLE_TYPE_INTERVAL_PLUS) { + json_cmd[cmd_cnt].json_string = "district"; + json_cmd[cmd_cnt].json_type = cJSON_String; + cmd_cnt++; + } + + json_cmd[cmd_cnt].json_string = "low_boundary"; + json_cmd[cmd_cnt].json_type = cJSON_Number; + cmd_cnt++; + + json_cmd[cmd_cnt].json_string = "up_boundary"; + json_cmd[cmd_cnt].json_type = cJSON_Number; + cmd_cnt++; + + json_cmd[cmd_cnt].json_string = "is_valid"; + json_cmd[cmd_cnt].json_type = cJSON_Number; + cmd_cnt++; + + return direct_write_rule(region_json, p_iris->str2int_map, json_cmd, cmd_cnt, table); +} + +int write_digest_line(cJSON *region_json, struct iris_description *p_iris, struct iris_table *table) +{ + struct translate_command json_cmd[MAX_COLUMN_NUM]; + int cmd_cnt = 0; + memset(json_cmd, 0, sizeof(json_cmd)); + + json_cmd[cmd_cnt].json_string = "region_id"; + json_cmd[cmd_cnt].json_type = cJSON_Number; + cmd_cnt++; + + json_cmd[cmd_cnt].json_string = "group_id"; + json_cmd[cmd_cnt].json_type = cJSON_Number; + cmd_cnt++; + + json_cmd[cmd_cnt].json_string = "raw_len"; + json_cmd[cmd_cnt].json_type = cJSON_Number; + cmd_cnt++; + + json_cmd[cmd_cnt].json_string = "digest"; + json_cmd[cmd_cnt].json_type = cJSON_String; + cmd_cnt++; + + json_cmd[cmd_cnt].json_string = "cfds_level"; + json_cmd[cmd_cnt].json_type = cJSON_Number; + cmd_cnt++; + + json_cmd[cmd_cnt].json_string = "is_valid"; + json_cmd[cmd_cnt].json_type = cJSON_Number; + cmd_cnt++; + + return direct_write_rule(region_json, p_iris->str2int_map, json_cmd, cmd_cnt, table); +} + +int write_similar_line(cJSON *region_json, struct iris_description *p_iris, struct iris_table *table) +{ + struct translate_command json_cmd[MAX_COLUMN_NUM]; + int cmd_cnt = 0; + memset(json_cmd, 0, sizeof(json_cmd)); + + json_cmd[cmd_cnt].json_string = "region_id"; + json_cmd[cmd_cnt].json_type = cJSON_Number; + cmd_cnt++; + + json_cmd[cmd_cnt].json_string = "group_id"; + json_cmd[cmd_cnt].json_type = cJSON_Number; + cmd_cnt++; + + json_cmd[cmd_cnt].json_string = "target"; + json_cmd[cmd_cnt].json_type = cJSON_String; + cmd_cnt++; + + json_cmd[cmd_cnt].json_string = "threshold"; + json_cmd[cmd_cnt].json_type = cJSON_Number; + cmd_cnt++; + + json_cmd[cmd_cnt].json_string = "is_valid"; + json_cmd[cmd_cnt].json_type = cJSON_Number; + cmd_cnt++; + + return direct_write_rule(region_json, p_iris->str2int_map, json_cmd, cmd_cnt, table); +} + +int write_region_rule(cJSON *region_json, int compile_id, int group_id, struct iris_description *p_iris) +{ + cJSON *item = cJSON_GetObjectItem(region_json, "table_name"); + if (NULL == item || item->type != cJSON_String) { + fprintf(stderr, "compile rule %d's table_name not defined or format error.", compile_id); + return -1; + } + const char *table_name = item->valuestring; + + item = cJSON_GetObjectItem(region_json, "table_type"); + if (NULL == item || item->type != cJSON_String) { + fprintf(stderr, "compile rule %d's table name %s's table_type not defined or format error.", + compile_id, table_name); + return -1; + } + + const char *table_type_str = item->valuestring; + enum table_type table_type = TABLE_TYPE_EXPR; + int ret = maat_kv_read(p_iris->str2int_map, table_type_str, (int*)&(table_type)); + if (ret != 1) { + fprintf(stderr, "compile rule %d table name %s's table_type %s invalid.", + compile_id, table_name, table_type_str); + return -1; + } + + cJSON *table_content = cJSON_GetObjectItem(region_json, "table_content"); + if (NULL == table_content || table_content->type != cJSON_Object) { + fprintf(stderr, "compile rule %d table name %s's table_content not defined or format error.", + compile_id, table_name); + return -1; + } + + int region_id = get_region_seq(p_iris); + cJSON_AddNumberToObject(table_content, "region_id", region_id); + cJSON_AddNumberToObject(table_content, "group_id", group_id); + cJSON_AddNumberToObject(table_content, "is_valid", 1); + + struct iris_table *table_info = query_table_info(p_iris, table_name, table_type); + switch(table_type) + { + case TABLE_TYPE_EXPR: + case TABLE_TYPE_EXPR_PLUS: + ret = write_expr_line(table_content, p_iris, table_info); + break; + case TABLE_TYPE_IP: + ret = write_ip_line(table_content, p_iris, table_info); + break; + case TABLE_TYPE_IP_PLUS: + ret = write_ip_plus_line(table_content, p_iris, table_info); + break; + case TABLE_TYPE_INTERVAL: + case TABLE_TYPE_INTERVAL_PLUS: + ret = write_intval_line(table_content, p_iris, table_info); + break; + case TABLE_TYPE_DIGEST: + ret = write_digest_line(table_content, p_iris, table_info); + break; + case TABLE_TYPE_SIMILARITY: + ret = write_similar_line(table_content, p_iris, table_info); + break; + default: + assert(0); + break; + } + return ret; +} + +int write_group2compile_line(int group_id, int compile_id, int group_not_flag, int clause_index, + const char *virtual_table, struct iris_description *p_iris) +{ + char buff[4096] = {0}; + struct iris_table *table = p_iris->group2compile_table; + + snprintf(buff, sizeof(buff), "%d\t%d\t1\t%d\t%s\t%d\n", group_id, compile_id, group_not_flag, virtual_table, clause_index); + table->write_pos += memcat(&(table->buff), table->write_pos, &(table->buff_sz), buff, strlen(buff)); + table->line_count++; + + return 0; +} + +int write_group2group_line(int group_id, int superior_group_id, struct iris_description *p_iris) +{ + char buff[4096] = {0}; + struct iris_table *table = p_iris->group2group_table; + + snprintf(buff, sizeof(buff), "%d\t%d\t1\n", group_id, superior_group_id); + table->write_pos += memcat(&(table->buff), table->write_pos, &(table->buff_sz), buff, strlen(buff)); + table->line_count++; + + return 0; +} + +int write_group_rule(cJSON *group_json, int parent_id, int parent_type, int tracking_compile_id, + int Nth_group, struct iris_description *p_iris) +{ + int ret = 0; + int group_not_flag = 0; + int clause_index = 0; + const char *str_parent_type[2] = {"compile", "group"}; + const char *group_name = NULL; + const char *virtual_table = NULL; + + cJSON *item = cJSON_GetObjectItem(group_json, "group_name"); + if (NULL == item || item->type != cJSON_String) { + group_name = untitled_group_name; + } else { + group_name = item->valuestring; + } + + if (parent_type == PARENT_TYPE_COMPILE) { + item = cJSON_GetObjectItem(group_json, "virtual_table"); + if (NULL == item || item->type != cJSON_String) { + virtual_table = "null"; + } else { + virtual_table = item->valuestring; + } + + item = cJSON_GetObjectItem(group_json, "not_flag"); + if (NULL == item || item->type != cJSON_Number) { + group_not_flag = 0; + } else { + group_not_flag = item->valueint; + } + + item = cJSON_GetObjectItem(group_json, "clause_index"); + if (NULL == item || item->type != cJSON_Number) { + clause_index = Nth_group; + } else { + clause_index = item->valueint; + } + } else { + group_not_flag = 0; + } + + struct group_info *group_info = group_info_read(p_iris->group_name_map, group_name); + //exist group name, regions and sub groups will be ommit. + if (NULL == group_info) { + group_info = group_info_add_unsafe(p_iris, group_name); + cJSON *region_json = cJSON_GetObjectItem(group_json, "regions"); + if (region_json != NULL) { + cJSON *region_rule = NULL; + cJSON_ArrayForEach(region_rule, region_json) { + ret = write_region_rule(region_rule, tracking_compile_id, group_info->group_id, p_iris); + if (ret < 0) { + fprintf(stderr, "compile rule %d write region error.", tracking_compile_id); + return -1; + } + } + } + + cJSON *sub_groups = cJSON_GetObjectItem(group_json,"sub_groups"); + if (sub_groups != NULL) { + //recursively + int i = 0; + cJSON_ArrayForEach(item, sub_groups) { + i++; + ret = write_group_rule(item, group_info->group_id, PARENT_TYPE_GROUP, tracking_compile_id, i, p_iris); + if (ret < 0) { + return -1; + } + } + } + + if (NULL == region_json && NULL == sub_groups) { + fprintf(stdout, "A group of compile rule %d has neither regions, sub groups, nor refered another exisited group.", + tracking_compile_id); + } + } + + if (parent_type == PARENT_TYPE_COMPILE) { + ret = write_group2compile_line(group_info->group_id, parent_id, group_not_flag, clause_index, virtual_table, p_iris); + } else { + ret = write_group2group_line(group_info->group_id, parent_id, p_iris); + } + + if (ret < 0) { + fprintf(stderr, "%s rule %d write group error.", str_parent_type[parent_type], parent_id); + return -1; + } + + return 0; +} + +int write_compile_line(cJSON *compile, struct iris_description *p_iris) +{ + cJSON *item=cJSON_GetObjectItem(compile, "compile_id"); + if (item->type != cJSON_Number) { + fprintf(stderr, "compile_id format not number."); + return -1; + } + int compile_id = item->valueint; + + cJSON *group_array = cJSON_GetObjectItem(compile, "groups"); + int group_num = cJSON_GetArraySize(group_array); + int *clause_ids = ALLOC(int, group_num); + int clause_num = 0; + cJSON *group_obj = NULL; + + cJSON_ArrayForEach(group_obj, group_array) { + item = cJSON_GetObjectItem(group_obj, "clause_index"); + if (item) { + int i = 0; + int clause_index = item->valueint; + for (i = 0; i < clause_num; i++) { + if (clause_ids[i] == clause_index) { + break; + } + } + + if (i == clause_num) { + clause_ids[clause_num] = clause_index; + clause_num++; + } + } + } + FREE(clause_ids); + + if (clause_num == 0) { + clause_num = group_num; + } + + cJSON_AddNumberToObject(compile, "clause_num", clause_num); + + struct translate_command compile_cmd[MAX_COLUMN_NUM]; + memset(compile_cmd, 0, sizeof(compile_cmd)); + + int cmd_cnt = 0; + compile_cmd[cmd_cnt].json_string = "compile_id"; + compile_cmd[cmd_cnt].json_type = cJSON_Number; + cmd_cnt++; + + compile_cmd[cmd_cnt].json_string = "service"; + compile_cmd[cmd_cnt].json_type = cJSON_Number; + cmd_cnt++; + + compile_cmd[cmd_cnt].json_string = "action"; + compile_cmd[cmd_cnt].json_type = cJSON_Number; + cmd_cnt++; + + compile_cmd[cmd_cnt].json_string = "do_blacklist"; + compile_cmd[cmd_cnt].json_type = cJSON_Number; + cmd_cnt++; + + compile_cmd[cmd_cnt].json_string = "do_log"; + compile_cmd[cmd_cnt].json_type = cJSON_Number; + cmd_cnt++; + + + compile_cmd[cmd_cnt].json_string = "tags"; + compile_cmd[cmd_cnt].json_type = cJSON_String; + compile_cmd[cmd_cnt].empty_allowed = 1; + compile_cmd[cmd_cnt].default_string = "0"; + cmd_cnt++; + + compile_cmd[cmd_cnt].json_string = "user_region"; + compile_cmd[cmd_cnt].json_type = cJSON_String; + cmd_cnt++; + + compile_cmd[cmd_cnt].json_string = "is_valid"; + compile_cmd[cmd_cnt].json_type = cJSON_String; + compile_cmd[cmd_cnt].str2int_flag = 1; + cmd_cnt++; + + compile_cmd[cmd_cnt].json_string = "clause_num"; + compile_cmd[cmd_cnt].json_type = cJSON_Number; + cmd_cnt++; + + compile_cmd[cmd_cnt].json_string = "evaluation_order"; + compile_cmd[cmd_cnt].json_type = cJSON_String; + compile_cmd[cmd_cnt].empty_allowed = 1; + compile_cmd[cmd_cnt].default_string = "0.0"; + cmd_cnt++; + + struct iris_table *table_info = NULL; + item = cJSON_GetObjectItem(compile,"table_name"); + if (NULL == item || item->type != cJSON_String) { + table_info = p_iris->compile_table; + } else { + table_info = query_table_info(p_iris, item->valuestring, TABLE_TYPE_COMPILE); + } + + int ret = direct_write_rule(compile, p_iris->str2int_map, compile_cmd, cmd_cnt, table_info); + if (ret < 0) { + return -1; + } + + return compile_id; +} + +void write_table_idx(struct iris_description *p_iris, struct iris_table *table) +{ + char line_cnt_str[32] = {0}; + char err_str[256] = {0}; + + snprintf(line_cnt_str, sizeof(line_cnt_str), "%010d\n", table->line_count); + + size_t table_file_sz = strlen(line_cnt_str) + table->write_pos; + unsigned char *buff = ALLOC(unsigned char, table_file_sz); + memcpy(buff, line_cnt_str, strlen(line_cnt_str)); + memcpy(buff + strlen(line_cnt_str), table->buff, table->write_pos); + + FILE *table_fp = fopen(table->table_path, "w"); + if (p_iris->encrypt_key) { + unsigned char *encrypt_buff = NULL; + size_t encrypt_buff_sz = 0; + int ret = crypt_memory(buff, table_file_sz, &encrypt_buff, &encrypt_buff_sz, p_iris->encrypt_key, + p_iris->encrypt_algo, 1, err_str, sizeof(err_str)); + assert(ret == 0); + fwrite(encrypt_buff, encrypt_buff_sz, 1, table_fp); + fprintf(p_iris->idx_fp, "%s\t%d\t%s\t%s\n", table->table_name, table->line_count, table->table_path, p_iris->encrypt_algo); + FREE(encrypt_buff); + } else { + fwrite(buff, table_file_sz, 1, table_fp); + fprintf(p_iris->idx_fp, "%s\t%d\t%s\n", table->table_name, table->line_count, table->table_path); + } + + fclose(table_fp); + FREE(buff); +} + +int write_index_file(struct iris_description *p_iris) +{ + p_iris->idx_fp = fopen(p_iris->index_path, "w"); + if (NULL == p_iris->idx_fp) { + fprintf(stderr, "index file %s fopen error %s.",p_iris->index_path, strerror(errno)); + return -1; + } + + struct iris_table *node = NULL; + struct iris_table *tmp = NULL; + HASH_ITER(hh, p_iris->iris_table_map, node, tmp) { + write_table_idx(p_iris, node); + } + + fclose(p_iris->idx_fp); + p_iris->idx_fp = NULL; + + return 0; +} + +int write_iris(cJSON *json, struct iris_description *p_iris) +{ + int i=0; + int ret=0; + static struct group_info *parent_group = NULL; //TODO + + cJSON *plug_tables = cJSON_GetObjectItem(json, "plugin_table"); + if (plug_tables != NULL) { + cJSON *each_plug_table = NULL; + cJSON_ArrayForEach(each_plug_table, plug_tables) { + write_plugin_line(each_plug_table, i, p_iris); + i++; + } + } + + cJSON *group_array = cJSON_GetObjectItem(json, "groups");//sub-group to group + if (group_array != NULL) { + cJSON *group_obj = NULL; + cJSON_ArrayForEach(group_obj, group_array) { + const char *parent_group_name = NULL; + cJSON *item = cJSON_GetObjectItem(group_obj, "parent_group"); + if (NULL == item || item->type!=cJSON_String) { + parent_group_name = untitled_group_name; + } else { + parent_group_name = item->string; + } + + parent_group = group_info_read(p_iris->group_name_map, parent_group_name); + if(NULL == parent_group) { + parent_group = group_info_add_unsafe(p_iris, parent_group_name); + } + + ret = write_group_rule(group_obj, parent_group->group_id, PARENT_TYPE_GROUP, 0, 0, p_iris); + if (ret < 0) { + return -1; + } + } + } + + int compile_cnt = 0; + cJSON *compile_array = cJSON_GetObjectItem(json, "rules"); + if (compile_array != NULL) { + compile_cnt = cJSON_GetArraySize(compile_array); + } + + if (compile_cnt > 0) { + cJSON *compile_obj = NULL; + cJSON_ArrayForEach(compile_obj, compile_array) { + int compile_id = write_compile_line(compile_obj,p_iris); + if (compile_id < 0) { + fprintf(stderr, "In %d compile rule.", i); + return -1; + } + + group_array = cJSON_GetObjectItem(compile_obj, "groups"); + if (NULL == group_array) { + fprintf(stderr, "compile rule %d have no group.",compile_id); + return -1; + } + + int group_cnt = cJSON_GetArraySize(group_array); + if (group_cnt <= 0) { + fprintf(stderr, "compile rule %d have no groups.", compile_id); + return -1; + } + + i = 0; + cJSON *group_obj = NULL; + cJSON_ArrayForEach(group_obj, group_array) { + ret = write_group_rule(group_obj, compile_id, PARENT_TYPE_COMPILE, compile_id, i, p_iris); + if (ret < 0) { + return -1; + } + i++; + } + } + } + + ret = write_index_file(p_iris); + if (ret < 0) { + return -1; + } + return 0; +} + +int json2iris(const char *json_buff, const char *json_filename, const char *compile_tn, + const char *group2compile_tn, const char *group2group_tn, redisContext *redis_write_ctx, + char *iris_dir_buf, int buf_len, char *encrypt_key, char *encrypt_algo) +{ + int ret = -1; + cJSON *tmp_obj = NULL; + struct iris_description iris_cfg; + + memset(&iris_cfg, 0, sizeof(iris_cfg)); + + cJSON *json = cJSON_Parse(json_buff); + if (!json) { + fprintf(stderr, "Error before: %-200.200s", cJSON_GetErrorPtr()); + goto error_out; + } + + tmp_obj = cJSON_GetObjectItem(json, "compile_table"); + if (tmp_obj) { + compile_tn = tmp_obj->valuestring; + } + + tmp_obj = cJSON_GetObjectItem(json, "group2compile_table"); + if (tmp_obj) { + group2compile_tn = tmp_obj->valuestring; + } + + tmp_obj = cJSON_GetObjectItem(json, "group2group_table"); + if (tmp_obj) { + group2group_tn = tmp_obj->valuestring; + } + + ret = set_iris_descriptor(json_filename, json, encrypt_key, encrypt_algo, + compile_tn, group2compile_tn, group2group_tn, + redis_write_ctx, &iris_cfg); + if (ret < 0) { + goto error_out; + } + + ret = create_tmp_dir(&iris_cfg); + if (ret < 0) { + fprintf(stderr, "create tmp folder %s error", iris_cfg.tmp_iris_dir); + goto error_out; + } + + ret = write_iris(json, &iris_cfg); + if (ret < 0) { + goto error_out; + } + memcpy(iris_dir_buf, iris_cfg.tmp_iris_index_dir, MIN(strlen(iris_cfg.tmp_iris_index_dir) + 1, (unsigned int)buf_len)); + + cJSON_Delete(json); + clear_iris_descriptor(&iris_cfg); + return 0; + +error_out: + cJSON_Delete(json); + clear_iris_descriptor(&iris_cfg); + return -1; +} \ No newline at end of file diff --git a/src/maat_api.cpp b/src/maat_api.cpp index 544bebe..bb9a47f 100644 --- a/src/maat_api.cpp +++ b/src/maat_api.cpp @@ -12,13 +12,16 @@ #include #include "utils.h" +#include "json2iris.h" #include "maat/maat.h" #include "maat_rule.h" #include "maat_common.h" #include "maat_kv.h" +#include "maat_command.h" #include "maat_table_schema.h" #include "maat_table_runtime.h" #include "maat_config_monitor.h" +#include "maat_redis_monitor.h" struct maat_options* maat_options_new(void) { @@ -65,6 +68,7 @@ int maat_options_set_gc_timeout_ms(struct maat_options *opts, int interval_ms) int maat_options_set_deferred_load_on(struct maat_options *opts) { opts->deferred_load_on = 1; + return 0; } @@ -74,8 +78,9 @@ int maat_options_set_iris_full_dir(struct maat_options *opts, const char *full_d return -1; } - memcpy(opts->iris_full_dir, full_dir, strlen(full_dir)); + memcpy(opts->iris_ctx.full_dir, full_dir, strlen(full_dir)); opts->input_mode = DATA_SOURCE_IRIS_FILE; + return 0; } @@ -85,26 +90,91 @@ int maat_options_set_iris_inc_dir(struct maat_options *opts, const char *inc_dir return -1; } - memcpy(opts->iris_inc_dir, inc_dir, strlen(inc_dir)); + memcpy(opts->iris_ctx.inc_dir, inc_dir, strlen(inc_dir)); opts->input_mode = DATA_SOURCE_IRIS_FILE; + + return 0; +} + +int maat_options_set_json_file(struct maat_options *opts, const char *json_filename) +{ + strncpy(opts->json_ctx.json_file, json_filename, sizeof(opts->json_ctx.json_file)); + opts->input_mode = DATA_SOURCE_JSON_FILE; + + return 0; +} + +int maat_options_set_redis_ip(struct maat_options *opts, const char *redis_ip) +{ + memcpy(opts->redis_ctx.redis_ip, redis_ip, strlen(redis_ip)); + opts->input_mode = DATA_SOURCE_REDIS; + + return 0; +} + +int maat_options_set_redis_port(struct maat_options *opts, uint16_t redis_port) +{ + opts->redis_ctx.redis_port = redis_port; + + return 0; +} + +int maat_options_set_redis_db_index(struct maat_options *opts, int db_index) +{ + opts->redis_ctx.redis_db = db_index; + return 0; } void maat_read_full_config(struct maat *maat_instance) { + int ret = -1; + char err_str[NAME_MAX] = {0}; + struct source_redis_ctx *mr_ctx = NULL; + switch (maat_instance->input_mode) { + case DATA_SOURCE_REDIS: + mr_ctx = &(maat_instance->mr_ctx); + fprintf(stdout, "Maat initiate from Redis %s:%hu db%d.", + mr_ctx->redis_ip, mr_ctx->redis_port, mr_ctx->redis_db); + mr_ctx->read_ctx = maat_cmd_connect_redis(mr_ctx->redis_ip, mr_ctx->redis_port, mr_ctx->redis_db); + if (mr_ctx->read_ctx != NULL) { + redis_monitor_traverse(maat_instance->maat_version, mr_ctx, + maat_start_cb, maat_update_cb, maat_finish_cb, + maat_instance); + } + + if (NULL == maat_instance->creating_maat_rt) { + fprintf(stderr, "At initiation: NO effective rule in %s.", + maat_instance->iris_ctx.full_dir); + } + break; case DATA_SOURCE_IRIS_FILE: config_monitor_traverse(maat_instance->maat_version, maat_instance->iris_ctx.full_dir, - maat_start_cb, - maat_update_cb, - maat_finish_cb, + maat_start_cb, maat_update_cb, maat_finish_cb, maat_instance); if (NULL == maat_instance->creating_maat_rt) { fprintf(stderr, "At initiation: NO effective rule in %s.", maat_instance->iris_ctx.full_dir); } break; + case DATA_SOURCE_JSON_FILE: + ret = load_maat_json_file(maat_instance, maat_instance->json_ctx.json_file, err_str, sizeof(err_str)); + if (ret < 0) { + fprintf(stderr, "Maat re-initiate with JSON file %s failed: %s", maat_instance->json_ctx.json_file, err_str); + return; + } + + config_monitor_traverse(maat_instance->maat_version, + maat_instance->json_ctx.iris_file, + maat_start_cb, maat_update_cb, maat_finish_cb, + maat_instance); + if (NULL == maat_instance->creating_maat_rt) { + fprintf(stderr, "At initiation: NO effective rule in %s.", + maat_instance->json_ctx.iris_file); + } + break; default: break; } @@ -134,9 +204,17 @@ struct maat *maat_new(struct maat_options *opts, const char *table_info_path) maat_instance->input_mode = opts->input_mode; switch (maat_instance->input_mode) { + case DATA_SOURCE_REDIS: + memcpy(maat_instance->mr_ctx.redis_ip, opts->redis_ctx.redis_ip, strlen(opts->redis_ctx.redis_ip)); + maat_instance->mr_ctx.redis_port = opts->redis_ctx.redis_port; + maat_instance->mr_ctx.redis_db = opts->redis_ctx.redis_db; + break; case DATA_SOURCE_IRIS_FILE: - memcpy(maat_instance->iris_ctx.full_dir, opts->iris_full_dir, strlen(opts->iris_full_dir)); - memcpy(maat_instance->iris_ctx.inc_dir, opts->iris_inc_dir, strlen(opts->iris_inc_dir)); + memcpy(maat_instance->iris_ctx.full_dir, opts->iris_ctx.full_dir, strlen(opts->iris_ctx.full_dir)); + memcpy(maat_instance->iris_ctx.inc_dir, opts->iris_ctx.inc_dir, strlen(opts->iris_ctx.inc_dir)); + break; + case DATA_SOURCE_JSON_FILE: + memcpy(maat_instance->json_ctx.json_file, opts->json_ctx.json_file, strlen(opts->json_ctx.json_file)); break; default: fprintf(stderr, "data source unsupported:%d\n", maat_instance->input_mode); diff --git a/src/maat_command.cpp b/src/maat_command.cpp new file mode 100644 index 0000000..7e30b1b --- /dev/null +++ b/src/maat_command.cpp @@ -0,0 +1,348 @@ +/* +********************************************************************************************** +* File: maat_command.cpp +* Description: +* Authors: Liu WenTan +* Date: 2022-10-31 +* Copyright: (c) 2018-2022 Geedge Networks, Inc. All rights reserved. +*********************************************************************************************** +*/ + +#include +#include +#include +#include + +#include "utils.h" +#include "maat_utils.h" +#include "maat_command.h" +#include "maat_rule.h" +#include "hiredis/hiredis.h" +#include "maat_config_monitor.h" +#include "maat_table_schema.h" + +extern const char *foreign_source_prefix; +extern const char *mr_key_prefix; + +extern const char *mr_expire_lock; +extern const long mr_expire_lock_time; + +extern const char *mr_status_sset; +extern const char *mr_version_sset; +extern const char *mr_label_sset; + +redisReply *maat_cmd_wrap_redis_command(redisContext *c, const char *format, ...) +{ + va_list ap; + void *reply = NULL; + int ret = REDIS_ERR; + int retry = 0; + + while (reply == NULL && retry < 2 && ret != REDIS_OK) { + va_start(ap,format); + reply = redisvCommand(c,format,ap); + va_end(ap); + if (NULL == reply) { + ret = redisReconnect(c); + retry++; + } + } + + return (redisReply *)reply; +} + +redisContext *maat_cmd_connect_redis(const char *redis_ip, int redis_port, int redis_db) +{ + struct timeval connect_timeout; + connect_timeout.tv_sec = 0; + connect_timeout.tv_usec = 100 * 1000; // 100 ms + + redisContext *c = redisConnectWithTimeout(redis_ip, redis_port, connect_timeout); + if (NULL == c || c->err) { + fprintf(stderr, "Unable to connect redis server %s:%d db%d, error: %s", + redis_ip, redis_port, redis_db, c == NULL ? "Unknown" : c->errstr); + + if (c != NULL) { + redisFree(c); + } + return NULL; + } + + redisEnableKeepAlive(c); + redisReply *reply = maat_cmd_wrap_redis_command(c, "select %d", redis_db); + freeReplyObject(reply); + reply = NULL; + + return c; +} + +struct s_rule_array +{ + int cnt; + int size; + struct serial_rule *array; +}; + +void save_serial_rule(void *data, void *user) +{ + struct s_rule_array *array = (struct s_rule_array *)user; + int i = array->cnt; + memcpy(&(array->array[i]), data, sizeof(struct serial_rule)); + array->array[i].op = MAAT_OP_ADD; +} + +void maat_cmd_empty_serial_rule(struct serial_rule *s_rule) +{ + if (s_rule->table_line != NULL) { + FREE(s_rule->table_line); + } + + if (s_rule->n_foreign > 0) { + for (int i = 0; i < s_rule->n_foreign; i++) { + FREE(s_rule->f_keys[i].filename); + FREE(s_rule->f_keys[i].key); + } + + FREE(s_rule->f_keys); + } + + memset(s_rule, 0, sizeof(struct serial_rule)); +} + +int connect_redis_for_write(struct source_redis_ctx *mr_ctx) +{ + assert(mr_ctx->write_ctx == NULL); + mr_ctx->write_ctx = maat_cmd_connect_redis(mr_ctx->redis_ip, mr_ctx->redis_port, mr_ctx->redis_db); + if (NULL == mr_ctx->write_ctx) { + return -1; + } else { + return 0; + } +} + +redisContext *get_redis_ctx_for_write(struct maat *maat_instance) +{ + if (NULL == maat_instance->mr_ctx.write_ctx) { + int ret = connect_redis_for_write(&(maat_instance->mr_ctx)); + if(ret!=0) + { + return NULL; + } + } + return maat_instance->mr_ctx.write_ctx; +} + +void maat_cmd_set_serial_rule(struct serial_rule *rule, enum maat_operation op, unsigned long rule_id, + const char *table_name, const char *line, long long timeout) +{ + memset(rule, 0, sizeof(struct serial_rule)); + rule->op = op; + rule->rule_id = rule_id; + rule->timeout = timeout; + assert(strlen(table_name) < sizeof(rule->table_name)); + strncpy(rule->table_name, table_name, sizeof(rule->table_name)); + if (line != NULL) { + rule->table_line = maat_strdup(line); + } +} + +int maat_cmd_get_valid_flag_offset(const char *line, enum table_type table_type, int valid_column_seq) +{ + int column_seq = 0; + + switch (table_type) { + case TABLE_TYPE_EXPR: + column_seq = 7; + break; + case TABLE_TYPE_IP: + column_seq = 14; + break; + case TABLE_TYPE_IP_PLUS: + column_seq = 18; + break; + case TABLE_TYPE_COMPILE: + column_seq = 8; + break; + case TABLE_TYPE_PLUGIN: + case TABLE_TYPE_IP_PLUGIN: + case TABLE_TYPE_FQDN_PLUGIN: + case TABLE_TYPE_BOOL_PLUGIN: + if (valid_column_seq < 0) { + return -1; + } + + column_seq = valid_column_seq; + break; + case TABLE_TYPE_INTERVAL: + column_seq = 5; + break; + case TABLE_TYPE_INTERVAL_PLUS: + column_seq = 6; + break; + case TABLE_TYPE_DIGEST: + column_seq = 6; + break; + case TABLE_TYPE_SIMILARITY: + column_seq = 5; + break; + case TABLE_TYPE_EXPR_PLUS: + column_seq = 8; + break; + case TABLE_TYPE_GROUP2COMPILE: + case TABLE_TYPE_GROUP2GROUP: + column_seq = 3; + break; + default: + assert(0); + } + + size_t offset = 0; + size_t len = 0; + int ret = get_column_pos(line, column_seq, &offset, &len); + // 0 is also a valid value for some non-MAAT producer. + if (ret < 0 || offset >= strlen(line) || (line[offset] != '1' && line[offset] != '0')) { + return -1; + } + + return offset; +} + +long long maat_cmd_redis_server_time_s(redisContext *c) +{ + long long server_time = 0; + + redisReply *data_reply = maat_cmd_wrap_redis_command(c, "TIME"); + if (data_reply->type == REDIS_REPLY_ARRAY) { + server_time = atoll(data_reply->element[0]->str); + freeReplyObject(data_reply); + data_reply = NULL; + } + + return server_time; +} + +const char *maat_cmd_find_Nth_column(const char *line, int Nth, int *column_len) +{ + size_t i = 0; + int j = 0; + size_t start=0, end=0; + size_t line_len = strlen(line); + + for (i = 0; i < line_len; i++) { + if (line[i] != ' ' && line[i] != '\t') { + continue; + } + + j++; + if (j == Nth - 1) { + start = i + 1; + } + + if(j == Nth) { + end = i; + break; + } + } + + if (start == end) { + return NULL; + } + + if (end == 0) { + end = i; + } + + *column_len = end - start; + + return line + start; +} + +long long maat_cmd_read_redis_integer(const redisReply *reply) +{ + switch (reply->type) { + case REDIS_REPLY_INTEGER: + return reply->integer; + break; + case REDIS_REPLY_ARRAY: + assert(reply->element[0]->type == REDIS_REPLY_INTEGER); + return reply->element[0]->integer; + break; + case REDIS_REPLY_STRING: + return atoll(reply->str); + break; + default: + return -1; + break; + } + return 0; +} + +int maat_cmd_wrap_redis_get_reply(redisContext *c, redisReply **reply) +{ + return redisGetReply(c, (void **)reply); +} + +int maat_cmd_set_line(struct maat *maat_instance, const struct maat_cmd_line *line_rule) +{ + int i = 0; + int ret = 0; + long long absolute_expire_time = 0; + + redisContext *write_ctx = get_redis_ctx_for_write(maat_instance); + if (NULL == write_ctx) { + return -1; + } + + long long server_time = maat_cmd_redis_server_time_s(write_ctx); + if(!server_time) { + return -1; + } + + struct serial_rule *s_rule = ALLOC(struct serial_rule, 1); + + int table_id = table_schema_manager_get_table_id(maat_instance->table_schema_mgr, line_rule->table_name); + if (table_id < 0) { + fprintf(stderr, "Command set line id %d failed: unknown table %s.", line_rule->rule_id, line_rule->table_name); + FREE(s_rule); + return -1; + } + + struct table_schema *table_schema = table_schema_get(maat_instance->table_schema_mgr, table_id); + if (!table_schema) { + FREE(s_rule); + return -1; + } + + int valid_flag_column = table_schema_get_valid_flag_column(table_schema); + if (valid_flag_column < 0) { + fprintf(stderr, "Command set line id %d failed: table %s is not a plugin or ip_plugin table.", + line_rule->rule_id, line_rule->table_name); + FREE(s_rule); + return -1; + } + + enum table_type table_type = table_schema_get_table_type(table_schema); + int is_valid = maat_cmd_get_valid_flag_offset(line_rule->table_line, table_type, valid_flag_column); + + if (line_rule->expire_after > 0) { + absolute_expire_time = server_time + line_rule->expire_after; + } + + maat_cmd_set_serial_rule(s_rule + i, (enum maat_operation)is_valid, line_rule->rule_id, line_rule->table_name, + line_rule->table_line, absolute_expire_time); + + int success_cnt = maat_cmd_exec_serial_rule(write_ctx, s_rule, 1, server_time); + if (success_cnt != 1) { + ret = -1; + goto error_out; + } + + ret = success_cnt; + maat_instance->line_cmd_acc_num += success_cnt; + +error_out: + maat_cmd_empty_serial_rule(s_rule); + FREE(s_rule); + + return ret; +} \ No newline at end of file diff --git a/src/maat_config_monitor.cpp b/src/maat_config_monitor.cpp index 9a9ab5d..3e2575b 100644 --- a/src/maat_config_monitor.cpp +++ b/src/maat_config_monitor.cpp @@ -14,10 +14,13 @@ #include #include #include +#include -#include "maat_config_monitor.h" -#include "maat_utils.h" #include "utils.h" +#include "maat_utils.h" +#include "maat_rule.h" +#include "json2iris.h" +#include "maat_config_monitor.h" #define CM_MAX_TABLE_NUM 256 #define MAX_CONFIG_LINE (1024 * 16) @@ -345,4 +348,81 @@ void config_monitor_traverse(long long current_version, const char *idx_dir, } free(idx_path_array); +} + +int load_maat_json_file(struct maat *maat_instance, const char *json_filename, char *err_str, size_t err_str_sz) +{ + int ret = 0; + struct stat fstat_buf; + unsigned char *json_buff = NULL; + unsigned char *decrypted_buff = NULL; + unsigned char *uncompressed_buff = NULL; + size_t json_buff_sz = 0; + size_t decrypted_buff_sz = 0; + size_t uncompressed_buff_sz = 0; + + fprintf(stdout, "Maat initial with JSON file %s, formating..", json_filename); + + if (strlen(maat_instance->decrypt_key) && strlen(maat_instance->decrypt_algo)) { + ret = decrypt_open(json_filename, maat_instance->decrypt_key, maat_instance->decrypt_algo, + (unsigned char **)&decrypted_buff, &decrypted_buff_sz, err_str, err_str_sz); + if (ret < 0) { + fprintf(stderr, "Decrypt Maat JSON file %s failed.", json_filename); + return -1; + } + + json_buff=decrypted_buff; + json_buff_sz=decrypted_buff_sz; + } + + if (maat_instance->maat_json_is_gzipped) { + ret = gzip_uncompress(json_buff, json_buff_sz, &uncompressed_buff, &uncompressed_buff_sz); + free(json_buff); + if (ret < 0) { + fprintf(stderr, "Uncompress Maat JSON file %s failed.", json_filename); + return -1; + } + + json_buff = uncompressed_buff; + json_buff_sz = uncompressed_buff_sz; + } + + //decryption failed or no decryption + if (NULL == json_buff) { + ret = load_file_to_memory(json_filename, &json_buff, &json_buff_sz); + if (ret < 0) { + fprintf(stderr, "Read Maat JSON file %s failed.", json_filename); + return -1; + } + } + + ret = json2iris((const char*)json_buff, json_filename, + maat_instance->compile_tn, maat_instance->group2compile_tn, maat_instance->group2group_tn, + NULL, + maat_instance->json_ctx.iris_file, + sizeof(maat_instance->json_ctx.iris_file), + strlen(maat_instance->decrypt_key) ? maat_instance->decrypt_key : NULL, + strlen(maat_instance->decrypt_algo) ? maat_instance->decrypt_algo : NULL); + + free(json_buff); + json_buff = NULL; + if (ret < 0) { + return -1; + } + + if (!maat_instance->is_running) { + strncpy(maat_instance->json_ctx.json_file, json_filename, sizeof(maat_instance->json_ctx.json_file)); + } + + ret=stat(json_filename, &fstat_buf); + maat_instance->json_ctx.last_md5_time = fstat_buf.st_ctim; + + md5_file(maat_instance->json_ctx.json_file, maat_instance->json_ctx.effective_json_md5); + fprintf(stdout, "JSON file %s md5: %s, generate index file %s OK.", + maat_instance->json_ctx.json_file, + maat_instance->json_ctx.effective_json_md5, + maat_instance->json_ctx.iris_file); + maat_instance->input_mode = DATA_SOURCE_JSON_FILE; + + return 0; } \ No newline at end of file diff --git a/src/maat_redis_monitor.cpp b/src/maat_redis_monitor.cpp new file mode 100644 index 0000000..bdc2815 --- /dev/null +++ b/src/maat_redis_monitor.cpp @@ -0,0 +1,1403 @@ +/********************************************************************************************** +* File: maat_redis_monitor.cpp +* Description: +* Authors: Liu WenTan +* Date: 2022-11-29 +* Copyright: (c) 2018-2022 Geedge Networks, Inc. All rights reserved. +*********************************************************************************************** +*/ + +#include +#include +#include +#include + +#include "utils.h" +#include "maat_utils.h" +#include "maat_command.h" +#include "maat_config_monitor.h" +#include "maat_redis_monitor.h" +#include "maat_table_schema.h" + +const time_t MAAT_REDIS_RECONNECT_INTERVAL_S = 5; +const static int MAAT_REDIS_SYNC_TIME = 30 * 60; + +const char *mr_expire_lock = "EXPIRE_OP_LOCK"; +const long mr_expire_lock_timeout_ms = 300 * 1000; + +const char *mr_version_sset = "MAAT_VERSION_TIMER"; +const char *mr_status_sset = "MAAT_UPDATE_STATUS"; +const char *mr_expire_sset = "MAAT_EXPIRE_TIMER"; +const char *mr_label_sset = "MAAT_LABEL_INDEX"; + +const char *mr_key_prefix[2] = {"OBSOLETE_RULE", "EFFECTIVE_RULE"}; +const char *foreign_source_prefix = "redis://"; +const char *foreign_key_prefix = "__FILE_"; + +const char *mr_op_str[] = {"DEL", "ADD", "RENEW_TIMEOUT"}; + +char *get_foreign_cont_filename(const char *table_name, int rule_id, const char *foreign_key, const char *dir) +{ + char buffer[512] = {0}; + + snprintf(buffer, sizeof(buffer),"%s/%s-%d-%s", dir, table_name, rule_id, foreign_key); + char *filename = ALLOC(char, strlen(buffer) + 1); + memcpy(filename, buffer, strlen(buffer)); + + return filename; +} + +void _get_foregin_keys(struct serial_rule *p_rule, int *foreign_columns, int n_foreign, const char *dir) +{ + int foreign_key_size = 0; + p_rule->f_keys = ALLOC(struct foreign_key, n_foreign); + + for (int i = 0; i < n_foreign; i++) { + const char *p_foreign = maat_cmd_find_Nth_column(p_rule->table_line, foreign_columns[i], &foreign_key_size); + if (NULL == p_foreign) { + fprintf(stderr, "Get %s,%lu foreign keys failed: No %dth column.", + p_rule->table_name, p_rule->rule_id, foreign_columns[i]); + continue; + } + + //emtpy file + if (0 == strncasecmp(p_foreign, "null", strlen("null"))) { + continue; + } + + if (0 != strncmp(p_foreign, foreign_source_prefix, strlen(foreign_source_prefix))) { + fprintf(stderr, "Get %s,%lu foreign key failed: Invalid source prefix %s.", + p_rule->table_name, p_rule->rule_id, p_foreign); + continue; + } + + p_rule->f_keys[p_rule->n_foreign].column = foreign_columns[i]; + foreign_key_size = foreign_key_size - strlen(foreign_source_prefix); + p_foreign += strlen(foreign_source_prefix); + + if (0 != strncmp(p_foreign, foreign_key_prefix, strlen(foreign_key_prefix))) { + fprintf(stdout, "%s, %lu foreign key prefix %s is not recommended.", + p_rule->table_name, p_rule->rule_id, p_foreign); + } + + p_rule->f_keys[p_rule->n_foreign].key = ALLOC(char, foreign_key_size+1); + memcpy(p_rule->f_keys[p_rule->n_foreign].key, p_foreign, foreign_key_size); + p_rule->f_keys[p_rule->n_foreign].filename = get_foreign_cont_filename(p_rule->table_name, p_rule->rule_id, p_rule->f_keys[p_rule->n_foreign].key, dir); + p_rule->n_foreign++; + } + + if (0 == p_rule->n_foreign) { + FREE(p_rule->f_keys); + } +} + +int get_foreign_keys_define(redisContext *ctx, struct serial_rule *rule_list, int rule_num, struct maat *maat_instance, const char *dir) +{ + int rule_with_foreign_key = 0; + struct table_schema *table_schema = NULL; + + for (int i = 0; i < rule_num; i++) { + if (NULL == rule_list[i].table_line) { + continue; + } + + int table_id = table_schema_manager_get_table_id(maat_instance->table_schema_mgr, rule_list[i].table_name); + table_schema = table_schema_get(maat_instance->table_schema_mgr, table_id); + enum table_type table_type = table_schema_get_table_type(table_schema); + if (!table_schema || table_type != TABLE_TYPE_PLUGIN) { + continue; + } + + int foreign_columns[8]; + int n_foreign_column = plugin_table_schema_get_foreign_column(table_schema, foreign_columns); + if (0 == n_foreign_column) { + continue; + } + + _get_foregin_keys(rule_list+i, foreign_columns, n_foreign_column, dir); + rule_with_foreign_key++; + } + + return rule_with_foreign_key; +} + +int maat_cmd_get_foreign_keys_by_prefix(redisContext *ctx, struct serial_rule *rule_list, + int rule_num, const char* dir) +{ + int j = 0; + int foreign_key_size = 0; + int rule_with_foreign_key = 0; + const char *p_foreign = NULL; + + int n_foreign = 0; + int foreign_columns[MAX_FOREIGN_CLMN_NUM]; + + for (int i = 0; i < rule_num; i++) { + j = 1; + n_foreign = 0; + do { + p_foreign = maat_cmd_find_Nth_column(rule_list[i].table_line, j, &foreign_key_size); + if (p_foreign != NULL && foreign_key_size > (int)strlen(foreign_source_prefix) && + 0 == strncmp(p_foreign, foreign_source_prefix, strlen(foreign_source_prefix))) { + foreign_columns[n_foreign] = j; + n_foreign++; + } + j++; + } while (p_foreign != NULL && n_foreign < MAX_FOREIGN_CLMN_NUM); + + if (n_foreign > 0) { + _get_foregin_keys(rule_list+i, foreign_columns, n_foreign, dir); + rule_with_foreign_key++; + } + } + + return rule_with_foreign_key; +} + +struct foreign_conts_track +{ + int rule_idx; + int foreign_idx; +}; + +int _get_maat_redis_value(redisContext *c, struct serial_rule *rule_list, int rule_num) +{ + int i = 0; + int failed_cnt = 0; + UNUSED int ret = 0; + int error_happened = 0; + int *retry_ids = ALLOC(int, rule_num); + char redis_cmd[256] = {0}; + redisReply* reply = NULL; + + for (i = 0; i < rule_num; i++) { + snprintf(redis_cmd, sizeof(redis_cmd), "GET %s:%s,%lu", mr_key_prefix[rule_list[i].op], + rule_list[i].table_name, + rule_list[i].rule_id); + ret = redisAppendCommand(c, redis_cmd); + assert(ret == REDIS_OK); + } + + for (i = 0; i < rule_num; i++) { + ret = maat_cmd_wrap_redis_get_reply(c, &reply); + if (ret == REDIS_ERR) { + fprintf(stderr, "Redis GET %s:%s,%lu failed, redis server error.", mr_key_prefix[rule_list[i].op], + rule_list[i].table_name, rule_list[i].rule_id); + error_happened = 1; + break; + } + + if (reply->type == REDIS_REPLY_STRING) { + rule_list[i].table_line = maat_strdup(reply->str); + } else { + if (reply->type == REDIS_REPLY_NIL) { + retry_ids[failed_cnt] = i; + failed_cnt++; + } else { + fprintf(stderr, "Redis GET %s:%s,%lu failed",mr_key_prefix[rule_list[i].op], + rule_list[i].table_name, rule_list[i].rule_id); + error_happened = 1; + } + } + + freeReplyObject(reply); + reply = NULL; + } + + if (1 == error_happened) { + FREE(retry_ids); + return -1; + } + + int idx = 0; + for (i = 0; i < failed_cnt; i++) { + idx = retry_ids[i]; + snprintf(redis_cmd, sizeof(redis_cmd), "GET %s:%s,%lu", mr_key_prefix[MAAT_OP_DEL], + rule_list[idx].table_name, + rule_list[idx].rule_id); + ret = redisAppendCommand(c, redis_cmd); + } + + for (i = 0; i < failed_cnt; i++) { + idx = retry_ids[i]; + ret = maat_cmd_wrap_redis_get_reply(c, &reply); + if (ret == REDIS_ERR) { + fprintf(stderr, "redis command %s failed, redis server error.", redis_cmd); + free(retry_ids); + return -1; + } + + if (reply->type == REDIS_REPLY_STRING) { + rule_list[idx].table_line = maat_strdup(reply->str); + } else if(reply->type==REDIS_REPLY_ERROR) { + //Deal with Redis response: "Loading Redis is loading the database in memory" + fprintf(stderr, "redis command %s error, reply type=%d, error str=%s", redis_cmd, reply->type, reply->str); + } else { + //Handle type "nil" + fprintf(stderr, "redis command %s failed, reply type=%d", redis_cmd, reply->type); + } + + freeReplyObject(reply); + reply = NULL; + + } + + FREE(retry_ids); + return 0; +} + +int maat_cmd_get_redis_value(redisContext *c, struct serial_rule *rule_list, int rule_num, int print_process) +{ + int max_redis_batch = 4096; + int success_cnt = 0; + int next_print = 10; + + while (success_cnt < rule_num) { + int batch_cnt = MIN(rule_num-success_cnt, max_redis_batch); + int ret = _get_maat_redis_value(c, rule_list+success_cnt, batch_cnt); + if (ret < 0) { + return -1; + } else { + success_cnt += batch_cnt; + } + + if (print_process == 1) { + if ((success_cnt * 100) / rule_num > next_print) { + printf(" >%d%%",next_print); + next_print += 10; + } + } + } + + if (print_process == 1) { + printf(" >100%%\n"); + } + + return 0; +} + +int get_inc_key_list(long long instance_version, long long target_version, + redisContext *c, struct serial_rule **list) +{ + //Returns all the elements in the sorted set at key with a score that instance_version < score <= redis_version. + //The elements are considered to be ordered from low to high scores(instance_version). + redisReply *reply = (redisReply *)redisCommand(c, "ZRANGEBYSCORE %s (%lld %lld", mr_status_sset, + instance_version,target_version); + if (NULL == reply) { + fprintf(stderr, "GET %s failed with a NULL reply, error: %s.", mr_status_sset, c->errstr); + return -1; + } + + assert(reply->type == REDIS_REPLY_ARRAY); + + int rule_num = reply->elements; + if (0 == reply->elements) { + freeReplyObject(reply); + reply = NULL; + return 0; + } + + redisReply *tmp_reply= maat_cmd_wrap_redis_command(c, "ZSCORE %s %s", mr_status_sset, reply->element[0]->str); + if (tmp_reply->type != REDIS_REPLY_STRING) { + fprintf(stderr, "ZSCORE %s %s failed Version: %lld->%lld", mr_status_sset, + reply->element[0]->str, instance_version, target_version); + freeReplyObject(tmp_reply); + tmp_reply = NULL; + freeReplyObject(reply); + reply = NULL; + return -1; + } + + long long nearest_rule_version = maat_cmd_read_redis_integer(tmp_reply); + freeReplyObject(tmp_reply); + tmp_reply = NULL; + + if (nearest_rule_version < 0) { + return -1; + } + + if (nearest_rule_version != instance_version + 1) { + fprintf(stdout, "Noncontinuous VERSION Redis: %lld MAAT: %lld.", + nearest_rule_version, instance_version); + } + + int i = 0; + int j = 0; + char op_str[4] = {0}; + struct serial_rule *s_rule = ALLOC(struct serial_rule, reply->elements); + + for (i = 0, j = 0; i < reply->elements; i++) { + assert(reply->element[i]->type == REDIS_REPLY_STRING); + int ret = sscanf(reply->element[i]->str, "%[^,],%[^,],%lu", op_str, + s_rule[j].table_name, &(s_rule[j].rule_id)); + if (ret != 3 || s_rule[i].rule_id < 0) { + fprintf(stderr, "Invalid Redis Key: %s", reply->element[i]->str); + continue; + } + + if (strncmp(op_str, "ADD", strlen("ADD")) == 0) { + s_rule[j].op = MAAT_OP_ADD; + } else if(strncmp(op_str, "DEL", strlen("DEL")) == 0) { + s_rule[j].op = MAAT_OP_DEL; + } else { + fprintf(stderr, "Invalid Redis Key: %s", reply->element[i]->str); + continue; + } + j++; + } + + rule_num = j; + *list = s_rule; + freeReplyObject(reply); + reply = NULL; + + return rule_num; +} + +void serial_rule_free(struct serial_rule *s_rule) +{ + if (s_rule->table_line != NULL) { + FREE(s_rule->table_line); + } + + if (s_rule->n_foreign > 0) { + for (int i = 0; i < s_rule->n_foreign; i++) { + FREE(s_rule->f_keys[i].filename); + FREE(s_rule->f_keys[i].key); + } + FREE(s_rule->f_keys); + } + + FREE(s_rule); +} + +struct serial_rule *serial_rule_clone(const struct serial_rule *s_rule) +{ + struct serial_rule *new_rule = ALLOC(struct serial_rule, 1); + + new_rule->op = s_rule->op; + new_rule->rule_id = s_rule->rule_id; + new_rule->label_id = s_rule->label_id; + new_rule->timeout = s_rule->timeout; + memcpy(new_rule->table_name, s_rule->table_name, strlen(s_rule->table_name)); + new_rule->n_foreign = s_rule->n_foreign; + new_rule->table_line = ALLOC(char, strlen(s_rule->table_line)); + memcpy(new_rule->table_line, s_rule->table_line, strlen(s_rule->table_line)); + + new_rule->f_keys = ALLOC(struct foreign_key, new_rule->n_foreign); + for (int j = 0; j < new_rule->n_foreign; j++) { + new_rule->f_keys[j].key = ALLOC(char, s_rule->f_keys[j].key_len); + memcpy(new_rule->f_keys[j].key, s_rule->f_keys[j].key, s_rule->f_keys[j].key_len); + new_rule->f_keys[j].filename = ALLOC(char, strlen(s_rule->f_keys[j].filename)); + memcpy(new_rule->f_keys[j].filename, s_rule->f_keys[j].filename, strlen(s_rule->f_keys[j].filename)); + } + + return new_rule; +} + +int recovery_history_version(const struct serial_rule *current, int current_num, + const struct serial_rule *changed, int changed_num, + struct serial_rule **history_result) +{ + int i = 0; + int ret = 0; + unsigned int history_num = 0; + int hash_slot_size = 1; + char hkey[256+20] = {0}; + int tmp = current_num + changed_num; + for (; tmp > 0; tmp = tmp/2) { + hash_slot_size *= 2; + } + + struct serial_rule *s_rule_map = NULL; + struct serial_rule *rule_node = NULL; + for (i = 0; i < current_num; i++) { + snprintf(hkey, sizeof(hkey), "%ld,%s", current[i].rule_id, current[i].table_name); + rule_node = serial_rule_clone(current + i); + HASH_ADD_KEYPTR(hh, s_rule_map, hkey, strlen(hkey), rule_node); + } + + for (i = changed_num - 1; i >= 0; i--) { + snprintf(hkey, sizeof(hkey), "%ld,%s", changed[i].rule_id, changed[i].table_name); + //newly added rule is need to delete from current, so that history version can be recovered. + if (changed[i].op == MAAT_OP_ADD) { + rule_node = NULL; + HASH_FIND(hh, s_rule_map, hkey, strlen(hkey), rule_node); + if (rule_node != NULL) { + HASH_DELETE(hh, s_rule_map, rule_node); + } + serial_rule_free(rule_node); + } else { + rule_node = serial_rule_clone(changed + i); + HASH_ADD_KEYPTR(hh, s_rule_map, hkey, strlen(hkey), rule_node); + } + } + + history_num = HASH_CNT(hh, s_rule_map); + struct serial_rule *array = ALLOC(struct serial_rule, history_num); + struct serial_rule *elem_node = NULL; + struct serial_rule *tmp_node = NULL; + i = 0; + HASH_ITER(hh, s_rule_map, elem_node, tmp_node) { + memcpy(&array[i], elem_node, sizeof(struct serial_rule)); + array[i].op = MAAT_OP_ADD; + i++; + } + elem_node = NULL; + tmp_node = NULL; + + *history_result = array; + ret = history_num; + + HASH_ITER(hh, s_rule_map, elem_node, tmp_node) { + HASH_DELETE(hh, s_rule_map, elem_node); + serial_rule_free(elem_node); + } + + return ret; +} + +int maat_cmd_get_rm_key_list(redisContext *c, long long instance_version, long long desired_version, + long long *new_version, struct table_schema_manager* table_schema_mgr, + struct serial_rule **list, int *update_type, int cumulative_off) +{ + int rule_num = 0; + long long target_version = 0; + struct serial_rule *s_rule_array = NULL; + + redisReply *reply = (redisReply *)redisCommand(c, "GET MAAT_VERSION"); + if (reply != NULL) { + if (reply->type == REDIS_REPLY_NIL || reply->type == REDIS_REPLY_ERROR) { + fprintf(stderr, "GET MAAT_VERSION failed, maybe Redis is busy."); + freeReplyObject(reply); + reply = NULL; + return -1; + } + } else { + fprintf(stderr, "GET MAAT_VERSION failed with NULL reply, error: %s.", c->errstr); + return -1; + } + + long long redis_version = maat_cmd_read_redis_integer(reply); + if (redis_version < 0) { + if (reply->type == REDIS_REPLY_ERROR) { + fprintf(stderr, "Redis Communication error: %s.", reply->str); + } + return -1; + } + + freeReplyObject(reply); + reply = NULL; + + if (redis_version == instance_version) { + return 0; + } + + if (0 == instance_version || desired_version != 0) { + goto FULL_UPDATE; + } + + if (redis_version < instance_version) { + fprintf(stderr, "VERSION roll back MAAT: %lld -> Redis: %lld.", instance_version, redis_version); + goto FULL_UPDATE; + } + + if (redis_version > instance_version && 1 == cumulative_off) { + target_version = instance_version; + } else { + target_version = redis_version - 1; + } + + do { + target_version++; + rule_num = get_inc_key_list(instance_version, target_version, c, &s_rule_array); + if (rule_num > 0) { + break; + } else if (rule_num < 0) { + goto FULL_UPDATE; + } else { + //ret=0, nothing to do. + } + + } while (0 == rule_num && target_version <= redis_version && 1 == cumulative_off); + + if (0 == rule_num) { + fprintf(stdout, "Got nothing after ZRANGEBYSCORE %s (%lld %lld, cumulative %s", mr_status_sset, + instance_version, target_version-1, cumulative_off == 1 ? "OFF" : "ON"); + return 0; + } + + fprintf(stdout, "Inc Update from instance_version %lld to %lld (%d entries).", + instance_version, target_version, rule_num); + + *list = s_rule_array; + *update_type = CM_UPDATE_TYPE_INC; + *new_version = target_version; + return rule_num; + +FULL_UPDATE: + fprintf(stdout, "Initiate full udpate from instance_version %lld to %lld.", instance_version, + desired_version == 0 ? redis_version : desired_version); + size_t append_cmd_cnt = 0; + int ret = redisAppendCommand(c, "MULTI"); + append_cmd_cnt++; + + ret = redisAppendCommand(c, "GET MAAT_VERSION"); + append_cmd_cnt++; + + ret = redisAppendCommand(c, "KEYS EFFECTIVE_RULE:*"); + append_cmd_cnt++; + + size_t i = 0; + //consume reply "OK" and "QUEUED". + for (i = 0; i < append_cmd_cnt; i++) { + maat_cmd_wrap_redis_get_reply(c, &reply); + freeReplyObject(reply); + reply = NULL; + } + + reply = maat_cmd_wrap_redis_command(c, "EXEC"); + if (NULL == reply) { + fprintf(stderr, "Redis Communication error: %s.", c->errstr); + return -1; + } + + if (reply->type != REDIS_REPLY_ARRAY) { + fprintf(stderr, "Invalid Redis Key List type %d", reply->type); + freeReplyObject(reply); + reply = NULL; + return -1; + } + + *new_version = maat_cmd_read_redis_integer(reply->element[0]); + redisReply *sub_reply = reply->element[1]; + if (sub_reply->type != REDIS_REPLY_ARRAY) { + fprintf(stderr, "Invalid Redis Key List type %d", sub_reply->type); + freeReplyObject(reply); + reply = NULL; + return -1; + } + + size_t full_idx = 0; + s_rule_array = ALLOC(struct serial_rule, sub_reply->elements); + for (i = 0, full_idx = 0; i < sub_reply->elements; i++) { + if (sub_reply->element[i]->type != REDIS_REPLY_STRING) { + fprintf(stderr, "Invalid Redis Key Type: %d", sub_reply->element[i]->type); + continue; + } + + ret = sscanf(sub_reply->element[i]->str, "%*[^:]:%[^,],%ld", + s_rule_array[full_idx].table_name, + &(s_rule_array[full_idx].rule_id)); + s_rule_array[full_idx].op = MAAT_OP_ADD; + + if (ret != 2 || s_rule_array[full_idx].rule_id < 0 || strlen(s_rule_array[full_idx].table_name) == 0) { + fprintf(stderr, "Invalid Redis Key Format: %s", sub_reply->element[i]->str); + continue; + } + + if (table_schema_mgr) { + int table_id = table_schema_manager_get_table_id(table_schema_mgr, s_rule_array[full_idx].table_name); + //Unrecognized table. + if (table_id < 0) { + continue; + } + } + full_idx++; + } + + rule_num = full_idx; + freeReplyObject(reply); + reply = NULL; + + if (desired_version != 0) { + struct serial_rule *changed_rule_array = NULL; + int changed_rule_num = get_inc_key_list(desired_version, redis_version, c, &changed_rule_array); + if (changed_rule_num < 0) { + fprintf(stderr, "Recover history version %lld faild where as redis version is %lld.", desired_version, redis_version); + } else if(0 == changed_rule_num) { + fprintf(stderr, "Nothing to recover from history version %lld to redis version is %lld.", desired_version, redis_version); + } else { + struct serial_rule *history_rule_array = NULL; + ret = recovery_history_version(s_rule_array, full_idx, changed_rule_array, changed_rule_num, &history_rule_array); + if (ret > 0) { + FREE(s_rule_array); + s_rule_array = history_rule_array; + rule_num = ret; + *new_version = desired_version; + fprintf(stdout, "Successfully recovered from history version %lld to redis version is %lld.", + desired_version, redis_version); + } + } + FREE(changed_rule_array); + } + + *list = s_rule_array; + *update_type = CM_UPDATE_TYPE_FULL; + fprintf(stdout, "Full update %d keys of version %lld.", rule_num, *new_version); + + return rule_num ; +} + +void _get_foreign_conts(redisContext *c, struct serial_rule *rule_list, int rule_num, int print_fn) +{ + int i = 0; + int j = 0; + UNUSED int ret = 0; + int key_num = 0; + struct serial_rule *s_rule = NULL; + struct foreign_conts_track *track = ALLOC(struct foreign_conts_track, rule_num * MAX_FOREIGN_CLMN_NUM); + + for (i = 0; i < rule_num; i++) { + s_rule = rule_list + i; + if (s_rule->n_foreign == 0) { + continue; + } + + if (s_rule->op == MAAT_OP_DEL) { + for (j = 0; j < rule_list[i].n_foreign; j++) { + if (NULL == rule_list[i].f_keys[j].filename) { + continue; + } + + ret = remove(rule_list[i].f_keys[j].filename); + if (ret == -1) { + fprintf(stderr, "Foreign content file %s remove failed.", + rule_list[i].f_keys[j].filename); + } + } + } else { + for (j = 0; j < s_rule->n_foreign; j++) { + if (NULL == rule_list[i].f_keys[j].filename) { + continue; + } + + struct stat file_info; + ret = stat(s_rule->f_keys[j].filename, &file_info); + if (0 == ret) { + continue; + } + + char redis_cmd[256] = {0}; + snprintf(redis_cmd, sizeof(redis_cmd), "GET %s", s_rule->f_keys[j].key); + ret = redisAppendCommand(c, redis_cmd); + track[key_num].rule_idx = i; + track[key_num].foreign_idx = j; + key_num++; + assert(ret == REDIS_OK); + } + } + } + + redisReply *reply = NULL; + for (i = 0; i < key_num; i++) { + ret = maat_cmd_wrap_redis_get_reply(c, &reply); + if (ret == REDIS_ERR) { + fprintf(stderr, "Get %s,%lu foreign key %s content failed, redis server error.", + rule_list[track[i].rule_idx].table_name, + rule_list[track[i].rule_idx].rule_id, + rule_list[track[i].rule_idx].f_keys[track[i].foreign_idx].key); + break; + } + + if (reply->type != REDIS_REPLY_STRING) { + fprintf(stderr, "Get %s,%lu foreign key %s content failed.", + rule_list[track[i].rule_idx].table_name, + rule_list[track[i].rule_idx].rule_id, + rule_list[track[i].rule_idx].f_keys[track[i].foreign_idx].key); + continue; + } else { + s_rule = rule_list+track[i].rule_idx; + FILE *fp = fopen(s_rule->f_keys[track[i].foreign_idx].filename, "w"); + if (NULL == fp) { + fprintf(stderr, "Write foreign content failed: fopen %s error.", + s_rule->f_keys[track[i].foreign_idx].filename); + } else { + fwrite(reply->str, 1, reply->len, fp); + fclose(fp); + fp = NULL; + if (1 == print_fn) { + printf("Written foreign content %s\n", s_rule->f_keys[track[i].foreign_idx].filename); + } + } + } + + freeReplyObject(reply); + reply = NULL; + } + + FREE(track); + return; +} + +void maat_cmd_get_foreign_conts(redisContext *c, struct serial_rule *rule_list, int rule_num, int print_fn) +{ + int max_redis_batch = 4096; + int success_cnt = 0; + + while (success_cnt < rule_num) { + int batch_cnt = MIN(rule_num - success_cnt, max_redis_batch); + _get_foreign_conts(c, rule_list + success_cnt, batch_cnt, print_fn); + success_cnt += batch_cnt; + } +} + +int invalidate_line(char *line, enum table_type table_type, int valid_column_seq) +{ + int i = maat_cmd_get_valid_flag_offset(line, table_type, valid_column_seq); + if (i < 0) { + return -1; + } + + line[i] = '0'; + + return 0; +} + +void maat_cmd_rewrite_table_line_with_foreign(struct serial_rule *s_rule) +{ + int i = 0; + size_t fn_size = 0; + + for (i = 0; i < s_rule->n_foreign; i++) { + fn_size += strlen(s_rule->f_keys[i].filename); + } + + char *rewrite_line = ALLOC(char, strlen(s_rule->table_line) + fn_size); + char *pos_rewrite_line = rewrite_line; + const char *pos_origin_line = s_rule->table_line; + + for (i = 0; i < s_rule->n_foreign; i++) { + int origin_column_size = 0; + const char *origin_column = maat_cmd_find_Nth_column(s_rule->table_line, + s_rule->f_keys[i].column, + &origin_column_size); + strncat(pos_rewrite_line, pos_origin_line, origin_column - pos_origin_line); + pos_rewrite_line += origin_column - pos_origin_line; + pos_origin_line = origin_column+origin_column_size; + strncat(pos_rewrite_line, s_rule->f_keys[i].filename, strlen(s_rule->f_keys[i].filename)); + pos_rewrite_line += strlen(s_rule->f_keys[i].filename); + } + + strncat(pos_rewrite_line, pos_origin_line, strlen(s_rule->table_line) - (pos_origin_line - s_rule->table_line)); + FREE(s_rule->table_line); + s_rule->table_line = rewrite_line; +} + +void expected_reply_add(struct expected_reply* expected, int s_rule_seq, int type, long long integer) +{ + int i = expected->possible_reply_num; + assert(i < POSSIBLE_REDIS_REPLY_SIZE); + expected->s_rule_seq = s_rule_seq; + expected->possible_replies[i].type = type; + expected->possible_replies[i].integer = integer; + expected->possible_reply_num++; +} + +int redlock_try_lock(redisContext *c, const char *lock_name, long long expire) +{ + int ret = 0; + + redisReply *reply = maat_cmd_wrap_redis_command(c, "SET %s locked NX PX %lld", lock_name, expire); + if (reply->type == REDIS_REPLY_NIL) { + ret = 0; + } else { + ret = 1; + } + + freeReplyObject(reply); + reply = NULL; + + return ret; +} + +long long _exec_serial_rule_begin(redisContext* c, size_t rule_num, size_t renew_rule_num, + int *renew_allowed, long long *transaction_version) +{ + int ret = -1; + redisReply *data_reply = NULL; + + if (renew_rule_num > 0) { + while (0 == redlock_try_lock(c, mr_expire_lock, mr_expire_lock_timeout_ms)) { + usleep(1000); + } + *renew_allowed = 1; + } + + if (rule_num > renew_rule_num) { + data_reply = maat_cmd_wrap_redis_command(c, "INCRBY MAAT_PRE_VER 1"); + *transaction_version = maat_cmd_read_redis_integer(data_reply); + freeReplyObject(data_reply); + data_reply = NULL; + if (*transaction_version < 0) { + return -1; + } + } + + if (*renew_allowed == 1 || rule_num > renew_rule_num) { + data_reply = maat_cmd_wrap_redis_command(c, "MULTI"); + freeReplyObject(data_reply); + data_reply = NULL; + ret = 0; + } + + return ret; +} + +void redlock_unlock(redisContext *c, const char *lock_name) +{ + redisReply *reply = maat_cmd_wrap_redis_command(c, "DEL %s", lock_name); + freeReplyObject(reply); + reply = NULL; +} + +const char* lua_exec_done= +"local maat_version=redis.call(\'incrby\', KEYS[1], 1);" +"local transaction=redis.call(\'lrange\', KEYS[4], 0, -1);" +"for k,v in pairs(transaction) do" +" redis.call(\'zadd\', KEYS[2], maat_version, v);" +"end;" +"redis.call(\'del\', KEYS[4]);" +"redis.call(\'zadd\', KEYS[3], ARGV[1], maat_version);" +"return maat_version;"; +redisReply* _exec_serial_rule_end(redisContext *c, const char *transaction_list, long long server_time, + int renew_allowed, struct expected_reply *expect_reply, size_t *cnt) +{ + redisReply *data_reply = NULL; + + if (1 == renew_allowed) { + redlock_unlock(c, mr_expire_lock); + expect_reply[*cnt].s_rule_seq = -1; + (*cnt)++; + } + + if (strlen(transaction_list) > 0) { + data_reply = maat_cmd_wrap_redis_command(c, "eval %s 4 MAAT_VERSION %s %s %s %lld", + lua_exec_done, + mr_status_sset, + mr_version_sset, + transaction_list, + server_time); + freeReplyObject(data_reply); + data_reply = NULL; + expected_reply_add(expect_reply + *cnt, -1, REDIS_REPLY_INTEGER, 0); + (*cnt)++; + } + + data_reply = maat_cmd_wrap_redis_command(c, "EXEC"); + + return data_reply; +} + +void _exec_serial_rule(redisContext *c, const char *transaction_list, struct serial_rule *s_rule, size_t rule_num, + struct expected_reply *expect_reply, size_t *cnt, size_t offset, int renew_allowed) +{ + size_t i = 0; + size_t append_cmd_cnt = 0; + redisReply *data_reply = NULL; + + for (i = 0; i < rule_num; i++) { + switch (s_rule[i].op) { + case MAAT_OP_ADD: + redisAppendCommand(c, "SET %s:%s,%lu %s", + mr_key_prefix[MAAT_OP_ADD], + s_rule[i].table_name, + s_rule[i].rule_id, + s_rule[i].table_line); + expected_reply_add(expect_reply+*cnt, i+offset, REDIS_REPLY_STATUS, 0); + (*cnt)++; + append_cmd_cnt++; + //Allowing add duplicated members for rule id recycling. + redisAppendCommand(c, "RPUSH %s ADD,%s,%lu", + transaction_list, + s_rule[i].table_name, + s_rule[i].rule_id); + expected_reply_add(expect_reply+*cnt, -1, REDIS_REPLY_INTEGER, 0); + (*cnt)++; + append_cmd_cnt++; + if (s_rule[i].timeout > 0) { + redisAppendCommand(c, "ZADD %s %lld %s,%lu", + mr_expire_sset, + s_rule[i].timeout, + s_rule[i].table_name, + s_rule[i].rule_id); + expected_reply_add(expect_reply+*cnt, i+offset, REDIS_REPLY_INTEGER, 1); + expected_reply_add(expect_reply+*cnt, i+offset, REDIS_REPLY_INTEGER, 0); + (*cnt)++; + append_cmd_cnt++; + } + + if (s_rule[i].label_id > 0) + { + redisAppendCommand(c, "ZADD %s %d %s,%lu", + mr_label_sset, + s_rule[i].label_id, + s_rule[i].table_name, + s_rule[i].rule_id); + expected_reply_add(expect_reply+*cnt, i+offset, REDIS_REPLY_INTEGER, 1); + expected_reply_add(expect_reply+*cnt, i+offset, REDIS_REPLY_INTEGER, 0); + + (*cnt)++; + + append_cmd_cnt++; + } + break; + case MAAT_OP_DEL: + redisAppendCommand(c, "RENAME %s:%s,%lu %s:%s,%lu", + mr_key_prefix[MAAT_OP_ADD], + s_rule[i].table_name, + s_rule[i].rule_id, + mr_key_prefix[MAAT_OP_DEL], + s_rule[i].table_name, + s_rule[i].rule_id); + expected_reply_add(expect_reply+*cnt, i+offset, REDIS_REPLY_STATUS, 0); + (*cnt)++; + append_cmd_cnt++; + + redisAppendCommand(c, "EXPIRE %s:%s,%lu %d", + mr_key_prefix[MAAT_OP_DEL], + s_rule[i].table_name, + s_rule[i].rule_id, + MAAT_REDIS_SYNC_TIME); + expected_reply_add(expect_reply+*cnt, i+offset, REDIS_REPLY_INTEGER, 1); + (*cnt)++; + append_cmd_cnt++; + + //NX: Don't update already exisiting elements. Always add new elements. + redisAppendCommand(c, "RPUSH %s DEL,%s,%lu", + transaction_list, + s_rule[i].table_name, + s_rule[i].rule_id); + expected_reply_add(expect_reply+*cnt, -1, REDIS_REPLY_INTEGER, 0); + (*cnt)++; + append_cmd_cnt++; + + // Try to remove from expiration sorted set, no matter wheather it exists or not. + redisAppendCommand(c, "ZREM %s %s,%lu", + mr_expire_sset, + s_rule[i].table_name, + s_rule[i].rule_id); + expected_reply_add(expect_reply+*cnt, -1, REDIS_REPLY_INTEGER, 0); + (*cnt)++; + append_cmd_cnt++; + + // Try to remove from label sorted set, no matter wheather it exists or not. + redisAppendCommand(c, "ZREM %s %s,%lu", + mr_label_sset, + s_rule[i].table_name, + s_rule[i].rule_id); + expected_reply_add(expect_reply+*cnt, -1, REDIS_REPLY_INTEGER, 0); + (*cnt)++; + append_cmd_cnt++; + break; + case MAAT_OP_RENEW_TIMEOUT: + if (renew_allowed != 1) { + continue; + } + //s_rule[i].timeout>0 was checked by caller. + redisAppendCommand(c, "ZADD %s %lld %s,%lu", + mr_expire_sset, + s_rule[i].timeout, + s_rule[i].table_name, + s_rule[i].rule_id); + expected_reply_add(expect_reply+*cnt, -1, REDIS_REPLY_INTEGER, 0); + (*cnt)++; + append_cmd_cnt++; + + break; + default: + assert(0); + break; + } + } + + for (i = 0; i < append_cmd_cnt; i++) { + maat_cmd_wrap_redis_get_reply(c, &data_reply); + freeReplyObject(data_reply); + data_reply=NULL; + } +} + +int mr_transaction_success(redisReply *data_reply) +{ + if (data_reply->type == REDIS_REPLY_NIL) { + return 0; + } else { + return 1; + } +} + +int mr_operation_success(redisReply *actual_reply, struct expected_reply *expected) +{ + if (expected->possible_replies[0].type != actual_reply->type) { + return 0; + } + + for (int i = 0; i < expected->possible_reply_num; i++) { + if (expected->possible_replies[i].type == REDIS_REPLY_INTEGER && + expected->possible_replies[i].type == actual_reply->type && + expected->possible_replies[i].integer == actual_reply->integer) { + return 1; + } + + if (expected->possible_replies[i].type == REDIS_REPLY_STATUS && + expected->possible_replies[i].type == actual_reply->type && + 0 == strcasecmp(actual_reply->str, "OK")) { + return 1; + } + } + + return 0; +} + +int maat_cmd_exec_serial_rule(redisContext *c, struct serial_rule *s_rule, size_t serial_rule_num, long long server_time) +{ + size_t i = 0; + size_t rule_seq = 0; + size_t multi_cmd_cnt = 0; + size_t success_cnt = 0; + size_t renew_num = 0; + size_t max_redis_batch = 1024; + int renew_allowed = 0; + int last_failed = -1; + redisReply *p = NULL; + redisReply *transaction_reply = NULL; + const int MAX_REDIS_OP_PER_SRULE = 8; + char transaction_list[NAME_MAX * 2] = {0}; + long long transaction_version = 0; + long long transaction_finished_version = 0; + size_t max_multi_cmd_num = MAX_REDIS_OP_PER_SRULE * serial_rule_num + 2;// 2 for operation in _exec_serial_rule_end() + struct expected_reply *expected_reply = ALLOC(struct expected_reply, max_multi_cmd_num); + + for (i = 0; i < serial_rule_num; i++) { + if (s_rule[i].op == MAAT_OP_RENEW_TIMEOUT) { + renew_num++; + } + } + + int ret = _exec_serial_rule_begin(c, serial_rule_num, renew_num, &renew_allowed, &transaction_version); + //Preconditions for transaction are not satisfied. + if (ret != 0) { + success_cnt = -1; + goto error_out; + } + + if (transaction_version > 0) { + snprintf(transaction_list, sizeof(transaction_list), "MAAT_TRANSACTION_%lld", transaction_version); + } + + while (success_cnt < serial_rule_num) { + size_t batch_cnt = MIN(serial_rule_num - success_cnt, max_redis_batch); + _exec_serial_rule(c, transaction_list, s_rule + success_cnt, batch_cnt, expected_reply, &multi_cmd_cnt, + success_cnt, renew_allowed); + assert(multi_cmd_cntelements == multi_cmd_cnt); + for (i = 0; i < multi_cmd_cnt; i++) { + p = transaction_reply->element[i]; + //failed is acceptable + //or transaciton is success + //or continuation of last failed + if (expected_reply[i].s_rule_seq == -1 || 1 == mr_operation_success(p, expected_reply+i) || last_failed == expected_reply[i].s_rule_seq) { + continue; + } + rule_seq = expected_reply[i].s_rule_seq; + fprintf(stderr, "%s %s %lu failed, rule id maybe conflict or not exist.", + mr_op_str[s_rule[rule_seq].op], s_rule[rule_seq].table_name, + s_rule[rule_seq].rule_id); + success_cnt--; + last_failed = rule_seq; + } + } else { + success_cnt = -1; + } + + if (transaction_version > 0) { + transaction_finished_version = maat_cmd_read_redis_integer(transaction_reply->element[multi_cmd_cnt-1]); + fprintf(stdout, "Redis transaction MAAT_PRE_VER = %lld , MAAT_VERSION = %lld ", + transaction_version, transaction_finished_version); + } + + freeReplyObject(transaction_reply); + transaction_reply = NULL; + +error_out: + if (renew_num > 0 && renew_allowed != 1) { + for (i = 0; i < (unsigned int)serial_rule_num; i++) { + if (s_rule[i].op == MAAT_OP_RENEW_TIMEOUT) { + fprintf(stdout, "%s %s %lu is not allowed due to lock contention.", + mr_op_str[MAAT_OP_RENEW_TIMEOUT], s_rule[i].table_name, + s_rule[i].rule_id); + } + } + + if (success_cnt > 0) { + success_cnt -= renew_num; + } + } + + FREE(expected_reply); + + return success_cnt; +} + +void cleanup_update_status(redisContext *c) +{ + long long version_upper_bound = 0; + long long version_lower_bound = 0; + long long version_num = 0; + long long entry_num = 0; + + long long server_time = maat_cmd_redis_server_time_s(c); + if (!server_time) { + return; + } + + redisReply *reply = maat_cmd_wrap_redis_command(c, "MULTI"); + freeReplyObject(reply); + reply = NULL; + + int append_cmd_cnt = 0; + redisAppendCommand(c, "ZRANGEBYSCORE %s -inf %lld", + mr_version_sset, server_time - MAAT_REDIS_SYNC_TIME); + append_cmd_cnt++; + + redisAppendCommand(c, "ZREMRANGEBYSCORE %s -inf %lld", + mr_version_sset,server_time - MAAT_REDIS_SYNC_TIME); + append_cmd_cnt++; + + //consume reply "OK" and "QUEUED". + for(int i = 0; i < append_cmd_cnt; i++) { + maat_cmd_wrap_redis_get_reply(c, &reply); + freeReplyObject(reply); + reply = NULL; + } + + redisReply *sub_reply = NULL; + reply = maat_cmd_wrap_redis_command(c, "EXEC"); + if (reply->type != REDIS_REPLY_ARRAY) { + goto error_out; + } + + sub_reply = reply->element[0]; + if (sub_reply->type != REDIS_REPLY_ARRAY) { + goto error_out; + } + + version_num = sub_reply->elements; + if (version_num == 0) { + goto error_out; + } + + version_lower_bound = maat_cmd_read_redis_integer(sub_reply->element[0]); + version_upper_bound = maat_cmd_read_redis_integer(sub_reply->element[sub_reply->elements-1]); + freeReplyObject(reply); + reply = NULL; + + //To deal with maat_version reset to 0, do NOT use -inf as lower bound intentionally. + reply = maat_cmd_wrap_redis_command(c, "ZREMRANGEBYSCORE %s %lld %lld", mr_status_sset, + version_lower_bound, version_upper_bound); + entry_num = maat_cmd_read_redis_integer(reply); + freeReplyObject(reply); + reply = NULL; + + fprintf(stdout, "Clean up update status from version %lld to %lld (%lld versions, %lld entries).", + version_lower_bound, version_upper_bound, version_num, entry_num); + return; + +error_out: + freeReplyObject(reply); + reply = NULL; + + return; +} + +void check_maat_expiration(redisContext *c) +{ + UNUSED int ret = 0; + + long long server_time = maat_cmd_redis_server_time_s(c); + if (!server_time) { + return; + } + + redisReply *data_reply= maat_cmd_wrap_redis_command(c, "ZRANGEBYSCORE %s -inf %lld", mr_expire_sset, server_time); + if (data_reply->type != REDIS_REPLY_ARRAY || 0 == data_reply->elements) { + freeReplyObject(data_reply); + data_reply = NULL; + return; + } + + size_t s_rule_num = data_reply->elements; + struct serial_rule *s_rule = ALLOC(struct serial_rule, s_rule_num); + + for (size_t i = 0; i < s_rule_num; i++) { + s_rule[i].op = MAAT_OP_DEL; + ret = sscanf(data_reply->element[i]->str, "%[^,],%ld", s_rule[i].table_name, &(s_rule[i].rule_id)); + assert(ret == 2); + } + freeReplyObject(data_reply); + data_reply = NULL; + + int success_cnt = maat_cmd_exec_serial_rule(c, s_rule, s_rule_num, server_time); + if (success_cnt < 0) { + fprintf(stderr, "maat_cmd_exec_serial_rule failed.\n"); + } else if (success_cnt == (int)s_rule_num) { + fprintf(stdout, "Succesfully expired %zu rules in Redis.", s_rule_num); + } else { + fprintf(stderr, "Failed to expired %d of %zu rules in Redis, try later.", + s_rule_num - success_cnt, s_rule_num); + } + + FREE(s_rule); +} + +void redis_monitor_traverse(long long version, struct source_redis_ctx *mr_ctx, + void (*start_fn)(long long, int, void *), + int (*update_fn)(const char *, const char *, void *), + void (*finish_fn)(void *), + void *u_param) +{ + int i = 0; + int ret = 0; + int table_id = 0; + int empty_value_num = 0; + int no_table_num = 0; + int call_update_num = 0; + int valid_column = -1; + enum table_type table_type; + enum scan_type scan_type; + struct table_schema *table_schema = NULL; + + //authorized to write + if (mr_ctx->write_ctx != NULL && mr_ctx->write_ctx->err == 0) { + //For thread safe, deliberately use redis_read_ctx but not redis_write_ctx. + if (1 == redlock_try_lock(mr_ctx->read_ctx, mr_expire_lock, mr_expire_lock_timeout_ms)) { + check_maat_expiration(mr_ctx->read_ctx); + cleanup_update_status(mr_ctx->read_ctx); + redlock_unlock(mr_ctx->read_ctx, mr_expire_lock); + } + } + + if (NULL == mr_ctx->read_ctx || mr_ctx->read_ctx->err) { + if (time(NULL) - mr_ctx->last_reconnect_time < MAAT_REDIS_RECONNECT_INTERVAL_S) { + return; + } + + mr_ctx->last_reconnect_time = time(NULL); + if (mr_ctx->read_ctx != NULL) { + redisFree(mr_ctx->read_ctx); + } + fprintf(stdout, "Reconnecting..."); + + mr_ctx->read_ctx = maat_cmd_connect_redis(mr_ctx->redis_ip, mr_ctx->redis_port, mr_ctx->redis_db); + if (NULL == mr_ctx->read_ctx) { + return; + } else { + version = 0; //Trigger full update when reconnect to redis. + } + } + + struct maat *maat_instance = (struct maat *)u_param; + struct serial_rule *rule_list = NULL; + long long new_version = 0; + int update_type = CM_UPDATE_TYPE_INC; + + int rule_num = maat_cmd_get_rm_key_list(mr_ctx->read_ctx, version, maat_instance->load_specific_version, + &new_version, maat_instance->table_schema_mgr, &rule_list, + &update_type, maat_instance->cumulative_update_off); + //redis communication error + if (rule_num < 0) { + redisFree(mr_ctx->read_ctx); + mr_ctx->read_ctx = NULL; + return; + } + + maat_instance->load_specific_version = 0;//only valid for one time. + //error or nothing changed + if (0 == rule_num && update_type == CM_UPDATE_TYPE_INC) { + return; + } + + if (rule_num > 0) { + ret = maat_cmd_get_redis_value(mr_ctx->read_ctx, rule_list, rule_num, 0); + //redis communication error + if (ret < 0) { + redisFree(mr_ctx->read_ctx); + mr_ctx->read_ctx = NULL; + fprintf(stderr, "Get Redis value failed, abandon update and close connection."); + goto clean_up; + } + + for (i = 0; i < rule_num; i++) { + if (NULL == rule_list[i].table_line) { + empty_value_num++; + } + } + + if (empty_value_num == rule_num) { + fprintf(stdout, "All %d rules are empty, abandon update.", empty_value_num); + goto clean_up; + } + + ret = get_foreign_keys_define(mr_ctx->read_ctx, rule_list, rule_num, maat_instance, maat_instance->foreign_cont_dir); + if (ret > 0) { + maat_cmd_get_foreign_conts(mr_ctx->read_ctx, rule_list, rule_num, 0); + } + } + + start_fn(new_version, update_type, u_param); + fprintf(stdout, "Start %s update: %lld -> %lld (%d entries).", + update_type==CM_UPDATE_TYPE_INC?"INC":"FULL", version, new_version, rule_num); + + for (i = 0; i < rule_num; i++) { + if (NULL == rule_list[i].table_line) { + continue; + } + + table_id = table_schema_manager_get_table_id(maat_instance->table_schema_mgr, rule_list[i].table_name); + //Unrecognized table. + if (table_id < 0) { + no_table_num++; + continue; + } + + table_schema = table_schema_get(maat_instance->table_schema_mgr, table_id); + if (rule_list[i].op == MAAT_OP_DEL) { + scan_type = table_schema_get_scan_type(table_schema); + table_type = table_schema_get_table_type(table_schema); + table_schema = table_schema_get_by_scan_type(maat_instance->table_schema_mgr, table_id, scan_type, NULL); + valid_column = table_schema_get_valid_flag_column(table_schema); + ret = invalidate_line(rule_list[i].table_line, table_type, valid_column); + if (ret < 0) { + fprintf(stdout, "Invalidate line failed, invaid format %s .", rule_list[i].table_line); + continue; + } + } + + if (rule_list[i].n_foreign > 0) { + maat_cmd_rewrite_table_line_with_foreign(rule_list+i); + } + + update_fn(rule_list[i].table_name, rule_list[i].table_line, u_param); + call_update_num++; + } + + finish_fn(u_param); + + if (call_update_num < rule_num) { + fprintf(stdout, "Load %d entries to match engine, no tablle: %d, empty value: %d.", + call_update_num, no_table_num, empty_value_num); + } + +clean_up: + for (i = 0; i < rule_num; i++) { + maat_cmd_empty_serial_rule(rule_list + i); + } + + FREE(rule_list); +} \ No newline at end of file diff --git a/src/maat_rule.cpp b/src/maat_rule.cpp index 45e57b9..8764212 100644 --- a/src/maat_rule.cpp +++ b/src/maat_rule.cpp @@ -14,12 +14,16 @@ #include #include #include +#include #include +#include "utils.h" +#include "json2iris.h" +#include "maat_utils.h" #include "maat_rule.h" #include "maat_config_monitor.h" -#include "utils.h" -#include "maat_utils.h" +#include "maat_redis_monitor.h" + #include "maat_table_runtime.h" #include "maat_table_schema.h" @@ -100,7 +104,16 @@ int maat_update_cb(const char *table_name, const char *line, void *u_param) struct maat *maat_instance =(struct maat *)u_param; struct maat_runtime* maat_rt = NULL; int table_id = table_schema_manager_get_table_id(maat_instance->table_schema_mgr, table_name); + if (table_id < 0) { + fprintf(stderr, "update warning, unknown table name %s\n", table_name); + return -1; + } + struct table_schema* table_schema = table_schema_get(maat_instance->table_schema_mgr, table_id); + if (NULL == table_schema) { + fprintf(stderr, "update warning, table name %s doesn't have table schema\n", table_name); + return -1; + } if (maat_instance->creating_maat_rt != NULL) { maat_rt = maat_instance->creating_maat_rt; @@ -177,10 +190,21 @@ void *rule_monitor_loop(void *arg) } pthread_mutex_unlock(&(maat_instance->background_update_mutex)); + char md5_tmp[MD5_DIGEST_LENGTH * 2 + 1] = {0}; + char err_str[NAME_MAX] = {0}; + struct stat attrib; while (maat_instance->is_running) { usleep(maat_instance->rule_update_checking_interval_ms * 1000); if( 0 == pthread_mutex_trylock(&(maat_instance->background_update_mutex))) { switch (maat_instance->input_mode) { + case DATA_SOURCE_REDIS: + redis_monitor_traverse(maat_instance->maat_version, + &(maat_instance->mr_ctx), + maat_start_cb, + maat_update_cb, + maat_finish_cb, + maat_instance); + break; case DATA_SOURCE_IRIS_FILE: config_monitor_traverse(maat_instance->maat_version, maat_instance->iris_ctx.inc_dir, @@ -189,6 +213,29 @@ void *rule_monitor_loop(void *arg) maat_finish_cb, maat_instance); break; + case DATA_SOURCE_JSON_FILE: + memset(md5_tmp, 0, sizeof(md5_tmp)); + stat(maat_instance->json_ctx.json_file, &attrib); + if (memcmp(&attrib.st_ctim, &(maat_instance->json_ctx.last_md5_time), sizeof(attrib.st_ctim))) { + maat_instance->json_ctx.last_md5_time = attrib.st_ctim; + md5_file(maat_instance->json_ctx.json_file, md5_tmp); + if (0 != strcmp(md5_tmp, maat_instance->json_ctx.effective_json_md5)) { + ret = load_maat_json_file(maat_instance, maat_instance->json_ctx.json_file, err_str, sizeof(err_str)); + if (ret < 0) { + fprintf(stdout, "Maat re-initiate with JSON file %s (md5=%s)failed: %s", + maat_instance->json_ctx.json_file, md5_tmp, err_str); + } else { + config_monitor_traverse(0, maat_instance->json_ctx.iris_file, + maat_start_cb, + maat_update_cb, + maat_finish_cb, + maat_instance); + fprintf(stdout, "Maat re-initiate with JSON file %s success, md5: %s", + maat_instance->json_ctx.json_file, md5_tmp); + } + } + } + break; default: break; } @@ -231,7 +278,20 @@ void *rule_monitor_loop(void *arg) maat_runtime_destroy(maat_instance->maat_rt); maat_garbage_bin_free(maat_instance->garbage_bin); table_schema_manager_destroy(maat_instance->table_schema_mgr); - free(maat_instance); + + if (maat_instance->input_mode == DATA_SOURCE_REDIS) { + if (maat_instance->mr_ctx.read_ctx != NULL) { + redisFree(maat_instance->mr_ctx.read_ctx); + maat_instance->mr_ctx.read_ctx = NULL; + } + + if (maat_instance->mr_ctx.write_ctx != NULL) { + redisFree(maat_instance->mr_ctx.write_ctx); + maat_instance->mr_ctx.write_ctx = NULL; + } + } + + FREE(maat_instance); return NULL; } \ No newline at end of file diff --git a/src/maat_table_runtime.cpp b/src/maat_table_runtime.cpp index a9eaa98..c935c86 100644 --- a/src/maat_table_runtime.cpp +++ b/src/maat_table_runtime.cpp @@ -36,7 +36,7 @@ struct plugin_user_ctx { }; struct expr_runtime { - enum scan_mode scan_mode; + enum hs_scan_mode scan_mode; struct adapter_hs *hs; struct adapter_hs_stream *hs_stream; struct rcu_hash_table *htable; diff --git a/src/maat_table_schema.cpp b/src/maat_table_schema.cpp index 48d44eb..32b5345 100644 --- a/src/maat_table_schema.cpp +++ b/src/maat_table_schema.cpp @@ -33,7 +33,7 @@ struct expr_table_schema { int match_method_column; int is_hexbin_column; int is_valid_column; /* valid means add, invalid means delete */ - enum scan_mode scan_mode; /* adapter_hs scan mode */ + enum hs_scan_mode scan_mode; /* adapter_hs scan mode */ }; #define MAX_PLUGIN_PER_TABLE 32 @@ -60,6 +60,10 @@ struct ip_plugin_table_schema { struct ex_data_schema ex_schema; }; +struct virtual_table_schema { + int physical_table_id[SCAN_TYPE_MAX]; +}; + struct table_schema { int table_id; char table_name[NAME_MAX]; @@ -68,6 +72,7 @@ struct table_schema { struct expr_table_schema expr; struct plugin_table_schema plugin; struct ip_plugin_table_schema ip_plugin; + struct virtual_table_schema virtual_table; }; }; @@ -388,6 +393,10 @@ struct table_schema_manager *table_schema_manager_create(const char *table_info_ unsigned char *json_buff = NULL; size_t json_buff_sz = 0; + if (NULL == table_info_path) { + return NULL; + } + int ret = load_file_to_memory(table_info_path, &json_buff, &json_buff_sz); if (ret < 0) { fprintf(stderr, "Maat read table info %s error.\n", table_info_path); @@ -417,8 +426,8 @@ struct table_schema_manager *table_schema_manager_create(const char *table_info_ maat_kv_register(reserved_word_map, "plugin", TABLE_TYPE_PLUGIN); maat_kv_register(reserved_word_map, "ip_plugin", TABLE_TYPE_IP_PLUGIN); - maat_kv_register(reserved_word_map, "block", SCAN_MODE_BLOCK); - maat_kv_register(reserved_word_map, "stream", SCAN_MODE_STREAM); + maat_kv_register(reserved_word_map, "block", HS_SCAN_MODE_BLOCK); + maat_kv_register(reserved_word_map, "stream", HS_SCAN_MODE_STREAM); struct table_schema_manager *table_schema_mgr = ALLOC(struct table_schema_manager, 1); struct table_schema **pptable = table_schema_mgr->schema_table; @@ -471,6 +480,10 @@ struct table_schema_manager *table_schema_manager_create(const char *table_info_ void table_schema_manager_destroy(struct table_schema_manager *table_schema_mgr) { + if (NULL == table_schema_mgr) { + return; + } + for (size_t i = 0; i < MAX_TABLE_NUM; i++) { if (NULL == table_schema_mgr->schema_table[i]) { continue; @@ -480,11 +493,15 @@ void table_schema_manager_destroy(struct table_schema_manager *table_schema_mgr) } maat_kv_store_free(table_schema_mgr->tablename2id_map); - free(table_schema_mgr); + FREE(table_schema_mgr); } int table_schema_manager_get_table_id(struct table_schema_manager* table_schema_mgr, const char *table_name) { + if (NULL == table_schema_mgr || NULL == table_name) { + return -1; + } + int table_id = -1; int ret = maat_kv_read(table_schema_mgr->tablename2id_map, table_name, &table_id); @@ -497,6 +514,10 @@ int table_schema_manager_get_table_id(struct table_schema_manager* table_schema_ enum table_type table_schema_manager_get_table_type(struct table_schema_manager *table_schema_mgr, int id) { + if (NULL == table_schema_mgr) { + return TABLE_TYPE_MAX; + } + if (table_schema_mgr->schema_table[id] == NULL) { return TABLE_TYPE_MAX; } @@ -511,6 +532,10 @@ size_t table_schema_manager_get_size(struct table_schema_manager* table_schema_m void table_schema_manager_all_plugin_cb_start(struct table_schema_manager *table_schema_mgr, int update_type) { + if (NULL == table_schema_mgr) { + return; + } + struct table_schema *ptable = NULL; struct plugin_table_schema *plugin_schema = NULL; @@ -536,6 +561,10 @@ void table_schema_manager_all_plugin_cb_start(struct table_schema_manager *table void table_schema_manager_all_plugin_cb_finish(struct table_schema_manager* table_schema_mgr) { + if (NULL == table_schema_mgr) { + return; + } + struct table_schema *ptable = NULL; struct plugin_table_schema *plugin_schema = NULL; @@ -606,19 +635,149 @@ enum match_method int_to_match_method_type(int match_method_type) struct table_schema *table_schema_get(struct table_schema_manager *table_schema_mgr, int table_id) { + if ((NULL == table_schema_mgr) || (table_id < 0)) { + return NULL; + } + return table_schema_mgr->schema_table[table_id]; } +struct table_schema *table_schema_get_by_scan_type(struct table_schema_manager *table_schema_mgr, + int table_id, enum scan_type scan_type, int *virtual_table_id) +{ + enum scan_type table_scan_type; + struct table_schema **pptable = table_schema_mgr->schema_table; + size_t n_table = MAX_TABLE_NUM; + + if ((unsigned int)table_id > n_table) { + return NULL; + } + + if (NULL == pptable[table_id]) { + return NULL; + } + + struct table_schema *ptable = pptable[table_id]; + if (NULL == ptable) { + return NULL; + } + + struct table_schema *p_physical_table = NULL; + if (ptable->table_type == TABLE_TYPE_VIRTUAL) { + p_physical_table = pptable[ptable->virtual_table.physical_table_id[scan_type]]; + *virtual_table_id = table_id; + } else { + p_physical_table = ptable; + if(virtual_table_id) { + *virtual_table_id = 0; + } + } + + table_scan_type = table_schema_get_scan_type(p_physical_table); + if (table_scan_type != scan_type) { + return NULL; + } + + return p_physical_table; +} + enum table_type table_schema_get_table_type(struct table_schema *table_schema) { + if (NULL == table_schema) { + return TABLE_TYPE_MAX; + } + return table_schema->table_type; } int table_schema_get_table_id(struct table_schema *table_schema) { + if (NULL == table_schema) { + return -1; + } + return table_schema->table_id; } +enum scan_type table_schema_get_scan_type(struct table_schema *table_schema) +{ + enum scan_type ret = SCAN_TYPE_INVALID; + + if (NULL == table_schema) { + return ret; + } + + switch (table_schema->table_type) { + case TABLE_TYPE_EXPR: + case TABLE_TYPE_EXPR_PLUS: + case TABLE_TYPE_SIMILARITY: + case TABLE_TYPE_DIGEST: + ret = SCAN_TYPE_STRING; + break; + case TABLE_TYPE_INTERVAL: + case TABLE_TYPE_INTERVAL_PLUS: + ret = SCAN_TYPE_INTERVAL; + break; + case TABLE_TYPE_IP: + case TABLE_TYPE_IP_PLUS: + case TABLE_TYPE_COMPOSITION: + ret = SCAN_TYPE_IP; + break; + case TABLE_TYPE_PLUGIN: + ret = SCAN_TYPE_PLUGIN; + break; + case TABLE_TYPE_IP_PLUGIN: + ret = SCAN_TYPE_IP; + break; + case TABLE_TYPE_FQDN_PLUGIN: + ret = SCAN_TYPE_FQDN_PLUGIN; + break; + case TABLE_TYPE_BOOL_PLUGIN: + ret = SCAN_TYPE_BOOL_PLUGIN; + break; + case TABLE_TYPE_COMPILE: + ret = SCAN_TYPE_NONE; + break; + default: + break; + } + + return ret; +} + +int table_schema_get_valid_flag_column(struct table_schema *table_schema) +{ + int valid_flag_column = -1; + + if (NULL == table_schema) { + return valid_flag_column; + } + + switch (table_schema->table_type) { + case TABLE_TYPE_EXPR: + valid_flag_column = table_schema->expr.is_valid_column; + break; + case TABLE_TYPE_PLUGIN: + valid_flag_column = table_schema->plugin.is_valid_column; + break; + case TABLE_TYPE_IP_PLUGIN: + valid_flag_column = table_schema->ip_plugin.is_valid_column; + break; + /* + case TABLE_TYPE_FQDN_PLUGIN: + valid_flag_column = table_schema->fqdn_plugin.valid_flag_column; + break; + case TABLE_TYPE_BOOL_PLUGIN: + valid_flag_column = table_schema->bool_plugin.valid_flag_column; + break;*/ + default: + valid_flag_column = -1; + break; + } + + return valid_flag_column; +} + int populate_expr_table_item(const char *line, struct expr_table_schema *expr_schema, struct expr_item *expr_item) { size_t column_offset = 0; @@ -778,6 +937,10 @@ int populate_ip_plugin_table_item(const char *line, struct ip_plugin_table_schem struct table_item * table_schema_line_to_item(const char *line, struct table_schema *table_schema) { + if (NULL == line || NULL == table_schema) { + return NULL; + } + int ret = -1; struct table_item *table_item = ALLOC(struct table_item, 1); @@ -812,10 +975,14 @@ error: return NULL; } -enum scan_mode expr_table_schema_get_scan_mode(struct table_schema *table_schema) +enum hs_scan_mode expr_table_schema_get_scan_mode(struct table_schema *table_schema) { + if (NULL == table_schema) { + return HS_SCAN_MODE_MAX; + } + if (table_schema->table_type != TABLE_TYPE_EXPR) { - return SCAN_MODE_MAX; + return HS_SCAN_MODE_MAX; } return table_schema->expr.scan_mode; @@ -827,7 +994,7 @@ int plugin_table_schema_set_ex_data_schema(struct table_schema *table_schema, maat_plugin_ex_dup_func_t *dup_func, long argl, void *argp) { - if (NULL == new_func || NULL == free_func || NULL == dup_func) { + if (NULL == table_schema || NULL == new_func || NULL == free_func || NULL == dup_func) { assert(0); fprintf(stderr, "%s failed: invalid parameter", __FUNCTION__); return -1; @@ -857,6 +1024,10 @@ int plugin_table_schema_set_ex_data_schema(struct table_schema *table_schema, struct ex_data_schema *plugin_table_schema_get_ex_data_schema(struct table_schema *table_schema) { + if (NULL == table_schema) { + return NULL; + } + struct ex_data_schema *ex_schema = NULL; switch (table_schema->table_type) { @@ -875,6 +1046,10 @@ struct ex_data_schema *plugin_table_schema_get_ex_data_schema(struct table_schem int plugin_table_schema_ex_data_schema_flag(struct table_schema *table_schema) { + if (NULL == table_schema) { + return -1; + } + struct ex_data_schema *ex_schema = NULL; switch (table_schema->table_type) { @@ -897,6 +1072,10 @@ int plugin_table_schema_add_callback(struct table_schema_manager* table_schema_m maat_finish_callback_t *finish, void *u_para) { + if ((NULL == table_schema_mgr) || (table_id < 0)) { + return -1; + } + struct table_schema *ptable = table_schema_get(table_schema_mgr, table_id); if (NULL == ptable) { fprintf(stderr, "table_id:%d unregistered, can't add callback func", table_id); @@ -945,6 +1124,10 @@ size_t plugin_table_schema_callback_count(struct table_schema *table_schema) void plugin_table_schema_all_cb_update(struct table_schema* table_schema, const char *row) { + if (NULL == table_schema || NULL == row) { + return; + } + struct plugin_table_schema *plugin_schema = NULL; switch (table_schema->table_type) { @@ -963,4 +1146,22 @@ void plugin_table_schema_all_cb_update(struct table_schema* table_schema, const default: break; } +} + +int plugin_table_schema_get_foreign_column(struct table_schema *table_schema, int *foreign_columns) +{ + if (NULL == table_schema) { + return -1; + } + + if (table_schema->table_type != TABLE_TYPE_PLUGIN) { + return 0; + } + + int n_foreign = table_schema->plugin.n_foreign; + for (int i = 0; i < n_foreign; i++) { + foreign_columns[i] = table_schema->plugin.foreign_columns[i]; + } + + return n_foreign; } \ No newline at end of file diff --git a/src/maat_utils.cpp b/src/maat_utils.cpp index 444704f..967e621 100644 --- a/src/maat_utils.cpp +++ b/src/maat_utils.cpp @@ -11,6 +11,9 @@ #include #include #include +#include +#include +#include #include "maat_utils.h" @@ -19,6 +22,7 @@ char *maat_strdup(const char *s) if (NULL == s) { return NULL; } + char *d = (char *)malloc(strlen(s) + 1); memcpy(d, s, strlen(s) + 1); @@ -27,10 +31,14 @@ char *maat_strdup(const char *s) int get_column_pos(const char *line, int column_seq, size_t *offset, size_t *len) { - const char *seps=" \t"; - char *saveptr=NULL, *subtoken=NULL, *str=NULL; + int i = 0; + int ret = -1; + char *str = NULL; + char *saveptr = NULL; + char *subtoken = NULL; + const char *seps = " \t"; char *dup_line = maat_strdup(line); - int i = 0, ret = -1; + for (str = dup_line; ; str = NULL) { subtoken = strtok_r(str, seps, &saveptr); if (subtoken == NULL) @@ -53,7 +61,7 @@ int load_file_to_memory(const char *file_name, unsigned char **pp_out, size_t *o int ret = 0; FILE *fp = NULL; struct stat fstat_buf; - size_t read_size=0; + size_t read_size = 0; ret = stat(file_name, &fstat_buf); if (ret != 0) { @@ -186,4 +194,181 @@ int system_cmd_mkdir(const char *path) char cmd[MAX_SYSTEM_CMD_LEN] = {0}; snprintf(cmd, sizeof(cmd), "mkdir -p %s", path); return system(cmd); -} \ No newline at end of file +} + +char *md5_file(const char *filename, char *md5string) +{ + unsigned char md5[MD5_DIGEST_LENGTH] = {0}; + struct stat file_info; + stat(filename, &file_info); + size_t file_size = file_info.st_size; + + FILE *fp = fopen(filename,"r"); + if (NULL == fp) { + return NULL; + } + + char *file_buff = (char *)malloc(file_size); + fread(file_buff, 1, file_size, fp); + fclose(fp); + + MD5((const unsigned char *)(file_buff), (unsigned long)(file_size), md5); + for (int i = 0; i < MD5_DIGEST_LENGTH; ++i) { + sprintf(&md5string[i*2], "%02x", (unsigned int)md5[i]); + } + + free(file_buff); + return md5string; +} + +int crypt_memory(const unsigned char *inbuf, size_t inlen, unsigned char **pp_out, size_t *out_sz, + const char *key, const char *algorithm, int do_encrypt, + char *err_str, size_t err_str_sz) +{ + OpenSSL_add_all_algorithms(); + const EVP_CIPHER *cipher = EVP_get_cipherbyname(algorithm); + if (NULL == cipher) { + snprintf(err_str, err_str_sz, "Cipher %s is not supported.", algorithm); + return 0; + } + + const EVP_MD *dgst = EVP_get_digestbyname("md5"); + if (NULL == dgst) { + snprintf(err_str, err_str_sz, "Get MD5 object failed."); + return 0; + } + + const unsigned char *salt = NULL; + unsigned char cipher_key[EVP_MAX_KEY_LENGTH]; + unsigned char cipher_iv[EVP_MAX_IV_LENGTH]; + + memset(cipher_key,0,sizeof(cipher_key)); + memset(cipher_iv,0,sizeof(cipher_iv)); + + int ret = EVP_BytesToKey(cipher, dgst, salt, (unsigned char *)key, + strlen((const char *)key), 1, cipher_key, cipher_iv); + if(0 == ret) { + snprintf(err_str, err_str_sz, "Key and IV generatioin failed."); + return 0; + } + + /* Don't set key or IV right away; we want to check lengths */ + EVP_CIPHER_CTX *ctx = EVP_CIPHER_CTX_new(); + EVP_CipherInit_ex(ctx, cipher, NULL, NULL, NULL, do_encrypt); + OPENSSL_assert(EVP_CIPHER_CTX_key_length(ctx) % 16 == 0); + OPENSSL_assert(EVP_CIPHER_CTX_iv_length(ctx) == 16); + + /* Now we can set key and IV */ + //It should be set to 1 for encryption, 0 for decryption and -1 to leave the value unchanged (the actual value of 'enc' being supplied in a previous call). + EVP_CipherInit_ex(ctx, NULL, NULL, cipher_key, cipher_iv, -1); + int out_blk_len = 0; + int out_buff_offset = 0; + int out_buff_len = inlen + EVP_CIPHER_block_size(cipher) - 1; + *pp_out = (unsigned char *)malloc(out_buff_len * sizeof(unsigned char)); + if (!EVP_CipherUpdate(ctx, *pp_out + out_buff_offset, &out_blk_len, inbuf, inlen)) { + snprintf(err_str, err_str_sz, "EVP_CipherUpdate failed."); + EVP_CIPHER_CTX_free(ctx); + goto error_out; + } + + out_buff_offset += out_blk_len; + if (!EVP_CipherFinal_ex(ctx, *pp_out+out_buff_offset, &out_blk_len)) { + snprintf(err_str, err_str_sz, "EVP_CipherFinal_ex failed. Maybe password is wrong?"); + EVP_CIPHER_CTX_free(ctx); + goto error_out; + } + + out_buff_offset += out_blk_len; + EVP_CIPHER_CTX_free(ctx); + *out_sz = out_buff_offset; + return 0; + +error_out: + free(*pp_out); + *pp_out = NULL; + return -1; +} + +int decrypt_open(const char* file_name, const char* key, const char* algorithm, + unsigned char**pp_out, size_t *out_sz, char* err_str, size_t err_str_sz) +{ + size_t file_sz = 0; + unsigned char *file_buff = NULL; + int ret = load_file_to_memory(file_name, &file_buff, &file_sz); + if (ret < 0) { + return -1; + } + + ret = crypt_memory(file_buff, file_sz, pp_out, out_sz, key, algorithm, 0, err_str, err_str_sz); + free(file_buff); + file_buff = NULL; + + return ret; +} + +int gzip_uncompress_one_try(const unsigned char *in_compressed_data, size_t in_compressed_sz, + unsigned char **out_uncompressed_data, size_t *out_uncompressed_sz) +{ + z_stream strm; + strm.zalloc = NULL; + strm.zfree = NULL; + strm.opaque = NULL; + + strm.avail_in = in_compressed_sz; + strm.avail_out = *out_uncompressed_sz; + strm.next_in = (Bytef *) in_compressed_data; + strm.next_out = *out_uncompressed_data; + + int ret = -1; + ret = inflateInit2(&strm, MAX_WBITS+16); + if (ret == Z_OK) { + ret = inflate(&strm, Z_FINISH); + if (ret == Z_STREAM_END) { + *out_uncompressed_sz = strm.total_out; + ret = inflateEnd(&strm); + return ret; + } + } + inflateEnd(&strm); + return ret; +} + +int gzip_uncompress(const unsigned char *in_compressed_data, size_t in_compressed_sz, + unsigned char **out_uncompressed_data, size_t *out_uncompressed_sz) +{ + int z_result; + int ret = -1; + size_t buffer_sz = in_compressed_sz * 2; + *out_uncompressed_data = (unsigned char *)malloc(buffer_sz); + + do { + *out_uncompressed_sz=buffer_sz; + z_result = gzip_uncompress_one_try(in_compressed_data, in_compressed_sz, + out_uncompressed_data, out_uncompressed_sz); + switch (z_result) { + case Z_OK: + ret = 0; + break; + case Z_BUF_ERROR: + buffer_sz *= 2; + *out_uncompressed_data = (unsigned char *)realloc(*out_uncompressed_data, buffer_sz); + break; + default: + ret = -1; + break; + } + } while (z_result == Z_BUF_ERROR); + + return ret; +} + +size_t memcat(void **dest, size_t offset, size_t *n_dest, const void *src, size_t n_src) +{ + if (*n_dest < offset + n_src) { + *n_dest = (offset + n_src) * 2; + *dest = realloc(*dest, sizeof(char) * (*n_dest)); + } + memcpy((char *) * dest + offset, src, n_src); + + return n_src; +} diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 835b206..b34a608 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -13,4 +13,5 @@ target_link_libraries(maat_framework_gtest maat_frame_static gtest_static) file(COPY rule DESTINATION ./) file(COPY table_info.conf DESTINATION ./) -file(COPY and_expr.conf DESTINATION ./) \ No newline at end of file +file(COPY and_expr.conf DESTINATION ./) +file(COPY maat_json.json DESTINATION ./) \ No newline at end of file diff --git a/test/maat_framework_gtest.cpp b/test/maat_framework_gtest.cpp index 714f1dc..b9750b5 100644 --- a/test/maat_framework_gtest.cpp +++ b/test/maat_framework_gtest.cpp @@ -5,11 +5,14 @@ #include "maat_utils.h" #include "maat_table_schema.h" #include "maat_table_runtime.h" +#include "maat_command.h" struct maat *g_maat_instance = NULL; const char *table_info_path = "./table_info.conf"; const char *rule_full_path = "./rule/full/index"; const char *rule_inc_path = "./rule/inc/index"; +const char *json_path="./maat_json.json"; +const char *iris_file = "./HTTP_URL.000001"; TEST(maat_scan_string, hit_one_expr) { struct table_schema_manager *table_schema_mgr = g_maat_instance->table_schema_mgr; @@ -31,20 +34,22 @@ TEST(maat_scan_string, hit_one_expr) { EXPECT_EQ(n_result_array, 1); EXPECT_EQ(result_array[0], 101); +/* memset(result_array, 0, sizeof(result_array)); char data3[64] = "maat"; ret = maat_scan_string(g_maat_instance, table_id, 0, data3, strlen(data3), result_array, &n_result_array, NULL); EXPECT_EQ(ret, 0); EXPECT_EQ(n_result_array, 1); - EXPECT_EQ(result_array[0], 102); + EXPECT_EQ(result_array[0], 102); */ +/* memset(result_array, 0, sizeof(result_array)); char data4[64] = "world"; ret = maat_scan_string(g_maat_instance, table_id, 0, data4, strlen(data4), result_array, &n_result_array, NULL); EXPECT_EQ(n_result_array, 1); - EXPECT_EQ(result_array[0], 103); + EXPECT_EQ(result_array[0], 103); */ } - +#if 0 TEST(maat_scan_string, hit_two_expr) { struct table_schema_manager *table_schema_mgr = g_maat_instance->table_schema_mgr; int table_id = table_schema_manager_get_table_id(table_schema_mgr, "HTTP_URL"); @@ -159,14 +164,82 @@ TEST(maat_scan_string, config_dynamic_update) { EXPECT_EQ(result_array[4], 101); } + +TEST(maat_scan_string, hit_one_expr) { + struct table_schema_manager *table_schema_mgr = g_maat_instance->table_schema_mgr; + int table_id = table_schema_manager_get_table_id(table_schema_mgr, "HTTP_URL"); + + char data[128] = "i.ytimg.com"; + int result_array[5] = {0}; + size_t n_result_array = 0; + int ret = maat_scan_string(g_maat_instance, table_id, 0, data, strlen(data), result_array, &n_result_array, NULL); + EXPECT_EQ(ret, 0); + EXPECT_EQ(n_result_array, 1); + EXPECT_EQ(result_array[0], 30); +} + +TEST(maat_scan_string, hit_two_expr) { + struct table_schema_manager *table_schema_mgr = g_maat_instance->table_schema_mgr; + int table_id = table_schema_manager_get_table_id(table_schema_mgr, "HTTP_URL"); + + char data[128] = "should hit aaa bbb"; + int result_array[5] = {0}; + size_t n_result_array = 0; + int ret = maat_scan_string(g_maat_instance, table_id, 0, data, strlen(data), result_array, &n_result_array, NULL); + EXPECT_EQ(ret, 0); + EXPECT_EQ(n_result_array, 2); + EXPECT_EQ(result_array[0], 28); + EXPECT_EQ(result_array[1], 27); +} + +TEST(maat_scan_string, hit_three_expr) { + struct table_schema_manager *table_schema_mgr = g_maat_instance->table_schema_mgr; + int table_id = table_schema_manager_get_table_id(table_schema_mgr, "HTTP_URL"); + + char data[128] = "should hit aaa bbb C#涓浗"; + int result_array[5] = {0}; + size_t n_result_array = 0; + int ret = maat_scan_string(g_maat_instance, table_id, 0, data, strlen(data), result_array, &n_result_array, NULL); + EXPECT_EQ(ret, 0); + EXPECT_EQ(n_result_array, 3); + EXPECT_EQ(result_array[0], 28); + EXPECT_EQ(result_array[1], 27); + EXPECT_EQ(result_array[2], 18); +} +#endif +TEST(maat_redis, write) { + char line[1024] = {0}; + char table_name[64] = "HTTP_URL"; + struct maat_cmd_line rule_line; + rule_line.table_name = (char *)malloc(sizeof(char) * 64); + + memcpy((void *)rule_line.table_name, table_name, strlen(table_name)); + FILE *fp = fopen(iris_file, "r"); + EXPECT_NE(fp, nullptr); + + int rule_num = 0; + fscanf(fp, "%d\n", &rule_num); + + while (NULL != fgets(line, sizeof(line), fp)) { + rule_line.table_line = (char *)malloc(sizeof(line)); + memcpy((void *)rule_line.table_line, line, strlen(line)); + maat_cmd_set_line(g_maat_instance, &rule_line); + break; + } + +} + int main(int argc, char ** argv) { int ret=0; ::testing::InitGoogleTest(&argc, argv); struct maat_options *opts = maat_options_new(); - maat_options_set_iris_full_dir(opts, rule_full_path); - maat_options_set_iris_inc_dir(opts, rule_inc_path); + //maat_options_set_iris_full_dir(opts, rule_full_path); + //maat_options_set_iris_inc_dir(opts, rule_inc_path); + maat_options_set_redis_ip(opts, "127.0.0.1"); + maat_options_set_redis_port(opts, 6379); + //maat_options_set_json_file(opts, json_path); g_maat_instance = maat_new(opts, table_info_path); EXPECT_NE(g_maat_instance, nullptr); diff --git a/test/maat_json.json b/test/maat_json.json new file mode 100644 index 0000000..42cf591 --- /dev/null +++ b/test/maat_json.json @@ -0,0 +1,2339 @@ +{ + "compile_table": "COMPILE", + "group2compile_table": "GROUP2COMPILE", + "group2group_table": "GROUP2GROUP", + "groups": [ + { + "group_name": "ASN1234", + "regions": [ + { + "table_name": "AS_NUMBER", + "table_type": "expr", + "table_content": { + "keywords": "AS1234", + "expr_type": "none", + "match_method": "exact", + "format": "uncase plain" + } + } + ] + }, + { + "group_name": "ASN2345", + "regions": [ + { + "table_name": "AS_NUMBER", + "table_type": "expr", + "table_content": { + "keywords": "AS2345", + "expr_type": "none", + "match_method": "exact", + "format": "uncase plain" + } + } + ] + }, + { + "group_name": "financial-department-ip", + "regions": [ + { + "table_name": "IP_CONFIG", + "table_type": "ip", + "table_content": { + "addr_type": "ipv4", + "src_ip": "192.168.40.88", + "mask_src_ip": "255.255.255.255", + "src_port": "0", + "mask_src_port": "65535", + "dst_ip": "0.0.0.0", + "mask_dst_ip": "255.255.255.255", + "dst_port": "0", + "mask_dst_port": "65535", + "protocol": 6, + "direction": "double" + } + } + ] + }, + { + "group_name": "Country-Sparta-IP", + "regions": [ + { + "table_name": "GeoLocation", + "table_type": "expr", + "table_content": { + "keywords": "Greece.Sparta", + "expr_type": "none", + "match_method": "exact", + "format": "uncase plain" + } + } + ] + }, + { + "group_name": "IPv4-composition-source-only", + "regions": [ + { + "table_type": "ip_plus", + "table_name": "IP_PLUS_CONFIG", + "table_content": { + "addr_type": "ipv4", + "saddr_format": "range", + "src_ip1": "192.168.50.24", + "src_ip2": "192.168.50.24", + "sport_format": "range", + "src_port1": "1", + "src_port2": "40000", + "daddr_format": "mask", + "dst_ip1": "0.0.0.0", + "dst_ip2": "255.255.255.0", + "dport_format": "range", + "dst_port1": "0", + "dst_port2": "65535", + "protocol": 6, + "direction": "double" + } + } + ] + }, + { + "group_name": "FQDN_OBJ1", + "regions": [ + { + "table_name": "KEYWORDS_TABLE", + "table_type": "expr", + "table_content": { + "keywords": "sports.example.com", + "expr_type": "none", + "match_method": "exact", + "format": "uncase plain" + } + } + ] + }, + { + "group_name": "FQDN_CAT1", + "regions": [ + { + "table_name": "INTERGER_PLUS", + "table_type": "interval_plus", + "table_content": { + "district": "fqdn_cat_id", + "low_boundary": 1724, + "up_boundary": 1724 + } + } + ] + }, + { + "group_name": "IPv4-composition-NOT-client-ip", + "regions": [ + { + "table_type": "ip_plus", + "table_name": "IP_PLUS_CONFIG", + "table_content": { + "addr_type": "ipv4", + "saddr_format": "range", + "src_ip1": "192.168.58.19", + "src_ip2": "192.168.58.19", + "sport_format": "range", + "src_port1": "20000", + "src_port2": "20000", + "daddr_format": "mask", + "dst_ip1": "0.0.0.0", + "dst_ip2": "255.255.255.0", + "dport_format": "range", + "dst_port1": "0", + "dst_port2": "65535", + "protocol": 6, + "direction": "double" + } + } + ] + }, + { + "group_name": "IPv4-composition-NOT-server-ip", + "regions": [ + { + "table_type": "ip_plus", + "table_name": "IP_PLUS_CONFIG", + "table_content": { + "addr_type": "ipv4", + "saddr_format": "range", + "src_ip1": "10.0.1.20", + "src_ip2": "10.0.1.25", + "sport_format": "range", + "src_port1": "1", + "src_port2": "443", + "daddr_format": "mask", + "dst_ip1": "0.0.0.0", + "dst_ip2": "255.255.255.0", + "dport_format": "range", + "dst_port1": "0", + "dst_port2": "65535", + "protocol": 6, + "direction": "double" + } + } + ] + } + ], + "rules": [ + { + "compile_id": 123, + "service": 1, + "action": 1, + "do_blacklist": 1, + "do_log": 1, + "user_region": "escaped\\bdata:have\\ba\\bspace\\band\\ba\\b\\&\\bsymbol.", + "is_valid": "yes", + "groups": [ + { + "group_name": "123_IP_group", + "regions": [ + { + "table_name": "IP_CONFIG", + "table_type": "ip", + "table_content": { + "addr_type": "ipv4", + "src_ip": "10.0.6.201", + "mask_src_ip": "255.255.0.0", + "src_port": "0", + "mask_src_port": "65535", + "dst_ip": "0.0.0.0", + "mask_dst_ip": "255.255.255.255", + "dst_port": "0", + "mask_dst_port": "65535", + "protocol": 6, + "direction": "double" + } + }, + { + "table_name": "IP_CONFIG", + "table_type": "ip", + "table_content": { + "addr_type": "ipv6", + "src_ip": "2001:da8:205:1::101", + "mask_src_ip": "ffff:ffff:ffff:ffff:ffff:ffff:ffff:0000", + "src_port": "0", + "mask_src_port": "65535", + "dst_ip": "0::0", + "mask_dst_ip": "ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff", + "dst_port": "0", + "mask_dst_port": "65535", + "protocol": 6, + "direction": "double" + } + } + ] + }, + { + "group_name": "Untitled", + "regions": [ + { + "table_name": "HTTP_URL", + "table_type": "expr", + "table_content": { + "keywords": "abckkk&123", + "expr_type": "and", + "match_method": "sub", + "format": "uncase plain" + } + } + ] + } + ] + }, + { + "compile_id": 124, + "service": 1, + "action": 1, + "do_blacklist": 1, + "do_log": 1, + "user_region": "anything", + "is_valid": "yes", + "groups": [ + { + "group_name": "123_IP_group" + }, + { + "group_name": "Untitled", + "regions": [ + { + "table_name": "CONTENT_SIZE", + "table_type": "interval", + "table_content": { + "low_boundary": 100, + "up_boundary": 500 + } + } + ] + } + ] + }, + { + "compile_id": 125, + "service": 1, + "action": 1, + "do_blacklist": 1, + "do_log": 1, + "user_region": "anything", + "is_valid": "yes", + "groups": [ + { + "regions": [ + { + "table_name": "HTTP_URL", + "table_type": "expr", + "table_content": { + "keywords": "action=search\\&query=(.*)", + "expr_type": "regex", + "match_method": "sub", + "format": "uncase plain" + } + } + ] + } + ] + }, + { + "compile_id": 126, + "service": 1, + "action": 1, + "do_blacklist": 1, + "do_log": 1, + "user_region": "anything", + "is_valid": "yes", + "groups": [ + { + "group_name": "Untitled", + "regions": [ + { + "table_name": "HTTP_URL", + "table_type": "expr", + "table_content": { + "keywords": "should_not_hit_any_rule", + "expr_type": "none", + "match_method": "sub", + "format": "uncase plain" + } + } + ] + }, + { + "group_name": "126_interval_group", + "regions": [ + { + "table_name": "CONTENT_SIZE", + "table_type": "interval", + "table_content": { + "low_boundary": 2014, + "up_boundary": 2016 + } + } + ] + } + ] + }, + { + "compile_id": 127, + "service": 1, + "action": 1, + "do_blacklist": 1, + "do_log": 1, + "user_region": "anything", + "is_valid": "yes", + "groups": [ + { + "group_name": "Untitled", + "regions": [ + { + "table_name": "FILE_DIGEST", + "table_type": "digest", + "table_content": { + "raw_len": 1160164, + "digest": "3072:Xk/maCm4yLYtRIFDFnVfHH+CAQI6VD5mekDmaa/4qCuFnqak1s3/+Gn1IJHa/AvybUsbGWcIAy9grTp2s5bbj/TaKxONfb[0:1160163]#12288:UChtbFS6pypdTy4m2[0:1160163]", + "cfds_level": 70 + } + } + ] + } + ] + }, + { + "compile_id": 128, + "service": 1, + "action": 1, + "do_blacklist": 1, + "do_log": 1, + "user_region": "StringScan.ExprPlus", + "is_valid": "yes", + "groups": [ + { + "group_name": "Untitled", + "regions": [ + { + "table_name": "HTTP_SIGNATURE", + "table_type": "expr_plus", + "table_content": { + "district": "HtTP\\bUrL", + "keywords": "abckkk&123", + "expr_type": "and", + "match_method": "sub", + "format": "uncase plain" + } + } + ] + } + ] + }, + { + "compile_id": 129, + "service": 1, + "action": 1, + "do_blacklist": 1, + "do_log": 1, + "user_region": "utf8_涓枃", + "is_valid": "yes", + "groups": [ + { + "group_name": "Untitled", + "regions": [ + { + "table_name": "HTTP_URL", + "table_type": "expr", + "table_content": { + "keywords": "C#涓浗", + "expr_type": "none", + "match_method": "sub", + "format": "uncase plain" + } + } + ] + } + ] + }, + { + "compile_id": 130, + "service": 1, + "action": 1, + "do_blacklist": 1, + "do_log": 1, + "user_region": "utf8_缁磋", + "is_valid": "yes", + "groups": [ + { + "group_name": "Untitled", + "regions": [ + { + "table_name": "KEYWORDS_TABLE", + "table_type": "expr", + "table_content": { + "keywords": "2010&賷賶賱賶丿賶賰賶", + "expr_type": "and", + "match_method": "sub", + "format": "uncase plain" + } + } + ] + } + ] + }, + { + "compile_id": 131, + "service": 1, + "action": 1, + "do_blacklist": 1, + "do_log": 1, + "user_region": "utf8_缁磋2", + "is_valid": "yes", + "groups": [ + { + "group_name": "Untitled", + "regions": [ + { + "table_name": "KEYWORDS_TABLE", + "table_type": "expr", + "table_content": { + "keywords": "爻賶賷丕爻賶賷", + "expr_type": "and", + "match_method": "sub", + "format": "uncase plain" + } + } + ] + } + ] + }, + { + "compile_id": 132, + "service": 1, + "action": 1, + "do_blacklist": 1, + "do_log": 1, + "user_region": "string\\bunescape", + "is_valid": "yes", + "groups": [ + { + "group_name": "TakeMeHome", + "regions": [ + { + "table_name": "KEYWORDS_TABLE", + "table_type": "expr", + "table_content": { + "keywords": "Take\\bme\\bHome&Batman\\", + "expr_type": "and", + "match_method": "sub", + "format": "uncase plain" + } + } + ] + } + ] + }, + { + "compile_id": 133, + "service": 1, + "action": 1, + "do_blacklist": 1, + "do_log": 1, + "user_region": "13018_table_conjunction_test_part1\bnow_its_very_very_long0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz0123456789abcdefghijklmnopkrstuvwxyz", + "is_valid": "yes", + "groups": [ + { + "group_name": "Untitled", + "regions": [ + { + "table_name": "HTTP_HOST", + "table_type": "expr", + "table_content": { + "keywords": "www.3300av.com", + "expr_type": "none", + "match_method": "sub", + "format": "uncase plain" + } + } + ] + } + ] + }, + { + "compile_id": 134, + "service": 1, + "action": 1, + "do_blacklist": 1, + "do_log": 1, + "user_region": "table_conjunction_test_part2", + "is_valid": "yes", + "groups": [ + { + "group_name": "Untitled", + "regions": [ + { + "table_name": "HTTP_URL", + "table_type": "expr", + "table_content": { + "keywords": "novel&27122.txt", + "expr_type": "and", + "match_method": "sub", + "format": "uncase plain" + } + } + ] + } + ] + }, + { + "compile_id": 135, + "service": 1, + "action": 1, + "do_blacklist": 1, + "do_log": 1, + "user_region": "anything", + "is_valid": "yes", + "groups": [ + { + "group_name": "Untitled", + "regions": [ + { + "table_name": "SIM_URL", + "table_type": "similar", + "table_content": { + "target": "mwss.xiu.youku.com/live/hls/v1/0000000000000000000000001526a0a8/709.ts?&token=98765", + "threshold": 90 + } + } + ] + } + ] + }, + { + "compile_id": 136, + "service": 1, + "action": 1, + "do_blacklist": 1, + "do_log": 1, + "user_region": "offset_string", + "is_valid": "yes", + "groups": [ + { + "group_name": "Untitled", + "regions": [ + { + "table_name": "IMAGE_FP", + "table_type": "expr", + "table_content": { + "keywords": "4362-4458:323031333A30333A30372032333A35363A313000323031333A30333A30372032333A35363A3130000000FFE20C584943435F50524F46494C4500010100000C484C696E6F021000006D6E74725247422058595A2007CE00020009000600310000", + "expr_type": "offset", + "match_method": "none", + "format": "hexbin" + } + } + ] + } + ] + }, + { + "compile_id": 137, + "service": 1, + "action": 1, + "do_blacklist": 1, + "do_log": 1, + "user_region": "offset_string", + "is_valid": "yes", + "groups": [ + { + "group_name": "Untitled", + "regions": [ + { + "table_name": "IMAGE_FP", + "table_type": "expr", + "table_content": { + "keywords": "19339-19467:6CB2CB2F2028474C994991CCFC65CCA5E3B6FF001673985D157358610CACC674EE64CC27B5721CCDABD9CCA7C8E9F7BB1F54A930A6034D50F92711F5B2DACCB0715D2E6873CE5CE431DC701A194C260E9DB78CC89F2C84745869AB88349A3AE0412AB59D9ABA84EDEFFF0057FA4DA66D333698B5AD6F844DA2226D1CADAD5E44", + "expr_type": "offset", + "match_method": "none", + "format": "hexbin" + } + } + ] + } + ] + }, + { + "compile_id": 138, + "service": 1, + "action": 1, + "do_blacklist": 1, + "do_log": 1, + "effective_range": 0, + "tags":"{\"tag_sets\":[[{\"tag\":\"location\",\"value\":[\"鍖椾含/鏈濋槼/鍗庝弗鍖楅噷\",\"涓婃捣/娴︿笢/闄嗗鍢碶"]},{\"tag\":\"isp\",\"value\":[\"鐢典俊\",\"鑱旈歕"]}],[{\"tag\":\"location\",\"value\":[\"鍖椾含\"]},{\"tag\":\"isp\",\"value\":[\"鑱旈歕"]}]]}", + "user_region": "Not\\baccepted\\btags", + "is_valid": "yes", + "groups": [ + { + "regions": [ + { + "table_name": "HTTP_URL", + "table_type": "expr", + "table_content": { + "keywords": "should&hit&aaa", + "expr_type": "and", + "match_method": "sub", + "format": "uncase plain" + } + } + ] + } + ] + }, + { + "compile_id": 139, + "service": 1, + "action": 1, + "do_blacklist": 1, + "do_log": 1, + "effective_range": 0, + "tags":"{\"tag_sets\":[[{\"tag\":\"location\",\"value\":[\"鍖椾含/鏈濋槼/鍗庝弗鍖楅噷\"]},{\"tag\":\"isp\",\"value\":[\"鐢典俊\",\"绉诲姩\"]}]]}", + "user_region": "Accepted\\btags", + "is_valid": "yes", + "groups": [ + { + "regions": [ + { + "table_name": "HTTP_URL", + "table_type": "expr", + "table_content": { + "keywords": "should&hit&bbb", + "expr_type": "and", + "match_method": "sub", + "format": "uncase plain" + } + } + ] + } + ] + }, + { + "compile_id": 140, + "service": 1, + "action": 1, + "do_blacklist": 1, + "do_log": 1, + "user_region": "file_streams", + "is_valid": "yes", + "groups": [ + { + "group_name": "Untitled", + "regions": [ + { + "table_name": "KEYWORDS_TABLE", + "table_type": "expr", + "table_content": { + "keywords": "2018-10-05", + "expr_type": "none", + "match_method": "sub", + "format": "uncase plain" + } + } + ] + } + ] + }, + { + "compile_id": 141, + "service": 1, + "action": 1, + "do_blacklist": 1, + "do_log": 1, + "user_region": "Something:I\\bhave\\ba\\bname,7799", + "table_name":"COMPILE_ALIAS", + "is_valid": "yes", + "groups": [ + { + "group_name": "Untitled", + "regions": [ + { + "table_name": "HTTP_URL", + "table_type": "expr", + "table_content": { + "keywords": "i.ytimg.com", + "expr_type": "none", + "match_method": "sub", + "format": "uncase plain" + } + } + ] + } + ] + }, + { + "compile_id": 142, + "service": 1, + "action": 1, + "do_blacklist": 1, + "do_log": 1, + "user_region": "StringScan.UTF8EncodedURL", + "is_valid": "yes", + "groups": [ + { + "regions": [ + { + "table_name": "HTTP_URL", + "table_type": "expr", + "table_content": { + "keywords": ",IgpwcjA0LnN2bzAzKgkxMjcuMC4wLjE", + "expr_type": "none", + "match_method": "sub", + "format": "uncase plain" + } + } + ] + } + ] + }, + { + "compile_id": 143, + "service": 1, + "action": 1, + "do_blacklist": 1, + "do_log": 1, + "user_region": "NOTLogic.OneRegion", + "is_valid": "yes", + "groups": [ + { + "not_flag":0, + "regions": [ + { + "table_name": "HTTP_URL", + "table_type": "expr", + "table_content": { + "keywords": "must-contained-string-of-rule-143", + "expr_type": "none", + "match_method": "sub", + "format": "uncase plain" + } + } + ] + }, + { + "not_flag":1, + "regions": [ + { + "table_name": "HTTP_URL", + "table_type": "expr", + "table_content": { + "keywords": "must-not-contained-string-of-rule-143", + "expr_type": "none", + "match_method": "sub", + "format": "uncase plain" + } + } + ] + } + ] + }, + { + "compile_id": 144, + "service": 1, + "action": 1, + "do_blacklist": 1, + "do_log": 1, + "user_region": "NOTLogic.ScanNotAtLast", + "is_valid": "yes", + "groups": [ + { + "not_flag":0, + "regions": [ + { + "table_name": "HTTP_URL", + "table_type": "expr", + "table_content": { + "keywords": "must-contained-string-of-rule-144", + "expr_type": "none", + "match_method": "sub", + "format": "uncase plain" + } + } + ] + }, + { + "not_flag":1, + "regions": [ + { + "table_name": "KEYWORDS_TABLE", + "table_type": "expr", + "table_content": { + "keywords": "must-not-contained-string-of-rule-144", + "expr_type": "none", + "match_method": "sub", + "format": "uncase plain" + } + } + ] + } + ] + }, + { + "compile_id": 145, + "service": 1, + "action": 1, + "do_blacklist": 1, + "do_log": 1, + "user_region": "NOTLogic.ScanNotIP", + "is_valid": "yes", + "groups": [ + { + "not_flag":0, + "regions": [ + { + "table_name": "HTTP_URL", + "table_type": "expr", + "table_content": { + "keywords": "must-contained-string-of-rule-145", + "expr_type": "none", + "match_method": "sub", + "format": "uncase plain" + } + } + ] + }, + { + "not_flag":1, + "group_name": "123_IP_group" + } + ] + }, + { + "compile_id": 146, + "service": 1, + "action": 1, + "do_blacklist": 1, + "do_log": 1, + "user_region": "StringScan.Regex", + "is_valid": "yes", + "groups": [ + { + "regions": [ + { + "table_name": "HTTP_URL", + "table_type": "expr", + "table_content": { + "keywords": "Cookie:\\s&head", + "expr_type": "regex", + "match_method": "sub", + "format": "uncase plain" + } + } + ] + } + ] + }, + { + "compile_id": 147, + "service": 1, + "action": 1, + "do_blacklist": 1, + "do_log": 1, + "user_region": "StringScan.UTF8EncodedURL", + "is_valid": "yes", + "groups": [ + { + "regions": [ + { + "table_name": "HTTP_URL", + "table_type": "expr", + "table_content": { + "keywords": "googlevideo.com/videoplayback&mn=sn-35153iuxa-5a56%2Csn-n8v7znz7", + "expr_type": "and", + "match_method": "sub", + "format": "uncase plain" + } + } + ] + } + ] + }, + { + "is_valid": "yes", + "do_log": 0, + "effective_rage": 0, + "action": 0, + "compile_id": 148, + "service": 0, + "do_blacklist": 0, + "user_region": "StringScan.ExprPlusWithOffset", + "groups": [ + { + "regions": [ + { + "table_name": "APP_PAYLOAD", + "table_content": { + "format": "hexbin", + "match_method": "sub", + "district": "Payload", + "keywords": "1-1:03&9-10:2d&14-16:2d34&19-21:2d&24-25:2d", + "expr_type": "offset" + }, + "table_type": "expr_plus" + } + ], + "group_name": "Untitled" + } + ] + }, + { + "compile_id": 150, + "service": 0, + "action": 0, + "do_blacklist": 0, + "do_log": 0, + "effective_rage": 0, + "user_region": "StringScan.BugReport20190325", + "is_valid": "yes", + "groups": [ + { + "regions": [ + { + "table_type": "expr", + "table_name": "TROJAN_PAYLOAD", + "table_content": { + "keywords": "0-4:01000000", + "expr_type": "offset", + "format": "hexbin", + "match_method": "sub" + } + } + ], + "group_name": "billgates_regist1" + }, + { + "regions": [ + { + "table_type": "expr", + "table_name": "TROJAN_PAYLOAD", + "table_content": { + "keywords": "1:G2.40", + "expr_type": "none", + "format": "uncase plain", + "match_method": "sub" + } + } + ], + "group_name": "billgates_regist2" + } + ] + }, + { + "compile_id": 151, + "service": 0, + "action": 0, + "do_blacklist": 0, + "do_log": 0, + "effective_rage": 0, + "user_region": "StringScan.PrefixAndSuffix", + "is_valid": "yes", + "groups": [ + { + "regions": [ + { + "table_type": "expr", + "table_name": "MAIL_ADDR", + "table_content": { + "keywords": "ceshi3@mailhost.cn", + "expr_type": "none", + "format": "uncase plain", + "match_method": "suffix" + } + } + ], + "group_name": "Untitled" + } + ] + }, + { + "compile_id": 152, + "service": 0, + "action": 0, + "do_blacklist": 0, + "do_log": 0, + "effective_rage": 0, + "user_region": "StringScan.PrefixAndSuffix", + "is_valid": "yes", + "groups": [ + { + "regions": [ + { + "table_type": "expr", + "table_name": "MAIL_ADDR", + "table_content": { + "keywords": "ceshi3@mailhost.cn", + "expr_type": "none", + "format": "uncase plain", + "match_method": "prefix" + } + }, + { + "table_type": "expr", + "table_name": "MAIL_ADDR", + "table_content": { + "keywords": "ceshi6@mailhost.cn", + "expr_type": "none", + "format": "uncase plain", + "match_method": "prefix" + } + } + ], + "group_name": "152_mail_addr" + }, + { + "group_name": "interval_group_refered", + "sub_groups": [ + {"group_name": "126_interval_group"} + ] + } + ] + }, + { + "compile_id": 153, + "service": 0, + "action": 0, + "do_blacklist": 0, + "do_log": 0, + "effective_rage": 0, + "user_region": "Policy.SubGroup", + "is_valid": "yes", + "groups": [ + { + "regions": [ + { + "table_type": "expr", + "table_name": "MAIL_ADDR", + "table_content": { + "keywords": "ceshi4@mailhost.cn", + "expr_type": "none", + "format": "uncase plain", + "match_method": "prefix" + } + } + ], + "group_name": "Untitled", + "sub_groups": [ + {"group_name": "152_mail_addr"} + ], + "not_flag" : 0 + }, + { + "group_name": "IP_group_refered", + "sub_groups": [ + {"group_name": "123_IP_group"} + ] + } + ] + }, + { + "compile_id": 154, + "service": 0, + "action": 0, + "do_blacklist": 0, + "do_log": 0, + "effective_rage": 0, + "user_region": "ipv4_plus", + "is_valid": "yes", + "groups": [ + { + "regions": [ + { + "table_type": "ip_plus", + "table_name": "IP_PLUS_CONFIG", + "table_content": { + "addr_type": "ipv4", + "saddr_format": "range", + "src_ip1": "10.0.7.100", + "src_ip2": "10.0.7.106", + "sport_format": "range", + "src_port1": "5000", + "src_port2": "5001", + "daddr_format": "mask", + "dst_ip1": "123.56.104.218", + "dst_ip2": "255.255.255.0", + "dport_format": "range", + "dst_port1": "7400", + "dst_port2": "7400", + "protocol": 6, + "direction": "double" + } + } + ], + "not_flag" : 0 + } + ] + }, + { + "compile_id": 155, + "service": 0, + "action": 0, + "do_blacklist": 0, + "do_log": 0, + "effective_rage": 0, + "user_region": "ipv6_plus", + "is_valid": "yes", + "groups": [ + { + "regions": [ + { + "table_type": "ip_plus", + "table_name": "IP_PLUS_CONFIG", + "table_content": { + "addr_type": "ipv6", + "saddr_format": "range", + "src_ip1": "1001:da8:205:1::101", + "src_ip2": "1001:da8:205:1::201", + "sport_format": "mask", + "src_port1": "5210", + "src_port2": "65520", + "daddr_format": "mask", + "dst_ip1": "3001:da8:205:1::401", + "dst_ip2": "ffff:ffff:ffff:ffff:ffff:ffff:ffff:0000", + "dport_format": "mask", + "dst_port1": "0", + "dst_port2": "65535", + "protocol": 6, + "direction": "double" + } + } + ], + "not_flag" : 0 + } + ] + }, + { + "compile_id": 156, + "service": 1, + "action": 1, + "do_blacklist": 1, + "do_log": 1, + "user_region": "ExprPlusWithHex", + "is_valid": "yes", + "groups": [ + { + "group_name": "Untitled", + "regions": [ + { + "table_name": "HTTP_SIGNATURE", + "table_type": "expr_plus", + "table_content": { + "district": "Content-Type", + "keywords": "2f68746d6c", + "expr_type": "none", + "match_method": "sub", + "format": "hexbin" + } + } + ] + } + ] + }, + { + "compile_id": 157, + "service": 0, + "action": 0, + "do_blacklist": 0, + "do_log": 0, + "effective_rage": 0, + "user_region": "StringScan.StreamScanUTF8", + "is_valid": "yes", + "groups": [ + { + "regions": [ + { + "table_type": "expr", + "table_name": "TROJAN_PAYLOAD", + "table_content": { + "keywords": "鎴戠殑璁㈠崟", + "expr_type": "none", + "format": "none", + "match_method": "sub" + } + } + ] + } + ] + }, + { + "compile_id": 158, + "service": 0, + "action": 0, + "do_blacklist": 0, + "do_log": 0, + "effective_rage": 0, + "user_region": "IPScan.IPv4_CIDR", + "is_valid": "yes", + "groups": [ + { + "regions": [ + { + "table_type": "ip_plus", + "table_name": "IP_PLUS_CONFIG", + "table_content": { + "addr_type": "ipv4", + "saddr_format": "CIDR", + "src_ip1": "192.168.0.1", + "src_ip2": "32", + "sport_format": "range", + "src_port1": "5210", + "src_port2": "5211", + "daddr_format": "CIDR", + "dst_ip1": "10.0.6.1", + "dst_ip2": "24", + "dport_format": "mask", + "dst_port1": "0", + "dst_port2": "65535", + "protocol": 6, + "direction": "double" + } + } + ], + "not_flag" : 0 + } + ] + }, + { + "compile_id": 159, + "service": 0, + "action": 0, + "do_blacklist": 0, + "do_log": 0, + "effective_rage": 0, + "user_region": "IPScan.IPv6_CIDR", + "is_valid": "yes", + "groups": [ + { + "regions": [ + { + "table_type": "ip_plus", + "table_name": "IP_PLUS_CONFIG", + "table_content": { + "addr_type": "ipv6", + "saddr_format": "CIDR", + "src_ip1": "2001:db8::", + "src_ip2": "120", + "sport_format": "mask", + "src_port1": "5210", + "src_port2": "65520", + "daddr_format": "CIDR", + "dst_ip1": "2001:4860:4860::8888", + "dst_ip2": "65", + "dport_format": "mask", + "dst_port1": "0", + "dst_port2": "65535", + "protocol": 6, + "direction": "double" + } + } + ], + "not_flag" : 0 + } + ] + }, + { + "compile_id": 160, + "service": 0, + "action": 0, + "do_blacklist": 0, + "do_log": 0, + "effective_rage": 0, + "user_region": "VirtualWithOnePhysical", + "is_valid": "yes", + "groups": [ + { + "group_name":"TakeMeHome", + "virtual_table":"HTTP_RESPONSE_KEYWORDS", + "not_flag" : 0 + }, + { + "not_flag":0, + "regions": [ + { + "table_name": "HTTP_URL", + "table_type": "expr", + "table_content": { + "keywords": "https://blog.csdn.net/littlefang/article/details/8213058", + "expr_type": "none", + "match_method": "sub", + "format": "uncase plain" + } + } + ] + } + ] + }, + { + "compile_id": 161, + "service": 0, + "action": 0, + "do_blacklist": 0, + "do_log": 0, + "effective_rage": 0, + "user_region": "virtual_table_test_temp", + "is_valid": "yes", + "groups": [ + { + "group_name":"vt_grp_http_sig1", + "not_flag":0, + "regions": [ + { + "table_name": "HTTP_SIGNATURE", + "table_type": "expr_plus", + "table_content": { + "district": "User-Agent", + "keywords": "Chrome/78.0.3904.108", + "expr_type": "none", + "match_method": "sub", + "format": "uncase plain" + } + } + ] + }, + { + "group_name":"vt_grp_http_sig2", + "not_flag":0, + "regions": [ + { + "table_name": "HTTP_SIGNATURE", + "table_type": "expr_plus", + "table_content": { + "district": "Cookie", + "keywords": "uid=12345678", + "expr_type": "none", + "match_method": "sub", + "format": "uncase plain" + } + }, + { + "table_name": "HTTP_SIGNATURE", + "table_type": "expr_plus", + "table_content": { + "district": "Cookie", + "keywords": "sessionid=888888", + "expr_type": "none", + "match_method": "sub", + "format": "uncase plain" + } + } + ] + } + ] + }, + { + "compile_id": 162, + "service": 0, + "action": 0, + "do_blacklist": 0, + "do_log": 0, + "effective_rage": 0, + "user_region": "VirtualWithVirtual", + "is_valid": "yes", + "groups": [ + { + "group_name":"vt_grp_http_sig1", + "virtual_table":"HTTP_REQUEST_HEADER", + "not_flag":0 + }, + { + "group_name":"vt_grp_http_sig2", + "virtual_table":"HTTP_RESPONSE_HEADER", + "not_flag":0 + } + ] + }, + { + "compile_id": 163, + "service": 0, + "action": 0, + "do_blacklist": 0, + "do_log": 0, + "effective_rage": 0, + "user_region": "OneGroupInTwoVirtual", + "is_valid": "yes", + "groups": [ + { + "group_name":"vt_grp_http_sig2", + "virtual_table":"HTTP_REQUEST_HEADER", + "not_flag":0 + }, + { + "group_name":"vt_grp_http_sig2", + "virtual_table":"HTTP_RESPONSE_HEADER", + "not_flag":0 + } + ] + }, + { + "compile_id": 164, + "service": 1, + "action": 1, + "do_blacklist": 1, + "do_log": 1, + "user_region": "CharsetWindows1251", + "is_valid": "yes", + "groups": [ + { + "group_name": "Untitled", + "regions": [ + { + "table_name": "KEYWORDS_TABLE", + "table_type": "expr", + "table_content": { + "keywords": ">袟袗袨\\b芦小械胁械褉谐邪蟹胁褌芯屑邪褌懈泻邪\\b袗泄小禄<", + "expr_type": "none", + "match_method": "sub", + "format": "uncase plain" + } + } + ] + } + ] + }, + { + "compile_id": 165, + "service": 1, + "action": 1, + "do_blacklist": 1, + "do_log": 1, + "user_region": "EvaluationOrder", + "is_valid": "yes", + "evaluation_order":"2.111", + "groups": [ + { + "group_name": "Untitled", + "regions": [ + { + "table_name": "HTTP_URL", + "table_type": "expr", + "table_content": { + "keywords": "cavemancircus.com/", + "expr_type": "none", + "match_method": "sub", + "format": "uncase plain" + } + } + ] + }, + { + "regions": [ + { + "table_type": "ip_plus", + "table_name": "IP_PLUS_CONFIG", + "table_content": { + "addr_type": "ipv4", + "saddr_format": "CIDR", + "src_ip1": "192.168.23.1", + "src_ip2": "24" + } + } + ], + "not_flag" : 0 + } + ] + }, + { + "compile_id": 166, + "service": 1, + "action": 1, + "do_blacklist": 1, + "do_log": 1, + "user_region": "EvaluationOrder", + "is_valid": "yes", + "evaluation_order":"100.233", + "groups": [ + { + "group_name": "Untitled", + "regions": [ + { + "table_name": "HTTP_URL", + "table_type": "expr", + "table_content": { + "keywords": "2019/12/27/pretty-girls-6", + "expr_type": "none", + "match_method": "sub", + "format": "uncase plain" + } + } + ] + } + ] + }, + { + "compile_id": 167, + "service": 1, + "action": 1, + "do_blacklist": 1, + "do_log": 1, + "user_region": "EvaluationOrder", + "is_valid": "yes", + "evaluation_order":"300.999", + "groups": [ + { + "group_name": "Untitled", + "regions": [ + { + "table_name": "HTTP_URL", + "table_type": "expr", + "table_content": { + "keywords": "2019/12/27", + "expr_type": "none", + "match_method": "sub", + "format": "uncase plain" + } + } + ] + } + ] + }, + { + "compile_id": 168, + "service": 1, + "action": 1, + "do_blacklist": 1, + "do_log": 1, + "user_region": "EvaluationOrder", + "is_valid": "yes", + "evaluation_order":"0", + "groups": [ + { + "group_name": "Untitled", + "regions": [ + { + "table_name": "HTTP_URL", + "table_type": "expr", + "table_content": { + "keywords": "2019/12/27", + "expr_type": "none", + "match_method": "sub", + "format": "uncase plain" + } + } + ] + } + ] + }, + { + "compile_id": 169, + "service": 0, + "action": 0, + "do_blacklist": 0, + "do_log": 0, + "effective_rage": 0, + "user_region": "IPScan.IPv4_Any", + "is_valid": "yes", + "groups": [ + { + "regions": [ + { + "table_type": "ip_plus", + "table_name": "IP_PLUS_CONFIG", + "table_content": { + "addr_type": "ipv4", + "saddr_format": "CIDR", + "src_ip1": "0.0.0.0", + "src_ip2": "0", + "sport_format": "mask", + "src_port1": "20304", + "src_port2": "65535", + "daddr_format": "CIDR", + "dst_ip1": "0.0.0.0", + "dst_ip2": "0", + "dport_format": "range", + "dst_port1": "0", + "dst_port2": "0", + "protocol": 6, + "direction": "single" + } + } + ], + "not_flag" : 0 + } + ] + }, + { + "compile_id": 170, + "service": 0, + "action": 0, + "do_blacklist": 0, + "do_log": 0, + "effective_rage": 0, + "user_region": "IPScan.IPv4_virtual.source", + "is_valid": "no", + "groups": [ + { + "group_name": "ipv4_virtual.source", + "regions": [ + { + "table_type": "ip_plus", + "table_name": "IP_PLUS_CONFIG", + "table_content": { + "addr_type": "ipv4", + "saddr_format": "CIDR", + "src_ip1": "192.168.40.10", + "src_ip2": "32", + "sport_format": "mask", + "src_port1": "443", + "src_port2": "65535", + "daddr_format": "CIDR", + "dst_ip1": "0.0.0.0", + "dst_ip2": "0", + "dport_format": "range", + "dst_port1": "0", + "dst_port2": "0", + "protocol": 6, + "direction": "double" + } + } + ], + "not_flag" : 0 + } + ] + }, + { + "compile_id": 171, + "service": 0, + "action": 0, + "do_blacklist": 0, + "do_log": 0, + "effective_rage": 0, + "user_region": "IPScan.IPv4_virtual.destination", + "is_valid": "no", + "groups": [ + { + "group_name": "ipv4_virtual.destination", + "regions": [ + { + "table_type": "ip_plus", + "table_name": "IP_PLUS_CONFIG", + "table_content": { + "addr_type": "ipv4", + "saddr_format": "CIDR", + "src_ip1": "192.168.231.46", + "src_ip2": "32", + "sport_format": "mask", + "src_port1": "25705", + "src_port2": "65535", + "daddr_format": "CIDR", + "dst_ip1": "0.0.0.0", + "dst_ip2": "0", + "dport_format": "range", + "dst_port1": "0", + "dst_port2": "0", + "protocol": 6, + "direction": "double" + } + } + ], + "not_flag" : 0 + } + ] + }, + { + "compile_id": 172, + "service": 0, + "action": 0, + "do_blacklist": 0, + "do_log": 0, + "effective_rage": 0, + "user_region": "ipv4_virtual.match", + "is_valid": "yes", + "groups": [ + { + "group_name":"ipv4_virtual.source", + "virtual_table":"VIRTUAL_IP_PLUS_SOURCE", + "not_flag":0 + }, + { + "group_name":"ipv4_virtual.destination", + "virtual_table":"VIRTUAL_IP_PLUS_DESTINATION", + "not_flag":0 + } + ] + }, + { + "compile_id": 173, + "service": 0, + "action": 0, + "do_blacklist": 0, + "do_log": 0, + "effective_rage": 0, + "user_region": "IPScan.IPv4_composition.source", + "is_valid": "no", + "groups": [ + { + "group_name": "ipv4_composition.source", + "regions": [ + { + "table_type": "ip_plus", + "table_name": "IP_PLUS_CONFIG", + "table_content": { + "addr_type": "ipv4", + "saddr_format": "CIDR", + "src_ip1": "192.168.40.11", + "src_ip2": "32", + "sport_format": "mask", + "src_port1": "443", + "src_port2": "65535", + "daddr_format": "CIDR", + "dst_ip1": "0.0.0.0", + "dst_ip2": "0", + "dport_format": "range", + "dst_port1": "0", + "dst_port2": "0", + "protocol": 6, + "direction": "double" + } + } + ], + "not_flag" : 0 + } + ] + }, + { + "compile_id": 174, + "service": 0, + "action": 0, + "do_blacklist": 0, + "do_log": 0, + "effective_rage": 0, + "user_region": "IPScan.IPv4_composition.destination", + "is_valid": "no", + "groups": [ + { + "group_name": "ipv4_composition.destination", + "regions": [ + { + "table_type": "ip_plus", + "table_name": "IP_PLUS_CONFIG", + "table_content": { + "addr_type": "ipv4", + "saddr_format": "CIDR", + "src_ip1": "192.168.231.47", + "src_ip2": "32", + "sport_format": "mask", + "src_port1": "25715", + "src_port2": "65535", + "daddr_format": "CIDR", + "dst_ip1": "0.0.0.0", + "dst_ip2": "0", + "dport_format": "range", + "dst_port1": "0", + "dst_port2": "0", + "protocol": 6, + "direction": "double" + } + } + ], + "not_flag" : 0 + } + ] + }, + { + "compile_id": 175, + "service": 0, + "action": 0, + "do_blacklist": 0, + "do_log": 0, + "effective_rage": 0, + "user_region": "ipv4_composition.match", + "is_valid": "yes", + "groups": [ + { + "group_name":"ipv4_composition.source", + "virtual_table":"COMPOSITION_IP_SOURCE", + "not_flag":0 + }, + { + "group_name":"ipv4_composition.destination", + "virtual_table":"COMPOSITION_IP_DESTINATION", + "not_flag":0 + } + ] + }, + { + "compile_id": 176, + "service": 0, + "action": 0, + "do_blacklist": 0, + "do_log": 0, + "effective_rage": 0, + "user_region": "IPScan.IPv4_composition.session", + "is_valid": "no", + "groups": [ + { + "group_name": "ipv4_composition.session", + "regions": [ + { + "table_type": "ip_plus", + "table_name": "IP_PLUS_CONFIG", + "table_content": { + "addr_type": "ipv4", + "saddr_format": "CIDR", + "src_ip1": "192.168.40.11", + "src_ip2": "2", + "sport_format": "mask", + "src_port1": "443", + "src_port2": "65535", + "daddr_format": "CIDR", + "dst_ip1": "192.168.231.47", + "dst_ip2": "32", + "dport_format": "range", + "dst_port1": "25715", + "dst_port2": "25715", + "protocol": 6, + "direction": "single" + } + } + ], + "not_flag" : 0 + } + ] + }, + { + "compile_id": 177, + "service": 0, + "action": 0, + "do_blacklist": 0, + "do_log": 0, + "effective_rage": 0, + "user_region": "ipv4_composition.session.match", + "is_valid": "yes", + "groups": [ + { + "group_name":"ipv4_composition.session", + "virtual_table":"COMPOSITION_IP_SESSION", + "not_flag":0, + "clause_index":1 + + } + ] + }, + { + "compile_id": 178, + "service": 1, + "action": 1, + "do_blacklist": 1, + "do_log": 1, + "user_region": "Hierarchy.TwoVirtualInOneClause", + "is_valid": "yes", + "groups": [ + { + "group_name":"ASN1234", + "virtual_table":"SOURCE_IP_ASN", + "not_flag":0, + "clause_index":0 + }, + { + "group_name":"financial-department-ip", + "virtual_table":"null", + "not_flag":0, + "clause_index":0 + }, + { + "group_name":"Country-Sparta-IP", + "virtual_table":"SOURCE_IP_GEO", + "not_flag":0, + "clause_index":0 + }, + { + "group_name":"ASN2345", + "virtual_table":"DESTINATION_IP_ASN", + "not_flag":0, + "clause_index":1 + } + ] + }, + { + "compile_id": 179, + "service": 1, + "action": 1, + "do_blacklist": 1, + "do_log": 1, + "user_region": "anything", + "is_valid": "yes", + "groups": [ + { + "group_name": "Untitled", + "regions": [ + { + "table_name": "INTERGER_PLUS", + "table_type": "interval_plus", + "table_content": { + "district": "interval.plus", + "low_boundary": 2020, + "up_boundary": 2020 + } + } + ] + } + ] + }, + { + "compile_id": 180, + "service": 0, + "action": 0, + "do_blacklist": 0, + "do_log": 0, + "effective_rage": 0, + "user_region": "Hierarchy_VirtualWithTwoPhysical", + "is_valid": "yes", + "groups": [ + { + "group_name":"FQDN_OBJ1", + "virtual_table":"VIRTUAL_SSL_SNI", + "not_flag" : 0, + "clause_index":0 + }, + { + "group_name":"FQDN_CAT1", + "virtual_table":"VIRTUAL_SSL_SNI", + "not_flag" : 0, + "clause_index":0 + } + ] + }, + { + "compile_id": 181, + "service": 0, + "action": 0, + "do_blacklist": 0, + "do_log": 0, + "effective_rage": 0, + "user_region": "ipv4_composition.match", + "is_valid": "yes", + "groups": [ + { + "group_name":"IPv4-composition-source-only", + "virtual_table":"COMPOSITION_IP_SOURCE", + "not_flag":0 + } + ] + }, + { + "compile_id": 182, + "service": 1, + "action": 1, + "do_blacklist": 1, + "do_log": 1, + "user_region": "8-expr", + "is_valid": "yes", + "groups": [ + { + "regions": [ + { + "table_name": "KEYWORDS_TABLE", + "table_type": "expr", + "table_content": { + "keywords": "string1&string2&string3&string4&string5&string6&string7&string8", + "expr_type": "and", + "match_method": "expr", + "format": "uncase plain" + } + } + ] + } + ] + }, + + { + "compile_id": 184, + "user_region": "APP_ID=6006740;Liumengyan-Bugreport-20210515", + "description": "Hulu", + "is_valid": "yes", + "do_blacklist": 0, + "do_log": 0, + "action": 0, + "service": 0, + "groups": [ + { + "group_name": "Untitled", + "regions": [ + + { + "table_name": "IP_CONFIG", + "table_type": "ip", + "table_content": { + "protocol": 0, + "addr_type": "ipv6", + "direction": "double", + "src_ip": "::", + "dst_ip": "2620:100:3000::", + "src_port": "0", + "dst_port": "0", + "mask_src_port": "65535", + "mask_src_ip": "ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff", + "mask_dst_port": "65535", + "mask_dst_ip": "ffff:ffff:ff00:0000:0000:0000:0000:0000" + } + } + ] + } + ] + }, + { + "compile_id": 185, + "service": 0, + "action": 0, + "do_blacklist": 0, + "do_log": 0, + "effective_rage": 0, + "user_region": "ipv4_composition.NOT_match", + "is_valid": "yes", + "groups": [ + { + "group_name":"IPv4-composition-NOT-client-ip", + "virtual_table":"COMPOSITION_IP_SOURCE", + "not_flag":0 + }, + { + "group_name":"IPv4-composition-NOT-server-ip", + "virtual_table":"COMPOSITION_IP_DESTINATION", + "not_flag":1 + } + ] + }, + { + "compile_id": 186, + "service": 1, + "action": 1, + "do_blacklist": 1, + "do_log": 1, + "user_region": "NOTLogic.ScanHitAtLast", + "is_valid": "yes", + "groups": [ + { + "not_flag": 1, + "regions": [ + { + "table_name": "HTTP_URL", + "table_type": "expr", + "table_content": { + "keywords": "must-not-contained-string-of-rule-186", + "expr_type": "none", + "match_method": "sub", + "format": "uncase plain" + } + } + ] + }, + { + "regions": [ + { + "table_type": "ip_plus", + "table_name": "IP_PLUS_CONFIG", + "table_content": { + "addr_type": "ipv4", + "saddr_format": "range", + "src_ip1": "10.0.8.186", + "src_ip2": "10.0.8.186", + "sport_format": "range", + "src_port1": "18611", + "src_port2": "18611", + "daddr_format": "range", + "dst_ip1": "10.0.8.20", + "dst_ip2": "10.0.8.20", + "dport_format": "range", + "dst_port1": "80", + "dst_port2": "80", + "protocol": 6, + "direction": "single" + } + } + ], + "not_flag": 0 + } + ] + }, + { + "compile_id": 187, + "service": 1, + "action": 1, + "do_blacklist": 1, + "do_log": 1, + "user_region": "NOTLogic.ScanHitAtLast", + "is_valid": "yes", + "groups": [ + { + "not_flag": 1, + "regions": [ + { + "table_name": "HTTP_URL", + "table_type": "expr", + "table_content": { + "keywords": "must-not-contained-string-of-rule-187", + "expr_type": "none", + "match_method": "sub", + "format": "uncase plain" + } + } + ] + }, + { + "regions": [ + { + "table_type": "ip_plus", + "table_name": "IP_PLUS_CONFIG", + "table_content": { + "addr_type": "ipv4", + "saddr_format": "range", + "src_ip1": "10.0.8.187", + "src_ip2": "10.0.8.187", + "sport_format": "range", + "src_port1": "18611", + "src_port2": "18611", + "daddr_format": "range", + "dst_ip1": "10.0.8.20", + "dst_ip2": "10.0.8.20", + "dport_format": "range", + "dst_port1": "80", + "dst_port2": "80", + "protocol": 6, + "direction": "single" + } + } + ], + "not_flag": 0 + } + ] + }, + { + "compile_id": 188, + "service": 1, + "action": 1, + "do_blacklist": 1, + "do_log": 1, + "user_region": "NOTLogic.ScanHitAtLast", + "is_valid": "yes", + "groups": [ + { + "not_flag": 1, + "regions": [ + { + "table_name": "HTTP_URL", + "table_type": "expr", + "table_content": { + "keywords": "must-not-contained-string-of-rule-188", + "expr_type": "none", + "match_method": "sub", + "format": "uncase plain" + } + } + ] + }, + { + "regions": [ + { + "table_type": "ip_plus", + "table_name": "IP_PLUS_CONFIG", + "table_content": { + "addr_type": "ipv4", + "saddr_format": "range", + "src_ip1": "10.0.8.188", + "src_ip2": "10.0.8.188", + "sport_format": "range", + "src_port1": "18611", + "src_port2": "18611", + "daddr_format": "range", + "dst_ip1": "10.0.8.20", + "dst_ip2": "10.0.8.20", + "dport_format": "range", + "dst_port1": "80", + "dst_port2": "80", + "protocol": 6, + "direction": "single" + } + } + ], + "not_flag": 0 + } + ] + }, + { + "is_valid": "yes", + "do_log": 0, + "effective_rage": 0, + "action": 0, + "compile_id": 189, + "service": 0, + "do_blacklist": 0, + "user_region": "StringScan.ShouldNotHitExprPlus", + "groups": [ + { + "regions": [ + { + "table_name": "APP_PAYLOAD", + "table_content": { + "format": "hexbin", + "match_method": "sub", + "district": "tcp.payload.c2s_first_data", + "keywords": "ab00", + "expr_type": "none" + }, + "table_type": "expr_plus" + } + ], + "group_name": "Untitled" + } + ] + }, + { + "compile_id": 190, + "service": 1, + "action": 1, + "do_blacklist": 1, + "do_log": 1, + "user_region": "StringScan.ExprPlus", + "is_valid": "yes", + "groups": [ + { + "group_name": "Untitled", + "regions": [ + { + "table_name": "HTTP_SIGNATURE", + "table_type": "expr_plus", + "table_content": { + "district": "鎴戠殑DistrIct", + "keywords": "addis&sapphire", + "expr_type": "and", + "match_method": "sub", + "format": "uncase plain" + } + } + ] + } + ] + } + ], + "plugin_table": [ + { + "table_name": "QD_ENTRY_INFO", + "table_content": [ + "1\t192.168.0.1\t101\t1", + "2\t192.168.0.2\t101\t1", + "3\t192.168.1.1\t102\t1" + ] + }, + { + "table_name": "TEST_PLUGIN_TABLE", + "table_content": [ + "1\t3388\t99\t1", + "2\t3355\t66\t1", + "3\tcccc\t11\t1" + ] + }, + { + "table_name": "TEST_EFFECTIVE_RANGE_TABLE", + "table_content": [ + "1\tSUCCESS\t99\t1\t{\"tag_sets\":[[{\"tag\":\"location\",\"value\":[\"鍖椾含/鏈濋槼/鍗庝弗鍖楅噷\"]},{\"tag\":\"isp\",\"value\":[\"鐢典俊\",\"绉诲姩\"]}]]}\t1111", + "2\tSUCCESS\t66\t1\t0\t222", + "3\tFAILED\t11\t1\t{\"tag_sets\":[[{\"tag\":\"location\",\"value\":[\"鍖椾含/鏈濋槼/鍗庝弗鍖楅噷\",\"涓婃捣/娴︿笢/闄嗗鍢碶"]},{\"tag\":\"isp\",\"value\":[\"鐢典俊\",\"鑱旈歕"]}],[{\"tag\":\"location\",\"value\":[\"鍖椾含\"]},{\"tag\":\"isp\",\"value\":[\"鑱旈歕"]}]]}\t333", + "4\tSUCCESS\t66\t1\t{}\t444", + "5\tSUCCESS\t66\t1\t{\"tag_sets\":[[{\"tag\":\"location\",\"value\":[\"鍖椾含\"]}]]}\t444", + "6\tSUCCESS\t66\t1\t{\"tag_sets\":[[{\"tag\":\"weather\",\"value\":[\"hot\"]}]]}\t444" + ] + }, + { + "table_name": "IR_INTERCEPT_IP", + "table_content": [ + "1000000130\t1000000130\t4\t192.168.10.99\t255.255.255.255\t0\t65535\t0.0.0.0\t255.255.255.255\t0\t65535\t0\t1\t1\t96\t1\tuser_region\t{}\t2019/1/24/18:0:34", + "161\t161\t4\t0.0.0.0\t255.255.255.255\t0\t65535\t61.135.169.121\t255.255.255.255\t0\t65535\t0\t0\t1\t96\t832\t0\t0\t2019/1/24/18:48:42" + ] + }, + { + "table_name": "TEST_IP_PLUGIN_WITH_EXDATA", + "table_content": [ + "101\t4\t192.168.30.99\t192.168.30.101\tSomething-like-json\t1", + "102\t4\t192.168.30.90\t192.168.30.128\tBigger-range-should-in-the-back\t1", + "103\t6\t2001:db8:1234::\t2001:db8:1235::\tBigger-range-should-in-the-back\t1", + "104\t6\t2001:db8:1234::1\t2001:db8:1234::5210\tSomething-like-json\t1", + "105\t6\t2620:100:3000::\t2620:0100:30ff:ffff:ffff:ffff:ffff:ffff\tBugreport-liumengyan-20210517\t1" + ] + }, + { + "table_name": "TEST_FQDN_PLUGIN_WITH_EXDATA", + "table_content": [ + "201\t0\twww.example1.com\tcatid=1\t1", + "202\t1\t.example1.com\tcatid=1\t1", + "203\t0\tnews.example1.com\tcatid=2\t1", + "204\t0\tr3---sn-i3belne6.example2.com\tcatid=3\t1", + "205\t0\tr3---sn-i3belne6.example2.com\tcatid=3\t1" + ] + }, + { + "table_name":"TEST_BOOL_PLUGIN_WITH_EXDATA", + "table_content": [ + "301\t1&2&1000\ttunnel1\t1", + "302\t101&102\ttunnel2\t1", + "303\t102\ttunnel3\t1", + "304\t101\ttunnel4\t1", + "305\t0&1&2&3&4&5&6&7\ttunnel5\t1", + "306\t101&101\tinvalid\t1" + ] + } + ] +} diff --git a/tools/CMakeLists.txt b/tools/CMakeLists.txt new file mode 100644 index 0000000..e1df602 --- /dev/null +++ b/tools/CMakeLists.txt @@ -0,0 +1,8 @@ +include_directories(${PROJECT_SOURCE_DIR}/src/inc_internal) +include_directories(${PROJECT_SOURCE_DIR}/deps) +include_directories(${PROJECT_SOURCE_DIR}/scanner) + +add_executable(maat_redis_tool maat_redis_tool.cpp) +target_link_libraries(maat_redis_tool maat_frame_static) + +install(TARGETS maat_redis_tool DESTINATION /usr/local/bin/ COMPONENT TOOLS) \ No newline at end of file diff --git a/tools/maat_redis_tool.cpp b/tools/maat_redis_tool.cpp new file mode 100644 index 0000000..7b969fa --- /dev/null +++ b/tools/maat_redis_tool.cpp @@ -0,0 +1,394 @@ +#include +#include +#include +#include +#include +#include + +#include "utils.h" +#include "maat_rule.h" +#include "maat_utils.h" +#include "maat_command.h" +#include "cJSON/cJSON.h" +#include "maat_config_monitor.h" +#include "json2iris.h" +#include "hiredis/hiredis.h" + +const char *redis_dump_dir = "./redis_dump"; +const char *default_table_info = "./table_info.conf"; + +void maat_tool_print_usage(void) +{ + printf("maat_redis_tool manipulate rules from redis.\n"); + printf("Usage:\n"); + printf("\t-h [host], redis IP, 127.0.0.1 as default.\n"); + printf("\t-p [port], redis port, 6379 as default.\n"); + printf("\t-n [db], redis db, 0 as default.\n"); + printf("\t-d [dir], dump rules from redis to [dir], %s as default.\n",redis_dump_dir); + printf("\t-v [version], dump specific [version] from redis, dump latest version as default.\n"); + printf("\t-j [payload.json], add or delete rules as maat json. Must have field compile_table field, and plugin table's valid flag must be in the last column.\n"); + printf("\t-t [timeout], timeout config after t seconds, default is 0 which means never timeout.\n"); + printf("example: ./maat_redis_tool -h 127.0.0.1 -p 6379 -d %s\n",redis_dump_dir); + printf(" ./maat_redis_tool -h 127.0.0.1 -p 6379 -j payload.json -t 300\n"); +} + +static int compare_serial_rule(const void *a, const void *b) +{ + struct serial_rule *ra=(struct serial_rule *)a; + struct serial_rule *rb=(struct serial_rule *)b; + + int ret = strcmp(ra->table_name, rb->table_name); + if (0 == ret) { + ret = ra->rule_id - rb->rule_id; + } + + return ret; +} + +int set_file_rulenum(const char *path, int rule_num) +{ + FILE* fp=NULL; + if (0 == rule_num) { + fp = fopen(path, "w"); + } else { + fp = fopen(path, "r+"); + } + + if (NULL == fp) { + fprintf(stderr, "fopen %s failed %s at set rule num.", path, strerror(errno)); + return -1; + } + + fprintf(fp, "%010d\n", rule_num); + fclose(fp); + + return 0; +} + +void read_rule_from_redis(redisContext *c, long long desire_version, const char* output_path) +{ + struct serial_rule *rule_list; + int line_count=0; + int i=0,ret=0; + int update_type = CM_UPDATE_TYPE_INC; + long long version=0; + const char* cur_table=NULL; + char foreign_files_dir[256] = {0}; + char table_path[256],index_path[256]; + FILE *table_fp=NULL, *index_fp=NULL; + + int rule_num = maat_cmd_get_rm_key_list(c, 0, desire_version, &version, NULL, &rule_list, &update_type, 0); + if (0 == rule_num) { + if (desire_version != 0) { + printf("Read desired version %lld failed.\n", desire_version); + } else { + printf("No Effective Rules.\n"); + } + return; + } + + if (rule_num < 0) { + printf("Read Redis Error.\n"); + return; + } + + assert(update_type == CM_UPDATE_TYPE_FULL); + printf("MAAT Version: %lld, key number: %d\n", version, rule_num); + if (0 == rule_num) { + goto clean_up; + } + + printf("Reading value: \n"); + ret = maat_cmd_get_redis_value(c, rule_list, rule_num, 1); + if (ret < 0) { + goto clean_up; + } + + printf("Sorting.\n"); + qsort(rule_list, rule_num, sizeof(struct serial_rule), compare_serial_rule); + if ((access(output_path, F_OK)) <0) { + if ((mkdir(output_path, S_IRWXU | S_IRWXG | S_IROTH | S_IXOTH)) < 0) { + printf("mkdir %s error\n", output_path); + } + } + + snprintf(foreign_files_dir, sizeof(foreign_files_dir), "%s/foreign_files/",output_path); + + if ((access(foreign_files_dir, F_OK)) <0) { + if((mkdir(foreign_files_dir, S_IRWXU | S_IRWXG | S_IROTH | S_IXOTH)) < 0) { + printf("mkdir %s error\n", foreign_files_dir); + } + } + + ret = maat_cmd_get_foreign_keys_by_prefix(c, rule_list, rule_num, foreign_files_dir); + if (ret > 0) { + printf("%d lines has foreign content.\n", ret); + maat_cmd_get_foreign_conts(c, rule_list, rule_num, 1); + } + + snprintf(index_path,sizeof(index_path),"%s/full_config_index.%020lld",output_path,version); + index_fp = fopen(index_path, "w"); + if (NULL == index_fp) { + printf("Open %s failed.\n",index_path); + goto clean_up; + } + + for (i = 0; i < rule_num; i++) { + if (rule_list[i].n_foreign > 0) { + maat_cmd_rewrite_table_line_with_foreign(rule_list+i); + } + + if (NULL == cur_table || 0 != strcmp(cur_table,rule_list[i].table_name)) { + if (table_fp != NULL) { + fprintf(index_fp, "%s\t%d\t%s\n", cur_table, line_count, table_path); + fclose(table_fp); + table_fp = NULL; + set_file_rulenum(table_path, line_count); + line_count = 0; + printf("Written table %s\n",table_path); + } + + snprintf(table_path, sizeof(table_path), "%s/%s.%020lld", output_path, + rule_list[i].table_name, version); + set_file_rulenum(table_path, 0); + + table_fp = fopen(table_path, "a"); + if (NULL == table_fp) { + printf("Open %s failed.\n", table_path); + goto clean_up; + } + + cur_table = rule_list[i].table_name; + } + fprintf(table_fp, "%s\tkey=%ld\n", rule_list[i].table_line, rule_list[i].rule_id); + line_count++; + } + + fclose(table_fp); + table_fp = NULL; + + fprintf(index_fp, "%s\t%d\t%s\n", cur_table, line_count, table_path); + set_file_rulenum(table_path, line_count); + printf("Written table %s\n", table_path); + printf("Written complete: %s\n", index_path); + +clean_up: + for (i = 0; i < rule_num; i++) { + maat_cmd_empty_serial_rule(rule_list+i); + } + + free(rule_list); + rule_list = NULL; + + if (c != NULL) { + redisFree(c); + } + + if (index_fp != NULL) { + fclose(index_fp); + } + + if (table_fp != NULL) { + fclose(table_fp); + } +} + +int count_line_num(const char *table_name,const char *line, void *u_para) +{ + (*((unsigned int *)u_para))++; + return 0; +} + +int line_idx = 0; +long long absolute_expire_time=0; +int make_serial_rule(const char *table_name, const char *line,void *u_para) +{ + struct serial_rule *s_rule=(struct serial_rule *)u_para; + int rule_id=0,is_valid=0; + char *buff = ALLOC(char, strlen(line) + 1); + + memcpy(buff, line, strlen(line) + 1); + + while (buff[strlen(buff) - 1] == '\n' || buff[strlen(buff) - 1] == '\t') { + buff[strlen(buff) - 1] = '\0'; + } + + int j = 0; + char *str1 = NULL; + char *token = NULL; + char *saveptr1 = NULL; + char *last1 =NULL; + for (j = 0,str1 = buff; ; j++, str1 = NULL) { + token = strtok_r(str1, "\t ", &saveptr1); + if (token == NULL) + break; + if(j==0) + { + sscanf(token,"%d",&rule_id); + } + last1=token; + } + sscanf(last1,"%d",&is_valid); + + memcpy(buff,line, strlen(line)+1); + while(buff[strlen(buff)-1]=='\n'||buff[strlen(buff)-1]=='\t') + { + buff[strlen(buff)-1]='\0'; + } + //printf("Writing table %s, rule_id=%d, timeout=%lld, is_valid=%d\n", table_name, rule_id,absolute_expire_time,is_valid); + maat_cmd_set_serial_rule(s_rule + line_idx, (enum maat_operation)is_valid, rule_id, table_name, buff, absolute_expire_time); + line_idx++; + free(str1); + + return 0; +} + +#define WORK_MODE_DUMP 0 +#define WORK_MODE_JSON 1 +int main(int argc, char * argv[]) +{ + int oc=0,ret=0; + int model=0; + char redis_ip[64]; + int redis_port=6379; + int redis_db=0; + strncpy(redis_ip, "127.0.0.1", sizeof(redis_ip)); + char dump_dir[128], json_file[128], tmp_iris_path[128]; + strncpy(dump_dir,redis_dump_dir,sizeof(dump_dir)); + redisContext * ctx=NULL; + unsigned int total_line_cnt=0, success_cnt=0, i=0; + int timeout=0; + FILE* json_fp=NULL; + cJSON *json=NULL, *tmp_obj=NULL; + struct stat fstat_buf; + unsigned long json_file_size=0,read_size=0; + long long desired_version=0; + char* json_buff=NULL; + size_t json_buff_sz=0; + char compile_table_name[128] = {0}; + + while ((oc = getopt(argc,argv,"h:p:n:d:v:f:j:t:")) != -1) { + switch (oc) { + case 'h': + strncpy(redis_ip, optarg, sizeof(redis_ip)); + break; + case 'p': + sscanf(optarg, "%d", &redis_port); + break; + case 'n': + sscanf(optarg, "%d", &redis_db); + break; + case 'd': + model = WORK_MODE_DUMP; + strncpy(dump_dir, optarg, sizeof(dump_dir)); + if (dump_dir[strlen(dump_dir)-1] == '/') { + dump_dir[strlen(dump_dir)-1] = '\0'; + } + break; + case 'v': + sscanf(optarg, "%lld", &desired_version); + break; + case 'j': + strncpy(json_file, optarg, sizeof(json_file)); + model = WORK_MODE_JSON; + ret = stat(json_file, &fstat_buf); + if (ret != 0) { + printf("fstat file %s error.\n", json_file); + return -1; + } + + json_fp = fopen(json_file, "r"); + if (NULL == json_fp) { + printf("fopen file %s error %s.\n", json_file, strerror(errno)); + return -1; + } + + json_file_size = fstat_buf.st_size; + json_buff = ALLOC(char, json_file_size); + read_size = fread(json_buff, 1, json_file_size, json_fp); + if (read_size != json_file_size) { + printf("fread file %s error.\n", json_file); + return -1; + } + + json = cJSON_Parse(json_buff); + if (!json) { + printf("%s format is error before: %-200.200s\n", json_file, cJSON_GetErrorPtr()); + return -1; + } + + tmp_obj=cJSON_GetObjectItem(json, "compile_table"); + if (NULL == tmp_obj) { + printf("No \"compile_table\" in %s.\n",json_file); + return -1; + } + strncpy(compile_table_name, tmp_obj->valuestring, sizeof(compile_table_name)); + free(json_buff); + cJSON_Delete(json); + fclose(json_fp); + break; + case 't': + sscanf(optarg,"%d",&timeout); + break; + case '?': + default: + maat_tool_print_usage(); + return 0; + break; + } + } + + ctx = maat_cmd_connect_redis(redis_ip, redis_port, redis_db); + if (NULL == ctx) { + return -1; + } + + if (model == WORK_MODE_DUMP) { + printf("Reading key list from %s:%d db%d.\n", redis_ip, redis_port, redis_db); + read_rule_from_redis(ctx, desired_version, dump_dir); + } else if(model == WORK_MODE_JSON) { + ret = load_file_to_memory(json_file, (unsigned char**)&json_buff, &json_buff_sz); + if (ret < 0) { + printf("open %s failed.\n", json_file); + } + + ret = json2iris(json_buff, json_file, NULL, NULL, NULL, ctx, tmp_iris_path, sizeof(tmp_iris_path), NULL, NULL); + if (ret < 0) { + printf("Invalid json format.\n"); + } + + total_line_cnt = 0; + config_monitor_traverse(0, tmp_iris_path, NULL, count_line_num, NULL, &total_line_cnt); + printf("Serialize %s to %d lines, write temp file to %s .\n",json_file, total_line_cnt, tmp_iris_path); + + struct serial_rule *s_rule = ALLOC(struct serial_rule, total_line_cnt); + long long server_time = maat_cmd_redis_server_time_s(ctx); + if (!server_time) { + printf("Get Redis Time failed.\n"); + } + + if (timeout > 0) { + absolute_expire_time = server_time + timeout; + } + + config_monitor_traverse(0, tmp_iris_path, NULL, make_serial_rule, NULL, s_rule); + printf("Timeout = %lld\n", absolute_expire_time); + + ret = 0; + do { + success_cnt = maat_cmd_exec_serial_rule(ctx, s_rule, total_line_cnt, server_time); + } while(success_cnt < 0); + + if (success_cnt != total_line_cnt) { + printf("Only Add %d of %d, rule id maybe conflicts.\n", success_cnt, total_line_cnt); + } + + for (i = 0; i < total_line_cnt; i++) { + maat_cmd_empty_serial_rule(s_rule+i); + } + free(s_rule); + redisFree(ctx); + } + + return 0; +} \ No newline at end of file diff --git a/vendor/CMakeLists.txt b/vendor/CMakeLists.txt index 8daca01..aa549f0 100644 --- a/vendor/CMakeLists.txt +++ b/vendor/CMakeLists.txt @@ -70,3 +70,16 @@ add_library(ipmatcher-static STATIC IMPORTED GLOBAL) add_dependencies(ipmatcher-static ipmatcher) set_property(TARGET ipmatcher-static PROPERTY IMPORTED_LOCATION ${INSTALL_DIR}/lib/ipmatcher.a) set_property(TARGET ipmatcher-static PROPERTY INTERFACE_INCLUDE_DIRECTORIES ${INSTALL_DIR}/include) + +# hiredis-1.1.0 +ExternalProject_Add(hiredis PREFIX hiredis + URL ${CMAKE_CURRENT_SOURCE_DIR}/hiredis-1.1.0.tar.gz + CMAKE_ARGS -DCMAKE_INSTALL_PREFIX=${VENDOR_BUILD} -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} -DCMAKE_C_FLAGS="-fPIC") + +ExternalProject_Get_Property(hiredis INSTALL_DIR) +file(MAKE_DIRECTORY ${VENDOR_BUILD}/include) + +add_library(hiredis-static STATIC IMPORTED GLOBAL) +add_dependencies(hiredis-static hiredis) +set_property(TARGET hiredis-static PROPERTY IMPORTED_LOCATION ${VENDOR_BUILD}/lib64/libhiredisd.a) +set_property(TARGET hiredis-static PROPERTY INTERFACE_INCLUDE_DIRECTORIES ${VENDOR_BUILD}/include) \ No newline at end of file diff --git a/vendor/hiredis-1.1.0.tar.gz b/vendor/hiredis-1.1.0.tar.gz new file mode 100644 index 0000000..edafe2c Binary files /dev/null and b/vendor/hiredis-1.1.0.tar.gz differ