diff --git a/deps/log/log.c b/deps/log/log.c index f21813e..89808eb 100644 --- a/deps/log/log.c +++ b/deps/log/log.c @@ -29,26 +29,24 @@ #include "log.h" -#define MAX_CALLBACKS 32 - #define ALLOC(type, number) ((type *)calloc(sizeof(type), number)) -typedef struct { - log_func_t fn; - void *user_data; - int level; -} Callback; +typedef enum { + LOG_OP_IFACE_CONSOLE, + LOG_OP_IFACE_FILE, + RT_LOG_OP_IFACE_MAX, +}log_op_iface; struct log_handle { int level; int enable; - void *user_data; - log_lock_func_t lock; - char run_log_path[1024]; - char cur_log_file[1024]; + FILE *fp; + va_list ap; + char defined_log_fn[1024]; + char runtime_log_fn[1024]; pthread_mutex_t mutex; - Callback callbacks[MAX_CALLBACKS]; + log_op_iface iface; }; static unsigned char weekday_str[7][4] = {"Sun", "Mon", "Tue", "Wed", "Thu", "Fri", "Sat"}; @@ -78,29 +76,28 @@ static int log_create_dir(const char *dir_path, int path_len) return ret; } -static void log_close_file(struct log_event *event) +static void log_close_file(struct log_handle *handle) { - pthread_mutex_lock(&event->mutex); - FILE *fp=(FILE *)event->user_data; - if(fp != NULL) + pthread_mutex_lock(&handle->mutex); + if(handle->fp != NULL) { - fclose(fp); - event->user_data = NULL; + fclose(handle->fp); + handle->fp = NULL; } - pthread_mutex_unlock(&event->mutex); + pthread_mutex_unlock(&handle->mutex); return; } -int log_open_file(char *file_name, struct log_event *event) +int log_open_file(char *file_name, struct log_handle *handle) { FILE *fp = NULL; - log_close_file(event); + log_close_file(handle); if(NULL == (fp = fopen(file_name, "a"))) { return -1; } - memcpy(event->cur_log_file, file_name, strlen(file_name)); - event->user_data = fp; + memcpy(handle->runtime_log_fn, file_name, strlen(file_name)); + handle->fp = fp; return 0; } @@ -137,61 +134,38 @@ static int log_create_path(const char *file_path) return 0; } fclose(fp); - return 1; } -int log_create_log_file(struct log_event *event) +int log_create_log_file(struct log_handle *handle) { time_t t; struct tm local_time; - char tmp_log_file_name[512]; - - FILE *fp = (FILE *)event->user_data; + char tmp_log_file_name[1024+128]; time(&t); if(NULL == (localtime_r(&t, &local_time))) { return 0; } - sprintf(tmp_log_file_name, "%s.%04d-%02d-%02d", event->run_log_path, local_time.tm_year + 1900, local_time.tm_mon + 1, local_time.tm_mday); + snprintf(tmp_log_file_name, sizeof(tmp_log_file_name), "%s.%04d-%02d-%02d", handle->defined_log_fn, local_time.tm_year + 1900, local_time.tm_mon + 1, local_time.tm_mday); - if(fp == NULL) + if(handle->fp == NULL) { - if(0 != log_open_file(tmp_log_file_name, event)) return 0; + if(0 != log_open_file(tmp_log_file_name, handle)) return 0; } else { - if (0 != memcmp(tmp_log_file_name, event->cur_log_file, strlen(tmp_log_file_name))) + if (0 != memcmp(tmp_log_file_name, handle->runtime_log_fn, strlen(tmp_log_file_name))) { - if(0 != log_open_file(tmp_log_file_name, event))return 0; + if(0 != log_open_file(tmp_log_file_name, handle))return 0; } } return 1; } -static void log_print_file(struct log_event *event) -{ - char buf[64]; - time_t t; - struct tm local_time; - const char *level_str_map[]= {"TRACE", "DEBUG", "INFO", "WARN", "ERROR", "FATAL"}; - - time(&t); - if(NULL == (localtime_r(&t, &local_time))) return; - snprintf(buf, sizeof(buf), "%s %s %d %02d:%02d:%02d %d", weekday_str[local_time.tm_wday], - month_str[local_time.tm_mon], local_time.tm_mday, local_time.tm_hour, local_time.tm_min, local_time.tm_sec, local_time.tm_year+1900); - - log_create_log_file(event); - fprintf((FILE *)event->user_data, "%s, %s, %s, ", buf, level_str_map[event->level], event->module); - - vfprintf((FILE *)event->user_data, event->fmt, event->ap); - fprintf((FILE *)event->user_data, "\n"); - fflush((FILE *)event->user_data); -} - -static void log_print_console(struct log_event *event) +static void log_print_file(struct log_handle *handle, int level, const char *module, va_list ap, const char *fmt) { char buf[64]={0}; time_t t; @@ -202,148 +176,83 @@ static void log_print_console(struct log_event *event) if(NULL == (localtime_r(&t, &local_time))) return; snprintf(buf, sizeof(buf), "%s %s %d %02d:%02d:%02d %d", weekday_str[local_time.tm_wday], month_str[local_time.tm_mon], local_time.tm_mday, local_time.tm_hour, local_time.tm_min, local_time.tm_sec, local_time.tm_year+1900); - fprintf((FILE *)event->user_data, "%s, %s, %s, ", buf, level_str_map[event->level], event->module); - vfprintf((FILE *)event->user_data, event->fmt, event->ap); - fprintf((FILE *)event->user_data, "\n"); - fflush((FILE *)event->user_data); + log_create_log_file(handle); + fprintf(handle->fp, "%s, %s, %s, ", buf, level_str_map[level], module); + + vfprintf(handle->fp, fmt, ap); + fprintf(handle->fp, "\n"); + fflush(handle->fp); } -void log_options_set_lock_fn(struct log_handle * handle, log_lock_func_t fn, void *user_data) +static void log_print_console(struct log_handle *handle, int level, const char *module, va_list ap, const char *fmt) { - struct log_handle *_handle_t = (struct log_handle *)handle; - if(_handle_t != NULL) - { - _handle_t->lock = fn; - _handle_t->user_data = user_data; - } + char buf[64]={0}; + time_t t; + struct tm local_time; + const char *level_str_map[]= {"TRACE", "DEBUG", "INFO", "WARN", "ERROR", "FATAL"}; + + time(&t); + if(NULL == (localtime_r(&t, &local_time))) return; + snprintf(buf, sizeof(buf), "%s %s %d %02d:%02d:%02d %d", weekday_str[local_time.tm_wday], + month_str[local_time.tm_mon], local_time.tm_mday, local_time.tm_hour, local_time.tm_min, local_time.tm_sec, local_time.tm_year+1900); + fprintf(handle->fp, "%s, %s, %s, ", buf, level_str_map[level], module); + + vfprintf(handle->fp, fmt, ap); + fprintf(handle->fp, "\n"); + fflush(handle->fp); } void log_options_set_level(struct log_handle * handle, int level) { - struct log_handle *_handle_t = (struct log_handle *)handle; - if(_handle_t != NULL) + if(handle != NULL) { - _handle_t->level = level; + handle->level = level; } } void log_options_set_enable(struct log_handle * handle, int enable) { - struct log_handle *_handle_t = (struct log_handle *)handle; - if(_handle_t != NULL) + if(handle != NULL) { - _handle_t->enable = enable; + handle->enable = enable; } } -int log_add_method_callback(void *handle, log_func_t fn, void *udata, int level) -{ - int i=0; - - struct log_handle *_handle_t = (struct log_handle *)handle; - if(_handle_t == NULL) - { - return -1; - } - - for (i = 0; i < MAX_CALLBACKS; i++) - { - if (!_handle_t->callbacks[i].fn) - { - _handle_t->callbacks[i] = (Callback) { fn, udata, level }; - return 0; - } - } - - return -1; -} - -int log_options_add_file_fp(struct log_handle *handle, int level) -{ - int ret=0; - FILE *fp=NULL; - - struct log_handle *_handle_t = (struct log_handle *)handle; - if(!_handle_t) - { - return -1; - } - ret =log_create_path(_handle_t->run_log_path); - if(ret==0) - { - fp=stderr; - } - return log_add_method_callback(handle, log_print_file, fp, level); -} - -int log_options_add_fp(struct log_handle * handle, FILE *fp, int level) -{ - return log_add_method_callback(handle, log_print_file, fp, level); -} - -static void log_lock(void *handle) -{ - struct log_handle *_handle_t = (struct log_handle *)handle; - if(_handle_t != NULL && _handle_t->lock) - { - _handle_t->lock(1, _handle_t->user_data); - } -} - -static void log_unlock(void *handle) -{ - struct log_handle *_handle_t = (struct log_handle *)handle; - if(_handle_t != NULL && _handle_t->lock) - { - _handle_t->lock(0, _handle_t->user_data); - } -} - -static void log_init_event(struct log_event *event, void *user_data) -{ - event->user_data = user_data; -} - struct log_handle *log_handle_create(const char *file_path, int level) { - struct log_handle *_handle_t = ALLOC(struct log_handle, 1); - if(!_handle_t) + struct log_handle *handle = ALLOC(struct log_handle, 1); + if(!handle) { return NULL; } - _handle_t->enable=1; - _handle_t->level = level; - strncpy(_handle_t->run_log_path, file_path, 1024); - pthread_mutex_init(&_handle_t->mutex,NULL); + handle->enable=1; + handle->level = level; + strncpy(handle->defined_log_fn, file_path, 1024); + pthread_mutex_init(&handle->mutex,NULL); - log_options_add_file_fp(_handle_t, level); + if(handle->enable) + { + log_create_path(handle->defined_log_fn); + } - return _handle_t; + return handle; } void log_handle_destroy(struct log_handle * handle) { - int i=0; - struct log_handle *_handle_t = (struct log_handle *)handle; - if(!_handle_t) + if(!handle) { return; } - if (_handle_t->user_data != NULL) + + if(handle->iface == LOG_OP_IFACE_FILE && handle->fp != NULL) { - free(_handle_t->user_data); - _handle_t->user_data = NULL; + fclose(handle->fp); + handle->fp=NULL; } - for(i=0; icallbacks && _handle_t->callbacks->user_data) - { - fclose(_handle_t->callbacks->user_data); - _handle_t->callbacks->user_data = NULL; - } - } - pthread_mutex_destroy(&(_handle_t->mutex)); + + pthread_mutex_destroy(&(handle->mutex)); free(handle); handle = NULL; return; @@ -351,41 +260,21 @@ void log_handle_destroy(struct log_handle * handle) void log_print(struct log_handle *handle, int level, const char *module, const char *fmt, ...) { - struct log_event event; - memset(&event, 0, sizeof(event)); + va_list ap; - struct log_handle *_handle_t = (struct log_handle *)handle; - - event.fmt=fmt; - event.level=level; - event.module=module; - event.mutex=_handle_t->mutex; - event.cur_log_file=_handle_t->cur_log_file; - event.run_log_path=_handle_t->run_log_path; - - if(_handle_t->enable != 1 && level >= _handle_t->level) + if(handle->enable != 1 && level >= handle->level) { - log_init_event(&event, stderr); - va_start(event.ap, fmt); - log_print_console(&event); - va_end(event.ap); + handle->fp = stdout; + handle->iface = LOG_OP_IFACE_CONSOLE; + va_start(handle->ap, fmt); + log_print_console(handle, level, module, ap, fmt); + va_end(handle->ap); } - - log_lock(handle); - - int i=0; - for(i=0; icallbacks[i].fn; i++) + if (handle->enable == 1 && level >= handle->level) { - Callback *cb = &_handle_t->callbacks[i]; - if (level >= cb->level) - { - log_init_event(&event, cb->user_data); - va_start(event.ap, fmt); - cb->fn(&event); - va_end(event.ap); - cb->user_data = event.user_data; - } + handle->iface = LOG_OP_IFACE_FILE; + va_start(ap, fmt); + log_print_file(handle, level, module, ap, fmt); + va_end(ap); } - - log_unlock(handle); } \ No newline at end of file diff --git a/deps/log/log.h b/deps/log/log.h index 7b71a24..d815ed4 100644 --- a/deps/log/log.h +++ b/deps/log/log.h @@ -17,27 +17,11 @@ extern "C" #include #include #include -#include #define LOG_VERSION "0.1.0" struct log_handle; -struct log_event -{ - int level; - va_list ap; - const char *fmt; - const char *module; - char *run_log_path; - char *cur_log_file; - void *user_data; - pthread_mutex_t mutex; -}; - -typedef void (*log_func_t)(struct log_event *ev); -typedef void (*log_lock_func_t)(int lock, void *udata); - enum { LOG_TRACE, LOG_DEBUG, LOG_INFO, LOG_WARN, LOG_ERROR, LOG_FATAL }; #define log_debug(handle, module, fmt, ...) log_print(handle, LOG_DEBUG, module, fmt, ##__VA_ARGS__) @@ -48,11 +32,6 @@ enum { LOG_TRACE, LOG_DEBUG, LOG_INFO, LOG_WARN, LOG_ERROR, LOG_FATAL }; #define log_fatal(handle, module, fmt, ...) log_print(handle, LOG_FATAL, module, fmt, ##__VA_ARGS__) void log_print(struct log_handle *, int level, const char *module, const char *fmt, ...); - -int log_options_add_file_fp(struct log_handle *, int level); -int log_options_add_fp(struct log_handle *, FILE *fp, int level); - -void log_options_set_lock_fn(struct log_handle *, log_lock_func_t fn, void *user_data); void log_options_set_enable(struct log_handle *, int enable); void log_options_set_level(struct log_handle *, int level); diff --git a/include/maat.h b/include/maat.h index c1a83f3..f8cdf38 100644 --- a/include/maat.h +++ b/include/maat.h @@ -48,6 +48,12 @@ enum maat_scan_status { MAAT_SCAN_HIT //scan hit compile }; +enum maat_update_type { + MAAT_UPDATE_TYPE_INVALID = 0, + MAAT_UPDATE_TYPE_FULL, + MAAT_UPDATE_TYPE_INC +}; + struct ip_addr { int ip_type; //4: IPv4, 6: IPv6 union { @@ -70,6 +76,7 @@ struct ipv6_tuple { unsigned short dport; /* network order */ }; +/* update_type: MAAT_UPDATE_TYPE_FULL or MAAT_UPDATE_TYPE_INC */ typedef void maat_start_callback_t(int update_type, void *u_param); typedef void maat_update_callback_t(int table_id, const char *table_line, void *u_para); typedef void maat_finish_callback_t(void *u_para); @@ -89,8 +96,9 @@ int maat_options_set_accept_tags(struct maat_options *opts, const char *accept_t int maat_options_set_rule_effect_interval_ms(struct maat_options *opts, int interval_ms); int maat_options_set_rule_update_checking_interval_ms(struct maat_options *opts, int interval_ms); int maat_options_set_gc_timeout_ms(struct maat_options *opts, int interval_ms); -int maat_options_set_instance_name(struct maat_options *opts, const char *instance_name, size_t name_len); +int maat_options_set_instance_name(struct maat_options *opts, const char *instance_name); int maat_options_set_deferred_load_on(struct maat_options *opts); +int maat_options_set_foreign_cont_dir(struct maat_options *opts, const char *dir); int maat_options_set_iris(struct maat_options *opts, const char *full_directory, const char *increment_directory); @@ -196,6 +204,12 @@ int maat_scan_stream(struct maat_stream **stream, const char *data, int data_len void maat_scan_stream_close(struct maat_stream **stream); /* maat state API */ +struct maat_state *maat_state_new(struct maat *instance, int thread_id); + +void maat_state_reset(struct maat_state *state); + +void maat_state_free(struct maat_state **state); + int maat_state_set_scan_district(struct maat *instance, struct maat_state **state, const char *district, size_t district_len); @@ -210,8 +224,6 @@ int maat_state_get_hit_paths(struct maat *instance, struct maat_state **state, int maat_state_get_hit_objects(struct maat *instance, struct maat_state **state, struct maat_hit_object *objs, size_t n_obj); -void maat_state_free(struct maat_state **state); - /* return hit object compile_id */ int maat_hit_object_compile_id(struct maat *instance, struct maat_hit_object *obj); diff --git a/scanner/adapter_hs/adapter_hs.cpp b/scanner/adapter_hs/adapter_hs.cpp index d863995..4ad7b90 100644 --- a/scanner/adapter_hs/adapter_hs.cpp +++ b/scanner/adapter_hs/adapter_hs.cpp @@ -62,6 +62,7 @@ struct adapter_hs { size_t n_expr; size_t n_patterns; struct adapter_hs_runtime *hs_rt; + struct hs_tag *tag_map; }; struct adapter_hs_stream { @@ -97,9 +98,13 @@ struct pattern_attribute { }; struct hs_tag { + char *key; + size_t key_len; + size_t n_pat_attr; struct pattern_attribute *pat_attr; void *user_tag; + UT_hash_handle hh; }; static int adpt_hs_alloc_scratch(struct adapter_hs_runtime *hs_rt, size_t n_worker_thread, @@ -222,10 +227,40 @@ void adpt_hs_compile_data_free(struct adpt_hs_compile_data *hs_cd, size_t n_patt FREE(hs_cd); } +struct hs_tag *hs_tag_new(long long expr_id, size_t n_pattern) +{ + struct hs_tag *tag = ALLOC(struct hs_tag, 1); + + tag->key = ALLOC(char, sizeof(long long)); + memcpy(tag->key, (char *)&expr_id, sizeof(long long)); + tag->key_len = sizeof(long long); + tag->pat_attr = ALLOC(struct pattern_attribute, n_pattern); + tag->n_pat_attr = n_pattern; + + return tag; +} + +void hs_tag_free(struct hs_tag *tag) +{ + if (NULL == tag) { + return; + } + + if (tag->key != NULL) { + FREE(tag->key); + } + + if (tag->pat_attr != NULL) { + FREE(tag->pat_attr); + } + + FREE(tag); +} + struct adapter_hs *adapter_hs_initialize(enum hs_scan_mode scan_mode, enum hs_pattern_type pattern_type, size_t n_worker_thread, - and_expr_t *exprs, size_t n_expr, + struct hs_expr *exprs, size_t n_expr, struct log_handle *logger) { if ((scan_mode != HS_SCAN_MODE_BLOCK && scan_mode != HS_SCAN_MODE_STREAM) || @@ -268,13 +303,13 @@ struct adapter_hs *adapter_hs_initialize(enum hs_scan_mode scan_mode, compile_data = adpt_hs_compile_data_new(pattern_num); uint32_t pattern_index = 0; - struct bool_expr *bool_exprs = ALLOC(struct bool_expr, n_expr); + struct adapter_hs *hs_instance = ALLOC(struct adapter_hs, 1); + hs_instance->tag_map = NULL; + struct bool_expr *bool_exprs = ALLOC(struct bool_expr, n_expr); /* populate adpt_hs_compile_data and bool_expr */ for (size_t i = 0; i < n_expr; i++) { - struct hs_tag *hs_tag = ALLOC(struct hs_tag, 1); - hs_tag->pat_attr = ALLOC(struct pattern_attribute, exprs[i].n_patterns); - hs_tag->n_pat_attr = exprs[i].n_patterns; + struct hs_tag *hs_tag = hs_tag_new(exprs[i].expr_id, exprs[i].n_patterns); hs_tag->user_tag = exprs[i].user_tag; for (size_t j = 0; j < exprs[i].n_patterns; j++) { @@ -306,13 +341,13 @@ struct adapter_hs *adapter_hs_initialize(enum hs_scan_mode scan_mode, bool_exprs[i].expr_id = exprs[i].expr_id; bool_exprs[i].item_num = exprs[i].n_patterns; bool_exprs[i].user_tag = hs_tag; + HASH_ADD_KEYPTR(hh, hs_instance->tag_map, hs_tag->key, hs_tag->key_len, hs_tag); } compile_data->n_patterns = pattern_index; int ret = -1; size_t mem_size = 0; - struct adapter_hs *hs_instance = ALLOC(struct adapter_hs, 1); - + hs_instance->n_worker_thread = n_worker_thread; hs_instance->n_patterns = pattern_index; hs_instance->n_expr = n_expr; @@ -325,6 +360,7 @@ struct adapter_hs *adapter_hs_initialize(enum hs_scan_mode scan_mode, // printf("item[%zu] item_id: %llu\n", j, exprs[i].items[j].item_id); // } // } + /* create bool matcher */ hs_instance->hs_rt->bm = bool_matcher_new(bool_exprs, n_expr, &mem_size); if (hs_instance->hs_rt->bm != NULL) { @@ -334,7 +370,10 @@ struct adapter_hs *adapter_hs_initialize(enum hs_scan_mode scan_mode, } else { log_error(logger, MODULE_ADAPTER_HS, "[%s:%d] Adapter_hs module: build bool matcher failed", __FUNCTION__, __LINE__); - goto error; + adpt_hs_compile_data_free(compile_data, pattern_index); + FREE(bool_exprs); + adapter_hs_destroy(hs_instance); + return NULL; } FREE(bool_exprs); @@ -344,17 +383,15 @@ struct adapter_hs *adapter_hs_initialize(enum hs_scan_mode scan_mode, goto error; } - if (compile_data != NULL) { - adpt_hs_compile_data_free(compile_data, pattern_index); - } - ret = adpt_hs_alloc_scratch(hs_instance->hs_rt, n_worker_thread, pattern_type, logger); if (ret < 0) { goto error; } + adpt_hs_compile_data_free(compile_data, pattern_index); return hs_instance; error: + adpt_hs_compile_data_free(compile_data, pattern_index); adapter_hs_destroy(hs_instance); return NULL; @@ -391,6 +428,14 @@ void adapter_hs_destroy(struct adapter_hs *hs_instance) FREE(hs_instance->hs_rt); } + if (hs_instance->tag_map != NULL) { + struct hs_tag *tag = NULL, *tmp_tag = NULL; + HASH_ITER(hh, hs_instance->tag_map, tag, tmp_tag) { + HASH_DEL(hs_instance->tag_map, tag); + hs_tag_free(tag); + } + } + FREE(hs_instance); } @@ -662,7 +707,7 @@ int adapter_hs_scan_stream(struct adapter_hs_stream *hs_stream, const char *data bool_matcher_results = ALLOC(struct bool_expr_match, hs_stream->n_expr); int bool_matcher_ret = bool_matcher_match(hs_stream->hs_rt->bm, items, pattern_set_size, - bool_matcher_results, hs_stream->n_expr); + bool_matcher_results, hs_stream->n_expr); if (bool_matcher_ret < 0) { ret = -1; goto next; diff --git a/scanner/adapter_hs/adapter_hs.h b/scanner/adapter_hs/adapter_hs.h index 5b622f5..db695ec 100644 --- a/scanner/adapter_hs/adapter_hs.h +++ b/scanner/adapter_hs/adapter_hs.h @@ -58,7 +58,7 @@ struct hs_scan_result { void *user_tag; }; -typedef struct { +struct hs_pattern { enum hs_case_sensitive case_sensitive; enum hs_match_mode match_mode; int is_hexbin; /* 1(yes) 0(no) */ @@ -76,15 +76,15 @@ typedef struct { char *pat; /* pattern length */ size_t pat_len; -} scan_pattern_t; +}; /* logic AND expression, such as (pattern1 & pattern2) */ -typedef struct { - uint32_t expr_id; +struct hs_expr { + long long expr_id; size_t n_patterns; - scan_pattern_t patterns[MAX_EXPR_PATTERN_NUM]; + struct hs_pattern patterns[MAX_EXPR_PATTERN_NUM]; void *user_tag; -} and_expr_t; +}; /** * @brief initialize adapter_hs instance @@ -99,7 +99,7 @@ typedef struct { struct adapter_hs *adapter_hs_initialize(enum hs_scan_mode scan_mode, enum hs_pattern_type pattern_type, size_t n_worker_thread, - and_expr_t *exprs, size_t n_expr, + struct hs_expr *exprs, size_t n_expr, struct log_handle *logger); /** diff --git a/src/inc_internal/maat_command.h b/src/inc_internal/maat_command.h index ddf8503..4c244b5 100644 --- a/src/inc_internal/maat_command.h +++ b/src/inc_internal/maat_command.h @@ -24,8 +24,7 @@ enum maat_operation { MAAT_OP_RENEW_TIMEOUT //Rule expire time is changed to now+cmd->expire_after }; -struct maat_cmd_line -{ +struct maat_cmd_line { const char *table_name; const char *table_line; long long rule_id; // for MAAT_OP_DEL, only rule_id and table_name are necessary. @@ -43,6 +42,15 @@ int maat_cmd_set_line(struct maat *maat_instance, const struct maat_cmd_line *li long long maat_cmd_incrby(struct maat *maat_instance, const char *key, int increment); +long long maat_cmd_get_config_version(struct maat *maat_instance); + +/* True(1), False(0) */ +int maat_cmd_config_is_updating(struct maat *maat_instance); + +char *maat_cmd_str_escape(char *dst, int size, const char *src); + +int maat_cmd_flushDB(struct maat *maat_instance); + #ifdef __cplusplus } #endif diff --git a/src/inc_internal/maat_common.h b/src/inc_internal/maat_common.h index 992c41b..e7dfd30 100644 --- a/src/inc_internal/maat_common.h +++ b/src/inc_internal/maat_common.h @@ -23,10 +23,9 @@ extern "C" struct maat_options { char instance_name[NAME_MAX]; + char foreign_cont_dir[NAME_MAX]; size_t nr_worker_threads; - char *accept_tags; - int rule_effect_interval_ms; int rule_update_checking_interval_ms; int gc_timeout_ms; diff --git a/src/inc_internal/maat_compile.h b/src/inc_internal/maat_compile.h index 93d1432..50a8c3b 100644 --- a/src/inc_internal/maat_compile.h +++ b/src/inc_internal/maat_compile.h @@ -17,6 +17,7 @@ extern "C" #endif #include "cJSON/cJSON.h" +#include "rcu_hash.h" #include "maat.h" #include "maat_kv.h" #include "maat_rule.h" @@ -49,6 +50,8 @@ void *compile_runtime_new(void *compile_schema, int max_thread_num, struct log_handle *logger); void compile_runtime_free(void *compile_runtime); +void compile_runtime_init(void *compile_runtime, struct maat_runtime *maat_rt); + int compile_runtime_update(void *compile_runtime, void *compile_schema, const char *line, int valid_column); int compile_runtime_commit(void *compile_runtime, const char *table_name); @@ -82,7 +85,7 @@ struct maat_compile_state; struct maat_compile_state *maat_compile_state_new(int thread_id); void maat_compile_state_free(struct maat_compile_state *compile_state); -int maat_compile_state_update(struct maat_item *item_hash, int vtable_id, +int maat_compile_state_update(struct rcu_hash_table *item_htable, int vtable_id, long long *hit_item_ids, size_t hit_item_cnt, size_t *n_hit_group_id, struct maat_state *state); diff --git a/src/inc_internal/maat_ex_data.h b/src/inc_internal/maat_ex_data.h index 3d5bf44..b64a0d2 100644 --- a/src/inc_internal/maat_ex_data.h +++ b/src/inc_internal/maat_ex_data.h @@ -18,18 +18,13 @@ extern "C" #include "maat.h" #include "rcu_hash.h" +#include "maat_garbage_collection.h" -struct ex_data_container { +struct ex_container { void *ex_data; void *custom_data; }; -struct ex_container_ctx { - int table_id; - void (*custom_data_free)(void *custom_data); - struct ex_data_schema *ex_schema; -}; - struct ex_data_schema { maat_ex_new_func_t *new_func; maat_ex_free_func_t *free_func; @@ -38,12 +33,18 @@ struct ex_data_schema { void *argp; }; +struct ex_container_schema { + int table_id; + void (*custom_data_free)(void *custom_data); + struct ex_data_schema *ex_schema; +}; + struct ex_data_runtime; /* ex_data_runtime API */ struct ex_data_runtime * ex_data_runtime_new(int table_id, rcu_hash_data_free_fn *data_free_fn, - struct log_handle *logger); + struct maat_garbage_bin *garbage_bin, struct log_handle *logger); void ex_data_runtime_free(struct ex_data_runtime *ex_data_rt); void ex_data_runtime_commit(struct ex_data_runtime *ex_data_rt); @@ -67,42 +68,45 @@ void ex_data_runtime_set_schema(struct ex_data_runtime *ex_data_rt, struct ex_data_schema *schema); /* set user_ctx API */ -void ex_data_runtime_set_ex_container_ctx(struct ex_data_runtime *ex_data_rt, - struct ex_container_ctx *container_ctx); +void ex_data_runtime_set_ex_container_schema(struct ex_data_runtime *ex_data_rt, + struct ex_container_schema *container_schema); -struct ex_container_ctx * -ex_data_runtime_get_ex_container_ctx(struct ex_data_runtime *ex_data_rt); +struct ex_container_schema * +ex_data_runtime_get_ex_container_schema(struct ex_data_runtime *ex_data_rt); -struct ex_data_container *ex_data_container_new(void *ex_data, void *custom_data); -void ex_data_container_free(void *ctx, void *data); +struct ex_container *ex_container_new(void *ex_data, void *custom_data); +void ex_container_free(void *ctx, void *data); /* ex_data_runtime ex data API */ -void *ex_data_runtime_row2ex_data(struct ex_data_runtime *ex_data_rt, const char *row, - const char *key, size_t key_len); +void *ex_data_runtime_row2ex_data(struct ex_data_runtime *ex_data_rt, + const char *row, const char *key, + size_t key_len); int ex_data_runtime_add_ex_container(struct ex_data_runtime *ex_data_rt, const char *key, size_t key_len, - struct ex_data_container *ex_container); + struct ex_container *ex_container); int ex_data_runtime_del_ex_container(struct ex_data_runtime *ex_data_rt, const char *key, size_t key_len); -size_t ex_data_runtime_list_updating_ex_container(struct ex_data_runtime *ex_data_rt, - struct ex_data_container ***ex_container); +size_t ex_data_runtime_list_ex_container(struct ex_data_runtime *ex_data_rt, + struct ex_container ***ex_container); + +size_t ex_data_runtime_ex_container_count(struct ex_data_runtime *ex_data_rt); + +int ex_data_runtime_is_updating(struct ex_data_runtime *ex_data_rt); void *ex_data_runtime_get_ex_data_by_key(struct ex_data_runtime *ex_data_rt, const char *key, size_t key_len); void *ex_data_runtime_get_ex_data_by_container(struct ex_data_runtime *ex_data_rt, - struct ex_data_container *ex_container); + struct ex_container *ex_container); void *ex_data_runtime_get_custom_data(struct ex_data_runtime *ex_data_rt, const char *key, size_t key_len); -size_t ex_data_runtime_ex_container_count(struct ex_data_runtime *ex_data_rt); - #ifdef __cplusplus } #endif -#endif \ No newline at end of file +#endif diff --git a/src/inc_internal/maat_ip.h b/src/inc_internal/maat_ip.h index f931f13..e4e7968 100644 --- a/src/inc_internal/maat_ip.h +++ b/src/inc_internal/maat_ip.h @@ -35,8 +35,6 @@ int ip_runtime_update(void *ip_runtime, void *ip_schema, const char *line, int valid_column); int ip_runtime_commit(void *ip_runtime, const char *table_name); -struct ex_data_runtime *ip_runtime_get_ex_data_rt(struct ip_runtime *ip_rt); - /* ip runtime scan API */ int ip_runtime_scan(struct ip_runtime *ip_rt, int thread_id, int ip_type, uint8_t *ip_addr, int vtable_id, struct maat_state *state); diff --git a/src/inc_internal/maat_kv.h b/src/inc_internal/maat_kv.h index 37217ee..3441225 100644 --- a/src/inc_internal/maat_kv.h +++ b/src/inc_internal/maat_kv.h @@ -24,11 +24,13 @@ struct maat_kv_store *maat_kv_store_new(void); void maat_kv_store_free(struct maat_kv_store *store); -int maat_kv_register(struct maat_kv_store *store, const char *key, int value); +int maat_kv_register(struct maat_kv_store *store, const char *key, long long value); -int maat_kv_read(struct maat_kv_store *store, const char *key, int *value); +int maat_kv_read(struct maat_kv_store *store, const char *key, long long *value); -int maat_kv_read_unNull(struct maat_kv_store *store, const char *key, size_t key_sz, int *value); +int maat_kv_read_unNull(struct maat_kv_store *store, const char *key, size_t key_sz, long long *value); + +int maat_kv_write(struct maat_kv_store *store, const char *key, long long value); struct maat_kv_store *maat_kv_store_duplicate(struct maat_kv_store *store); diff --git a/src/inc_internal/maat_rule.h b/src/inc_internal/maat_rule.h index b5c340a..0caba39 100644 --- a/src/inc_internal/maat_rule.h +++ b/src/inc_internal/maat_rule.h @@ -37,14 +37,14 @@ extern "C" #define MAX_COMPILE_TABLE_NUM 16 #define MAX_PHYSICAL_TABLE_NUM 16 -#define MAAT_UPDATE_TYPE_NONE 0 -#define MAAT_UPDATE_TYPE_FULL 1 -#define MAAT_UPDATE_TYPE_INC 2 +#define DISTRICT_ANY -1 +#define DISTRICT_UNKNOWN -2 -#define DISTRICT_ANY -1 -#define DISTRICT_UNKNOWN -2 +#define MAX_DISTRICT_STR 128 +#define INVALID_VERSION -1 -#define MAX_DISTRICT_STR 128 +#define mr_region_id_var "SEQUENCE_REGION" +#define mr_group_id_var "SEQUENCE_GROUP" enum last_scan_flag { LAST_SCAN_UNSET, @@ -63,7 +63,7 @@ struct maat_item_inner { long long magic_num; long long item_id; long long group_id; - int district_id; + long long district_id; int expr_id_cnt; int expr_id_lb; //low boundary int expr_id_ub; //up boundary @@ -72,8 +72,9 @@ struct maat_item_inner { struct maat_item { long long item_id; long long group_id; - UT_hash_handle hh; + void *user_data; + void (*user_data_free)(void *data); }; #define COMPILE_RULE_MAGIC 0x1a2b3c4d @@ -105,6 +106,7 @@ struct maat_runtime { size_t max_table_num; uint32_t rule_num; + struct maat_kv_store *sequence_map; struct maat_garbage_bin *ref_garbage_bin; struct log_handle *logger; @@ -216,6 +218,9 @@ struct maat { long long load_specific_version; //Default: Load the Latest. Only valid in redis mode, and maybe failed for too old char foreign_cont_dir[NAME_MAX]; + /* internal state */ + long long new_version; + /* statistics */ long long line_cmd_acc_num; @@ -235,7 +240,7 @@ struct maat_state { char compile_tables[MAX_COMPILE_TABLE_NUM][NAME_MAX]; //caller can select compile table to scan unsigned char is_set_district; unsigned char is_last_scan; - int district_id; //-1: Any District; -2: Unkonwn District; + long long district_id; //-1: Any District; -2: Unkonwn District; int scan_cnt; struct maat_compile_state *compile_state; }; @@ -246,11 +251,11 @@ size_t parse_accept_tag(const char *value, struct rule_tag **result, struct log_ int compare_accept_tag(const char *value, const struct rule_tag *accept_tags, size_t n_accept_tag); -struct maat_item *maat_item_new(long long item_id, long long group_id, void *user_data); +struct maat_item *maat_item_new(long long item_id, long long group_id, void *user_data, void (*user_data_free)(void *)); -void maat_item_free(struct maat_item *item, void (* item_user_data_free)(void *)); +void maat_item_free(void *user_ctx, void *maat_item); -struct maat_item_inner *maat_item_inner_new(long long group_id, long long item_id, int district_id); +struct maat_item_inner *maat_item_inner_new(long long group_id, long long item_id, long long district_id); void maat_item_inner_free(void *item_inner); @@ -262,6 +267,8 @@ void maat_finish_cb(void *u_para); void *rule_monitor_loop(void *arg); +long long maat_runtime_get_sequence(struct maat_runtime *maat_rt, const char *key); + void maat_read_full_config(struct maat *maat_instance); /* maat command API for internal */ diff --git a/src/inc_internal/maat_table.h b/src/inc_internal/maat_table.h index cd132c0..e0a0867 100644 --- a/src/inc_internal/maat_table.h +++ b/src/inc_internal/maat_table.h @@ -68,8 +68,8 @@ size_t table_manager_accept_tags_count(struct table_manager *tbl_mgr); int table_manager_accept_tags_match(struct table_manager *tbl_mgr, const char *tags); int table_manager_set_scan_district(struct table_manager *tbl_mgr, const char *district_str, - size_t district_str_len, int *district_id); -int table_manager_get_district_id(struct table_manager *tbl_mgr, const char *district); + size_t district_str_len, long long *district_id); +long long table_manager_get_district_id(struct table_manager *tbl_mgr, const char *district); void *table_manager_get_schema(struct table_manager *tbl_mgr, int table_id); void *table_manager_get_runtime(struct table_manager *tbl_mgr, int table_id); diff --git a/src/inc_internal/rcu_hash.h b/src/inc_internal/rcu_hash.h index 8267a19..3908055 100644 --- a/src/inc_internal/rcu_hash.h +++ b/src/inc_internal/rcu_hash.h @@ -50,8 +50,23 @@ void rcu_hash_del(struct rcu_hash_table *htable, const char *key, size_t key_len */ void *rcu_hash_find(struct rcu_hash_table *htable, const char *key, size_t key_len); +void *rcu_hash_updating_find(struct rcu_hash_table *htable, const char *key, size_t key_len); + +/** + * @brief list all effective nodes + * + * @retval the number of effective nodes +*/ +size_t rcu_hash_list(struct rcu_hash_table *htable, void ***data_array); + size_t rcu_hash_count(struct rcu_hash_table *htable); -size_t rcu_hash_updating_count(struct rcu_hash_table *htable); + +/** + * @brief if rcu hash is updating + * + * @retval True(1) False(0) +*/ +int rcu_hash_is_updating(struct rcu_hash_table *htable); /** * @brief make add/del effective @@ -60,8 +75,6 @@ void rcu_hash_commit(struct rcu_hash_table *htable); size_t rcu_hash_garbage_queue_len(struct rcu_hash_table *htable); -size_t rcu_hash_list_updating_data(struct rcu_hash_table *htable, void ***data_array); - #ifdef __cplusplus } #endif diff --git a/src/json2iris.c b/src/json2iris.c index b1720c2..725c47e 100644 --- a/src/json2iris.c +++ b/src/json2iris.c @@ -25,10 +25,6 @@ #define MODULE_JSON2IRIS module_name_str("maat.json2iris") #define MAX_COLUMN_NUM 32 - -#define mr_region_id_var "SEQUENCE_REGION" -#define mr_group_id_var "SEQUENCE_GROUP" - #define MAX_PATH_LINE 512 const int json_version = 1; @@ -104,7 +100,7 @@ struct iris_table *query_table_info(struct iris_description *p_iris, strlen(table_name))); snprintf(table_info->table_path, sizeof(table_info->table_path), "%s/%s.local", p_iris->tmp_iris_dir, table_info->table_name); - HASH_ADD_KEYPTR(hh, p_iris->iris_table_map, table_name, strlen(table_name), table_info); + HASH_ADD_KEYPTR(hh, p_iris->iris_table_map, table_info->table_name, strlen(table_name), table_info); } return table_info; @@ -315,7 +311,7 @@ static struct group_info *group_info_add_unsafe(struct iris_description *p_iris, group_info = ALLOC(struct group_info, 1); group_info->group_id = get_group_seq(p_iris); strncpy(group_info->group_name, group_name, sizeof(group_info->group_name)); - HASH_ADD_KEYPTR(hh, p_iris->group_name_map, group_name, strlen(group_name), group_info); + HASH_ADD_KEYPTR(hh, p_iris->group_name_map, group_info->group_name, strlen(group_name), group_info); } return group_info; } @@ -364,7 +360,7 @@ int direct_write_rule(cJSON *json, struct maat_kv_store *str2int, } if (1 == cmd[i].str2int_flag) { - int int_value = 0; + long long int_value = 0; char *p = item->valuestring; ret = maat_kv_read(str2int, p, &int_value); if (ret < 0) { @@ -376,7 +372,7 @@ int direct_write_rule(cJSON *json, struct maat_kv_store *str2int, goto error_out; } cmd[i].json_value = (char *)malloc(21);/* 2^64+1 can be represented in 21 chars. */ - snprintf(cmd[i].json_value, 21, "%d", int_value); + snprintf(cmd[i].json_value, 21, "%d", (int)int_value); } else { switch (item->type) { case cJSON_Number: @@ -667,7 +663,7 @@ int write_region_rule(cJSON *region_json, int compile_id, int group_id, const char *table_type_str = item->valuestring; enum table_type table_type = TABLE_TYPE_EXPR; - int ret = maat_kv_read(p_iris->str2int_map, table_type_str, (int*)&(table_type)); + int ret = maat_kv_read(p_iris->str2int_map, table_type_str, (long long *)&(table_type)); if (ret != 1) { log_error(logger, MODULE_JSON2IRIS, "[%s:%d] compile rule %d table name %s's table_type %s invalid", diff --git a/src/maat_api.c b/src/maat_api.c index d289750..064babd 100644 --- a/src/maat_api.c +++ b/src/maat_api.c @@ -128,58 +128,90 @@ void maat_options_free(struct maat_options *opts) int maat_options_set_caller_thread_number(struct maat_options *opts, size_t n_thread) { + if (NULL == opts) { + return -1; + } + opts->nr_worker_threads = n_thread; - return 0; } int maat_options_set_accept_tags(struct maat_options *opts, const char *accept_tags) { - opts->accept_tags = maat_strdup(accept_tags); + if (NULL == opts || NULL == accept_tags) { + return -1; + } + opts->accept_tags = maat_strdup(accept_tags); return 0; } int maat_options_set_rule_effect_interval_ms(struct maat_options *opts, int interval_ms) { + if (NULL == opts || interval_ms < 0) { + return -1; + } + opts->rule_effect_interval_ms = interval_ms; - return 0; } int maat_options_set_rule_update_checking_interval_ms(struct maat_options *opts, int interval_ms) { - opts->rule_update_checking_interval_ms = interval_ms; + if (NULL == opts || interval_ms < 0) { + return -1; + } + opts->rule_update_checking_interval_ms = interval_ms; return 0; } int maat_options_set_gc_timeout_ms(struct maat_options *opts, int interval_ms) { - opts->gc_timeout_ms = interval_ms; + if (NULL == opts || interval_ms < 0) { + return -1; + } + opts->gc_timeout_ms = interval_ms; return 0; } -int maat_options_set_instance_name(struct maat_options *opts, const char *instance_name, - size_t name_len) +int maat_options_set_instance_name(struct maat_options *opts, const char *instance_name) { - memcpy(opts->instance_name, instance_name, name_len); + if (NULL == opts || NULL == instance_name || + strlen(instance_name) >= NAME_MAX) { + return -1; + } + memcpy(opts->instance_name, instance_name, strlen(instance_name)); return 0; } int maat_options_set_deferred_load_on(struct maat_options *opts) { - opts->deferred_load_on = 1; + if (NULL == opts) { + return -1; + } + opts->deferred_load_on = 1; + return 0; +} + +int maat_options_set_foreign_cont_dir(struct maat_options *opts, const char *dir) +{ + if (NULL == opts || strlen(dir) >= NAME_MAX) { + return -1; + } + + memcpy(opts->foreign_cont_dir, dir, strlen(dir)); return 0; } int maat_options_set_iris(struct maat_options *opts, const char *full_directory, const char *increment_directory) { - if (strlen(full_directory) >= NAME_MAX || strlen(increment_directory) >= NAME_MAX) { + if (NULL == opts || strlen(full_directory) >= NAME_MAX || + strlen(increment_directory) >= NAME_MAX) { return -1; } @@ -345,6 +377,7 @@ struct maat *maat_new(struct maat_options *opts, const char *table_info_path) maat_instance->rule_update_checking_interval_ms = opts->rule_update_checking_interval_ms; maat_instance->gc_timeout_ms = opts->gc_timeout_ms; maat_instance->deferred_load = opts->deferred_load_on; + memcpy(maat_instance->foreign_cont_dir, opts->foreign_cont_dir, strlen(opts->foreign_cont_dir)); garbage_gc_timeout_s = (maat_instance->rule_effect_interval_ms / 1000) + (maat_instance->gc_timeout_ms / 1000); maat_instance->garbage_bin = maat_garbage_bin_new(garbage_gc_timeout_s); @@ -593,10 +626,10 @@ void generic_plugin_runtime_commit_ex_schema(void *runtime, void *schema, int ta } ex_data_runtime_set_schema(ex_data_rt, ex_data_schema); - struct ex_container_ctx *ex_ctx = ALLOC(struct ex_container_ctx, 1); - ex_ctx->table_id = table_id; - ex_ctx->ex_schema = ex_data_schema; - ex_data_runtime_set_ex_container_ctx(ex_data_rt, ex_ctx); + struct ex_container_schema *container_schema = ALLOC(struct ex_container_schema, 1); + container_schema->table_id = table_id; + container_schema->ex_schema = ex_data_schema; + ex_data_runtime_set_ex_container_schema(ex_data_rt, container_schema); size_t n_cached_row = ex_data_runtime_cached_row_count(ex_data_rt); for (size_t i = 0; i < n_cached_row; i++) { @@ -897,7 +930,7 @@ struct maat_state *grab_state(struct maat_state **state, struct maat *maat_insta alignment_int64_array_add(maat_instance->outer_state_cnt, thread_id, 1); } - if (NULL == mid->compile_state) { + if ((thread_id >= 0) && (NULL == mid->compile_state)) { mid->compile_state = maat_compile_state_new(thread_id); alignment_int64_array_add(maat_instance->compile_state_cnt, thread_id, 1); } @@ -1720,6 +1753,39 @@ void maat_scan_stream_close(struct maat_stream **maat_stream) FREE(stream); } +struct maat_state *maat_state_new(struct maat *maat_instance, int thread_id) +{ + return NULL; +} + +void maat_state_reset(struct maat_state *state) +{ + +} + +void maat_state_free(struct maat_state **state) +{ + struct maat_state *mid = NULL; + if (NULL == *state) { + return; + } + + mid = *state; + if (mid->thread_id >= 0) { + alignment_int64_array_add(mid->maat_instance->outer_state_cnt, mid->thread_id, -1); + } + + if (mid->compile_state != NULL) { + maat_compile_state_free(mid->compile_state); + mid->compile_state = NULL; + alignment_int64_array_add(mid->maat_instance->compile_state_cnt, mid->thread_id, -1); + } + + mid->maat_instance = NULL; + free(mid); + *state = NULL; +} + int maat_state_set_scan_district(struct maat *maat_instance, struct maat_state **state, const char *district, @@ -1834,29 +1900,6 @@ int maat_state_get_hit_objects(struct maat *instance, struct maat_state **state, return 0; } -void maat_state_free(struct maat_state **state) -{ - struct maat_state *mid = NULL; - if (NULL == *state) { - return; - } - - mid = *state; - if (mid->thread_id >= 0) { - alignment_int64_array_add(mid->maat_instance->outer_state_cnt, mid->thread_id, -1); - } - - if (mid->compile_state != NULL) { - maat_compile_state_free(mid->compile_state); - mid->compile_state = NULL; - alignment_int64_array_add(mid->maat_instance->compile_state_cnt, mid->thread_id, -1); - } - - mid->maat_instance = NULL; - free(mid); - *state = NULL; -} - int maat_hit_object_compile_id(struct maat *instance, struct maat_hit_object *obj) { return 0; diff --git a/src/maat_bool_plugin.c b/src/maat_bool_plugin.c index 852cfe1..c6ae600 100644 --- a/src/maat_bool_plugin.c +++ b/src/maat_bool_plugin.c @@ -104,7 +104,7 @@ void bool_plugin_schema_free(void *bool_plugin_schema) schema->ex_schema = NULL; } - FREE(schema); + free(schema); } /* ip plugin table ex data API */ @@ -182,8 +182,9 @@ void *bool_plugin_runtime_new(void *bool_plugin_schema, int max_thread_num, struct bool_plugin_schema *schema = (struct bool_plugin_schema *)bool_plugin_schema; struct bool_plugin_runtime *bool_plugin_rt = ALLOC(struct bool_plugin_runtime, 1); - bool_plugin_rt->ex_data_rt = ex_data_runtime_new(schema->table_id, ex_data_container_free, - logger); + bool_plugin_rt->ex_data_rt = ex_data_runtime_new(schema->table_id, + ex_container_free, + garbage_bin, logger); bool_plugin_rt->ref_garbage_bin = garbage_bin; bool_plugin_rt->logger = logger; @@ -226,7 +227,7 @@ int bool_plugin_runtime_update_row(struct bool_plugin_runtime *bool_plugin_rt, } else { // add void *ex_data = ex_data_runtime_row2ex_data(ex_data_rt, row, key, key_len); - struct ex_data_container *ex_container = ex_data_container_new(ex_data, (void *)expr); + struct ex_container *ex_container = ex_container_new(ex_data, (void *)expr); ret = ex_data_runtime_add_ex_container(ex_data_rt, key, key_len, ex_container); if (ret < 0) { return -1; @@ -385,12 +386,6 @@ int bool_plugin_runtime_update(void *bool_plugin_runtime, void *bool_plugin_sche bool_plugin_expr_free(bool_expr); } return -1; - } else { - if (0 == is_valid) { - bool_plugin_rt->rule_num--; - } else { - bool_plugin_rt->rule_num++; - } } } else { //ex_schema not set @@ -407,49 +402,64 @@ int bool_plugin_runtime_commit(void *bool_plugin_runtime, const char *table_name return -1; } - int ret = 0; - struct ex_data_container **ex_container = NULL; struct bool_plugin_runtime *bool_plugin_rt = (struct bool_plugin_runtime *)bool_plugin_runtime; struct ex_data_runtime *ex_data_rt = bool_plugin_rt->ex_data_rt; + if (NULL == ex_data_rt) { + return -1; + } - size_t expr_cnt = ex_data_runtime_list_updating_ex_container(ex_data_rt, &ex_container); - if (0 == expr_cnt) { - FREE(ex_container); + int updating_flag = ex_data_runtime_is_updating(ex_data_rt); + if (0 == updating_flag) { return 0; } - struct bool_expr *exprs = ALLOC(struct bool_expr, expr_cnt); + ex_data_runtime_commit(ex_data_rt); - for (size_t i = 0; i < expr_cnt; i++) { - exprs[i] = *(struct bool_expr *)(struct bool_expr *)ex_container[i]->custom_data; - assert(exprs[i].user_tag == ex_container[i] || NULL == exprs[i].user_tag); - exprs[i].user_tag = ex_container[i]; + struct bool_expr *rules = NULL; + struct ex_container **ex_container = NULL; + size_t rule_cnt = ex_data_runtime_list_ex_container(ex_data_rt, &ex_container); + if (rule_cnt > 0) { + rules = ALLOC(struct bool_expr, rule_cnt); + for (size_t i = 0; i < rule_cnt; i++) { + rules[i] = *(struct bool_expr *)(struct bool_expr *)ex_container[i]->custom_data; + assert(rules[i].user_tag == ex_container[i] || NULL == rules[i].user_tag); + rules[i].user_tag = ex_container[i]; + } } - struct bool_matcher *new_bool_matcher = NULL; - struct bool_matcher *old_bool_matcher = NULL; - size_t mem_used = 0; - log_info(bool_plugin_rt->logger, MODULE_BOOL_PLUGIN, "table[%s] committing %zu bool_plugin rules for rebuilding bool_matcher engine", - table_name, expr_cnt); + table_name, rule_cnt); - new_bool_matcher = bool_matcher_new(exprs, expr_cnt, &mem_used); + int ret = 0; + size_t mem_used = 0; + struct bool_matcher *new_bool_matcher = NULL; + struct bool_matcher *old_bool_matcher = NULL; + + new_bool_matcher = bool_matcher_new(rules, rule_cnt, &mem_used); if (NULL == new_bool_matcher) { log_error(bool_plugin_rt->logger, MODULE_BOOL_PLUGIN, "[%s:%d] table[%s] rebuild bool_matcher engine failed when update %zu bool_plugin rules", - __FUNCTION__, __LINE__, table_name, expr_cnt); + __FUNCTION__, __LINE__, table_name, rule_cnt); ret = -1; } old_bool_matcher = bool_plugin_rt->matcher; bool_plugin_rt->matcher = new_bool_matcher; - maat_garbage_bagging(bool_plugin_rt->ref_garbage_bin, old_bool_matcher, - (void (*)(void*))bool_matcher_free); - ex_data_runtime_commit(ex_data_rt); + if (old_bool_matcher != NULL) { + maat_garbage_bagging(bool_plugin_rt->ref_garbage_bin, old_bool_matcher, + (void (*)(void*))bool_matcher_free); + } - FREE(exprs); - FREE(ex_container); + bool_plugin_rt->rule_num = rule_cnt; + + if (rules != NULL) { + FREE(rules); + } + + if (ex_container != NULL) { + FREE(ex_container); + } return ret; } @@ -485,7 +495,7 @@ int bool_plugin_runtime_get_ex_data(void *bool_plugin_runtime, unsigned long lon int n_result = bool_matcher_match(bool_plugin_rt->matcher, item_ids, n_item, results, n_ex_data); for (int i = 0; i < n_result; i++) { ex_data_array[i] = ex_data_runtime_get_ex_data_by_container(bool_plugin_rt->ex_data_rt, - (struct ex_data_container *)results[i].user_tag); + (struct ex_container *)results[i].user_tag); } return n_result; diff --git a/src/maat_command.c b/src/maat_command.c index 97cbf4c..fbcc1ee 100644 --- a/src/maat_command.c +++ b/src/maat_command.c @@ -84,19 +84,118 @@ redisContext *maat_cmd_connect_redis(const char *redis_ip, int redis_port, return c; } -struct s_rule_array +long long maat_cmd_read_redis_integer(const redisReply *reply) { - int cnt; - int size; - struct serial_rule *array; -}; + switch (reply->type) { + case REDIS_REPLY_INTEGER: + return reply->integer; + break; + case REDIS_REPLY_ARRAY: + assert(reply->element[0]->type == REDIS_REPLY_INTEGER); + return reply->element[0]->integer; + break; + case REDIS_REPLY_STRING: + return atoll(reply->str); + break; + default: + return -1; + break; + } + return 0; +} -void save_serial_rule(void *data, void *user) +int redis_flushDB(redisContext *ctx, int db_index, struct log_handle *logger) { - struct s_rule_array *array = (struct s_rule_array *)user; - int i = array->cnt; - memcpy(&(array->array[i]), data, sizeof(struct serial_rule)); - array->array[i].op = MAAT_OP_ADD; + long long maat_redis_version = 0; + + redisReply *data_reply = maat_cmd_wrap_redis_command(ctx, "WATCH MAAT_VERSION"); + freeReplyObject(data_reply); + data_reply = NULL; + + data_reply = maat_cmd_wrap_redis_command(ctx, "GET MAAT_VERSION"); + if (data_reply->type == REDIS_REPLY_NIL) { + maat_redis_version = 0; + } else { + maat_redis_version = maat_cmd_read_redis_integer(data_reply); + maat_redis_version++; + freeReplyObject(data_reply); + data_reply = NULL; + } + + data_reply = maat_cmd_wrap_redis_command(ctx, "DBSIZE"); + long long dbsize = maat_cmd_read_redis_integer(data_reply); + freeReplyObject(data_reply); + data_reply = NULL; + + data_reply = maat_cmd_wrap_redis_command(ctx, "MULTI"); + freeReplyObject(data_reply); + data_reply = NULL; + + int append_cmd_cnt = 0; + redisAppendCommand(ctx, "FLUSHDB"); + append_cmd_cnt++; + redisAppendCommand(ctx, "SET MAAT_VERSION %lld", maat_redis_version); + append_cmd_cnt++; + redisAppendCommand(ctx, "SET MAAT_PRE_VER %lld", maat_redis_version); + append_cmd_cnt++; + redisAppendCommand(ctx, "SET %s 1", mr_region_id_var); + append_cmd_cnt++; + redisAppendCommand(ctx, "SET %s 1", mr_group_id_var); + append_cmd_cnt++; + redisAppendCommand(ctx, "EXEC"); + append_cmd_cnt++; + + int ret = 0; + int redis_transaction_success = 1; + for (int i = 0; i < append_cmd_cnt; i++) { + ret = maat_cmd_wrap_redis_get_reply(ctx, &data_reply); + if (ret == REDIS_OK) { + if (0 == mr_transaction_success(data_reply)) { + redis_transaction_success = 0; + } + freeReplyObject(data_reply); + data_reply = NULL; + } + } + + if (redis_transaction_success == 1) { + log_info(logger, MODULE_MAAT_COMMAND, + "FlushDB %d, MAAT_VERSION:%llu, DBSize:%llu.", + db_index, (maat_redis_version == 0)?0:(maat_redis_version-1),dbsize); + } + + return redis_transaction_success; +} + +redisContext *get_redis_ctx_for_write(struct maat *maat_instance) +{ + if (NULL == maat_instance->mr_ctx.write_ctx) { + int ret = connect_redis_for_write(&(maat_instance->mr_ctx), + maat_instance->logger); + if(ret!=0) + { + return NULL; + } + } + return maat_instance->mr_ctx.write_ctx; +} + +int maat_cmd_flushDB(struct maat *maat_instance) +{ + int ret = 0; + + redisContext *write_ctx = get_redis_ctx_for_write(maat_instance); + if (NULL == write_ctx) { + return -1; + } + + do { + ret = redis_flushDB(maat_instance->mr_ctx.write_ctx, + maat_instance->mr_ctx.redis_db, + maat_instance->logger); + } while(0 == ret); + + return 0; } void maat_cmd_clear_rule_cache(struct serial_rule *s_rule) @@ -130,19 +229,6 @@ int connect_redis_for_write(struct source_redis_ctx *mr_ctx, } } -redisContext *get_redis_ctx_for_write(struct maat *maat_instance) -{ - if (NULL == maat_instance->mr_ctx.write_ctx) { - int ret = connect_redis_for_write(&(maat_instance->mr_ctx), - maat_instance->logger); - if(ret!=0) - { - return NULL; - } - } - return maat_instance->mr_ctx.write_ctx; -} - void maat_cmd_set_serial_rule(struct serial_rule *rule, enum maat_operation op, long long rule_id, const char *table_name, const char *line, long long timeout) @@ -266,26 +352,6 @@ const char *maat_cmd_find_Nth_column(const char *line, int Nth, int *column_len) return line + start; } -long long maat_cmd_read_redis_integer(const redisReply *reply) -{ - switch (reply->type) { - case REDIS_REPLY_INTEGER: - return reply->integer; - break; - case REDIS_REPLY_ARRAY: - assert(reply->element[0]->type == REDIS_REPLY_INTEGER); - return reply->element[0]->integer; - break; - case REDIS_REPLY_STRING: - return atoll(reply->str); - break; - default: - return -1; - break; - } - return 0; -} - int maat_cmd_wrap_redis_get_reply(redisContext *c, redisReply **reply) { return redisGetReply(c, (void **)reply); @@ -373,4 +439,64 @@ long long maat_cmd_incrby(struct maat *maat_instance, const char *key, int incre data_reply = NULL; return result; +} + +long long maat_cmd_get_config_version(struct maat *maat_instance) +{ + long long new_version = -1; + + if (maat_instance->new_version != INVALID_VERSION) { + new_version = maat_instance->new_version; + } else { + new_version = maat_instance->maat_version; + } + + return new_version; +} + +int maat_cmd_config_is_updating(struct maat *maat_instance) +{ + int ret = 0; + + if (0 == pthread_mutex_trylock(&(maat_instance->background_update_mutex))) { + ret = 0; + pthread_mutex_unlock(&(maat_instance->background_update_mutex)); + } else { + ret = 1; + } + + return ret; +} + +char *maat_cmd_str_escape(char *dst, int size, const char *src) +{ + int i = 0, j = 0; + int len = strlen(src); + + for (i = 0, j = 0; i < len && j < size; i++) { + switch (src[i]) { + case '&': + dst[j] = '\\'; + dst[j+1] = '&'; + j += 2; + break; + case ' ': + dst[j] = '\\'; + dst[j+1] = 'b';//space,0x20; + j += 2; + break; + case '\\': + dst[j] = '\\'; + dst[j+1] = '\\'; + j += 2; + break; + default: + dst[j] = src[i]; + j++; //undo the followed i++ + break; + } + } + dst[j] = '\0'; + + return dst; } \ No newline at end of file diff --git a/src/maat_compile.c b/src/maat_compile.c index bc9b00f..14bf699 100644 --- a/src/maat_compile.c +++ b/src/maat_compile.c @@ -22,7 +22,6 @@ #include "maat_garbage_collection.h" #include "maat_group.h" #include "maat_ex_data.h" -#include "rcu_hash.h" #include "maat_table.h" #define MODULE_COMPILE module_name_str("maat.compile") @@ -71,11 +70,11 @@ struct group2compile_item { struct compile_runtime { struct bool_matcher *bm; struct maat_compile *compile_hash; // - unsigned long long clause_id_generator; - + struct maat_runtime *ref_maat_rt; uint32_t rule_num; pthread_rwlock_t rwlock; /* TODO: replaced with mutex? */ + struct bool_expr_match *expr_match_buff; struct maat_garbage_bin *ref_garbage_bin; struct log_handle *logger; @@ -88,10 +87,10 @@ struct group2compile_runtime { }; struct maat_clause_state { - unsigned long long clause_id; + long long clause_id; char not_flag; char in_use; - UT_array *literal_ids; + UT_array *ut_literal_ids; }; struct maat_literal_id { @@ -100,7 +99,7 @@ struct maat_literal_id { }; struct maat_clause { - unsigned long long clause_id; + long long clause_id; size_t n_literal_id; struct maat_literal_id *literal_ids; UT_hash_handle hh; @@ -510,8 +509,15 @@ void maat_compile_free(struct maat_compile *compile) for (int i = 0; i < MAX_ITEMS_PER_BOOL_EXPR; i++) { clause_state = compile->clause_states + i; - utarray_free(clause_state->literal_ids); - clause_state->literal_ids = NULL; + + // size_t literal_len = utarray_len(clause_state->ut_literal_ids); + // printf("maat_compile_free compile_id:%lld index:%d literal_len:%zu\n", + // compile->compile_id, i, literal_len); + if (clause_state->ut_literal_ids != NULL) { + utarray_free(clause_state->ut_literal_ids); + clause_state->ut_literal_ids = NULL; + } + clause_state->in_use = 0; } compile->magic = 0; @@ -546,15 +552,29 @@ void compile_runtime_free(void *compile_runtime) if (compile_rt->compile_hash != NULL) { maat_compile_hash_free(&(compile_rt->compile_hash)); } - pthread_rwlock_unlock(&compile_rt->rwlock); - + if (compile_rt->expr_match_buff != NULL) { FREE(compile_rt->expr_match_buff); } + pthread_rwlock_unlock(&compile_rt->rwlock); + pthread_rwlock_destroy(&compile_rt->rwlock); FREE(compile_rt); } +void compile_runtime_init(void *compile_runtime, struct maat_runtime *maat_rt) +{ + if (NULL == compile_runtime) { + return; + } + + struct compile_runtime *compile_rt = (struct compile_runtime *)compile_runtime; + + pthread_rwlock_wrlock(&compile_rt->rwlock); + compile_rt->ref_maat_rt = maat_rt; + pthread_rwlock_unlock(&compile_rt->rwlock); +} + void *group2compile_runtime_new(void *g2c_schema, int max_thread_num, struct maat_garbage_bin *garbage_bin, struct log_handle *logger) @@ -695,7 +715,7 @@ struct maat_compile *maat_compile_new(long long compile_id) compile->compile_id = compile_id; for(int i = 0; i < MAX_ITEMS_PER_BOOL_EXPR; i++) { - utarray_new(compile->clause_states[i].literal_ids, &ut_literal_id_icd); + utarray_new(compile->clause_states[i].ut_literal_ids, &ut_literal_id_icd); compile->clause_states[i].in_use=0; } @@ -771,7 +791,7 @@ int maat_compile_hash_remove(struct maat_compile **compile_hash, struct maat_com size_t compile_cnt = HASH_COUNT(*compile_hash); struct maat_compile *compile1 = NULL, *tmp_compile1 = NULL; HASH_ITER (hh, *compile_hash, compile1, tmp_compile1) { - printf(" compile_id:%lu, compile_cnt:%zu\n", + printf(" compile_id:%lld, compile_cnt:%zu\n", compile1->compile_id, compile_cnt); } #endif @@ -787,26 +807,6 @@ struct maat_compile *maat_compile_hash_find(struct maat_compile **compile_hash, return compile; } -size_t maat_compile_in_use_count(struct maat_compile *compile_hash) -{ - struct maat_compile *compile = NULL, *tmp_compile = NULL; - size_t in_use_compile_cnt = 0; - struct maat_clause_state *clause_state = NULL; - - HASH_ITER(hh, compile_hash, compile, tmp_compile) { - //find how much compile whose clause is in_use - for (int i = 0; i < MAX_ITEMS_PER_BOOL_EXPR; i++) { - clause_state = compile->clause_states + i; - if (clause_state->in_use) { - in_use_compile_cnt++; - break; - } - } - } - - return in_use_compile_cnt; -} - int compare_literal_id(const void *pa, const void *pb) { struct maat_literal_id *la = (struct maat_literal_id *)pa; @@ -833,15 +833,15 @@ int maat_compile_clause_add_literal(struct maat_compile *compile, } struct maat_literal_id *tmp = NULL; - tmp = (struct maat_literal_id *)utarray_find(clause_state->literal_ids, + tmp = (struct maat_literal_id *)utarray_find(clause_state->ut_literal_ids, literal_id, compare_literal_id); if (tmp) { assert(tmp->group_id == literal_id->group_id); assert(tmp->vtable_id == literal_id->vtable_id); return -1; } else { - utarray_push_back(clause_state->literal_ids, literal_id); - utarray_sort(clause_state->literal_ids, compare_literal_id); + utarray_push_back(clause_state->ut_literal_ids, literal_id); + utarray_sort(clause_state->ut_literal_ids, compare_literal_id); } return 0; @@ -853,7 +853,7 @@ int maat_compile_clause_remove_literal(struct maat_compile *compile, { struct maat_clause_state* clause_state = compile->clause_states + clause_index; struct maat_literal_id *tmp = NULL; - tmp = (struct maat_literal_id *)utarray_find(clause_state->literal_ids, + tmp = (struct maat_literal_id *)utarray_find(clause_state->ut_literal_ids, literal_id, compare_literal_id); if (tmp) { assert(*(unsigned long long*)tmp == *(unsigned long long*)(literal_id)); @@ -861,11 +861,11 @@ int maat_compile_clause_remove_literal(struct maat_compile *compile, return -1; } - size_t remove_idx = utarray_eltidx(clause_state->literal_ids, tmp); - utarray_erase(clause_state->literal_ids, remove_idx, 1); + size_t remove_idx = utarray_eltidx(clause_state->ut_literal_ids, tmp); + utarray_erase(clause_state->ut_literal_ids, remove_idx, 1); - if (0 == utarray_len(clause_state->literal_ids)) { - clause_state->in_use=0; + if (0 == utarray_len(clause_state->ut_literal_ids)) { + clause_state->in_use = 0; compile->actual_clause_num--; } @@ -874,7 +874,7 @@ int maat_compile_clause_remove_literal(struct maat_compile *compile, static const struct maat_clause * maat_clause_hash_fetch_clause(struct maat_clause **clause_hash, - unsigned long long *clause_id_generator, + struct maat_runtime *ref_maat_rt, struct maat_literal_id *literal_ids, size_t n_literal_id) { @@ -883,12 +883,11 @@ maat_clause_hash_fetch_clause(struct maat_clause **clause_hash, HASH_FIND(hh, *clause_hash, literal_ids, n_literal_id * sizeof(struct maat_literal_id), clause); if (!clause) { clause = ALLOC(struct maat_clause, 1); - clause->clause_id = *clause_id_generator; + clause->clause_id = maat_runtime_get_sequence(ref_maat_rt, "clause_id"); clause->n_literal_id = n_literal_id; clause->literal_ids = ALLOC(struct maat_literal_id, n_literal_id); memcpy(clause->literal_ids, literal_ids, n_literal_id * sizeof(struct maat_literal_id)); - (*clause_id_generator)++; HASH_ADD_KEYPTR(hh, *clause_hash, clause->literal_ids, n_literal_id * sizeof(struct maat_literal_id), clause); } @@ -896,24 +895,21 @@ maat_clause_hash_fetch_clause(struct maat_clause **clause_hash, return clause; } -static void maat_clause_hash_free(struct maat_clause *clause_hash) +static void maat_clause_hash_free(struct maat_clause **clause_hash) { struct maat_clause *clause = NULL, *tmp_clause = NULL; - HASH_ITER (hh, clause_hash, clause, tmp_clause) { - HASH_DELETE(hh, clause_hash, clause); - free(clause->literal_ids); + HASH_ITER (hh, *clause_hash, clause, tmp_clause) { + HASH_DEL(*clause_hash, clause); + FREE(clause->literal_ids); clause->n_literal_id = 0; - free(clause); + FREE(clause); } } -struct bool_matcher * -maat_compile_bool_matcher_new(struct maat_compile *compile_hash, - unsigned long long *clause_id_generator, - struct log_handle *logger) +struct bool_matcher *maat_compile_bool_matcher_new(struct compile_runtime *compile_rt) { - if (NULL == compile_hash || NULL == logger) { + if (NULL == compile_rt) { return NULL; } @@ -924,11 +920,12 @@ maat_compile_bool_matcher_new(struct maat_compile *compile_hash, const struct maat_clause *clause = NULL; struct maat_clause *clause_hash = NULL; // + pthread_rwlock_rdlock(&compile_rt->rwlock); //STEP 1, update clause_id of each compile and literal struct maat_compile *compile = NULL, *tmp_compile = NULL; struct maat_literal_id *literal_ids = NULL; size_t n_literal_id = 0; - HASH_ITER(hh, compile_hash, compile, tmp_compile) { + HASH_ITER(hh, compile_rt->compile_hash, compile, tmp_compile) { has_clause_num = 0; for (i = 0; i < MAX_ITEMS_PER_BOOL_EXPR; i++) { clause_state = compile->clause_states + i; @@ -938,9 +935,9 @@ maat_compile_bool_matcher_new(struct maat_compile *compile_hash, } has_clause_num++; - literal_ids = (struct maat_literal_id *)utarray_eltptr(clause_state->literal_ids, 0); - n_literal_id = utarray_len(clause_state->literal_ids); - clause = maat_clause_hash_fetch_clause(&clause_hash, clause_id_generator, + literal_ids = (struct maat_literal_id *)utarray_eltptr(clause_state->ut_literal_ids, 0); + n_literal_id = utarray_len(clause_state->ut_literal_ids); + clause = maat_clause_hash_fetch_clause(&clause_hash, compile_rt->ref_maat_rt, literal_ids, n_literal_id); clause_state->clause_id = clause->clause_id; } @@ -949,9 +946,9 @@ maat_compile_bool_matcher_new(struct maat_compile *compile_hash, //STEP 2, serial compile clause states to a bool expression array size_t expr_cnt = 0; - size_t compile_cnt = maat_compile_in_use_count(compile_hash); + size_t compile_cnt = HASH_COUNT(compile_rt->compile_hash); struct bool_expr *bool_expr_array = ALLOC(struct bool_expr, compile_cnt); - HASH_ITER(hh, compile_hash, compile, tmp_compile) { + HASH_ITER(hh, compile_rt->compile_hash, compile, tmp_compile) { for (i = 0, j = 0; i < MAX_ITEMS_PER_BOOL_EXPR; i++) { if (compile->clause_states[i].in_use) { if (compile->clause_states[i].not_flag) { @@ -961,9 +958,9 @@ maat_compile_bool_matcher_new(struct maat_compile *compile_hash, //TODO:mytest need to delete #if 0 struct maat_literal_id *p = NULL; - for(p = (struct maat_literal_id *)utarray_front(compile->clause_states[i].literal_ids); p!=NULL; p=(struct maat_literal_id *)utarray_next(compile->clause_states[i].literal_ids,p)) { - printf(" compile_id:%lld, clause_id:%llu, literal{%lld: %d}\n", - compile->compile_id, compile->clause_states[i].clause_id, p->group_id, p->vtable_id); + for(p = (struct maat_literal_id *)utarray_front(compile->clause_states[i].ut_literal_ids); p!=NULL; p=(struct maat_literal_id *)utarray_next(compile->clause_states[i].ut_literal_ids,p)) { + printf(" compile_rt:%p compile_id:%lld, clause_id:%llu, literal{%lld: %d}\n", + compile_rt, compile->compile_id, compile->clause_states[i].clause_id, p->group_id, p->vtable_id); } #endif bool_expr_array[expr_cnt].items[j].item_id = compile->clause_states[i].clause_id; @@ -973,7 +970,7 @@ maat_compile_bool_matcher_new(struct maat_compile *compile_hash, } } - // printf("bool_matcher_new compile_id:%lu j:%zu, compile->declared_clause_num:%d\n", + // printf("bool_matcher_new compile_id:%lld j:%zu, compile->declared_clause_num:%d\n", // compile->compile_id, j, compile->declared_clause_num); //some compile may have zero groups, e.g. default policy. if (j == (size_t)compile->declared_clause_num && j > 0) { @@ -983,13 +980,13 @@ maat_compile_bool_matcher_new(struct maat_compile *compile_hash, expr_cnt++; } } - + pthread_rwlock_unlock(&compile_rt->rwlock); //size_t expr_index = 0, item_index = 0; // STEP 3, build bool matcher size_t mem_size = 0; if (0 == expr_cnt) { - log_error(logger, MODULE_COMPILE, + log_error(compile_rt->logger, MODULE_COMPILE, "[%s:%d] No bool expression to build bool matcher.", __FUNCTION__, __LINE__); goto error; @@ -999,28 +996,32 @@ maat_compile_bool_matcher_new(struct maat_compile *compile_hash, #if 0 printf("bool_matcher_new....................expr_cnt:%zu\n", expr_cnt); for (expr_index = 0; expr_index < expr_cnt; expr_index++) { - printf("bool_expr_array[%zu].expr_id:%llu, item_num:%zu\n", expr_index, bool_expr_array[expr_index].expr_id, - bool_expr_array[expr_index].item_num); - for (item_index = 0; item_index < bool_expr_array[expr_index].item_num; item_index++) { - printf("bool_expr_array[%zu].items[%zu]:%llu, not_flag:%d\n", expr_index, item_index, - bool_expr_array[expr_index].items[item_index].item_id, - bool_expr_array[expr_index].items[item_index].not_flag); - } - printf("\n"); + if (bool_expr_array[expr_index].expr_id == 141 || bool_expr_array[expr_index].expr_id == 197) { + printf("compile_rt:%p expr_id:%llu\n", compile_rt, bool_expr_array[expr_index].expr_id); + // for (item_index = 0; item_index < bool_expr_array[expr_index].item_num; item_index++) { + // printf("bool_expr_array[%zu].items[%zu]:%llu, not_flag:%d\n", expr_index, item_index, + // bool_expr_array[expr_index].items[item_index].item_id, + // bool_expr_array[expr_index].items[item_index].not_flag); + // } + } } #endif bm = bool_matcher_new(bool_expr_array, expr_cnt, &mem_size); if (bm != NULL) { - log_info(logger, MODULE_COMPILE, + log_info(compile_rt->logger, MODULE_COMPILE, "Build bool matcher of %zu expressions with %zu bytes memory.", expr_cnt, mem_size); } else { - log_error(logger, MODULE_COMPILE, "[%s:%d] Build bool matcher failed!", + log_error(compile_rt->logger, MODULE_COMPILE, "[%s:%d] Build bool matcher failed!", __FUNCTION__, __LINE__); } error: - maat_clause_hash_free(clause_hash); + if (clause_hash != NULL) { + maat_clause_hash_free(&clause_hash); + clause_hash = NULL; + } + FREE(bool_expr_array); return bm; @@ -1031,8 +1032,7 @@ void maat_compile_bool_matcher_free(struct bool_matcher *bm) bool_matcher_free(bm); } -static int maat_compile_has_clause(struct maat_compile *compile, - unsigned long long clause_id) +static int maat_compile_has_clause(struct maat_compile *compile, long long clause_id) { struct maat_clause_state *clause_state = NULL; @@ -1056,9 +1056,9 @@ static size_t compile_state_if_new_hit_compile(struct maat_compile_state *compil size_t r_in_c_cnt = 0; int ret = 0; - unsigned long long new_hit_clause_id = 0; + long long new_hit_clause_id = 0; for (size_t i = 0; i < utarray_len(compile_state->this_scan_hit_clauses); i++) { - new_hit_clause_id = *(unsigned long long*)utarray_eltptr(compile_state->this_scan_hit_clauses, i); + new_hit_clause_id = *(long long*)utarray_eltptr(compile_state->this_scan_hit_clauses, i); ret = maat_compile_has_clause(compile, new_hit_clause_id); if (ret) { r_in_c_cnt++; @@ -1068,25 +1068,25 @@ static size_t compile_state_if_new_hit_compile(struct maat_compile_state *compil return r_in_c_cnt; } -size_t maat_compile_bool_matcher_match(struct bool_matcher *bm, int is_last_scan, +size_t maat_compile_bool_matcher_match(struct compile_runtime *compile_rt, int is_last_scan, struct maat_compile_state *compile_state, void **user_data_array, size_t ud_array_size) { + size_t ud_result_cnt = 0; struct maat_compile *compile = NULL; - struct bool_expr_match *expr_match = ALLOC(struct bool_expr_match, MAX_SCANNER_HIT_COMPILE_NUM); - size_t ud_result_cnt = 0; + struct bool_expr_match *expr_match = compile_rt->expr_match_buff + compile_state->thread_id * MAX_SCANNER_HIT_COMPILE_NUM; + assert(compile_state->thread_id >= 0); //TODO:mytest need to delete #if 0 - unsigned long long *p; - printf("utarray_len:%u\n", utarray_len(compile_state->all_hit_clauses)); + unsigned long long *p = utarray_eltptr(compile_state->all_hit_clauses, 0); for (p = (unsigned long long *)utarray_front(compile_state->all_hit_clauses); p != NULL; p = (unsigned long long *)utarray_next(compile_state->all_hit_clauses, p)) { - printf("before bool_matcher_match compile_state clause_id:%llu\n", *p); + printf("before bool_matcher_match compile_rt:%p compile_state clause_id:%llu\n", compile_rt, *p); } #endif - int bool_match_ret = bool_matcher_match(bm, (unsigned long long *)utarray_eltptr(compile_state->all_hit_clauses, 0), + int bool_match_ret = bool_matcher_match(compile_rt->bm, (unsigned long long *)utarray_eltptr(compile_state->all_hit_clauses, 0), utarray_len(compile_state->all_hit_clauses), expr_match, MAX_SCANNER_HIT_COMPILE_NUM); for (int i = 0; i < bool_match_ret && ud_result_cnt < ud_array_size; i++) { @@ -1112,6 +1112,7 @@ size_t maat_compile_bool_matcher_match(struct bool_matcher *bm, int is_last_scan } compile_state->this_scan_hit_item_cnt = 0; + return ud_result_cnt; } @@ -1143,7 +1144,7 @@ int maat_add_group_to_compile(struct maat_compile **compile_hash, struct group2c ret = 0; } - // printf("group2compile update compile_id:%lu, compile->declared_clause_num:%d\n", + // printf("group2compile update compile_id:%lld, compile->declared_clause_num:%d\n", // compile->compile_id, compile->declared_clause_num); return ret; } @@ -1167,7 +1168,8 @@ int maat_remove_group_from_compile(struct maat_compile **compile_hash, if (ret < 0) { log_error(logger, MODULE_COMPILE, "[%s:%d] Remove group %d vtable_id %d from clause %d of compile %d failed, literal is not in compile.", - __FUNCTION__, __LINE__, g2c_item->group_id, g2c_item->vtable_id, g2c_item->clause_index, g2c_item->compile_id); + __FUNCTION__, __LINE__, g2c_item->group_id, g2c_item->vtable_id, g2c_item->clause_index, + g2c_item->compile_id); return -1; } @@ -1181,7 +1183,7 @@ int maat_remove_group_from_compile(struct maat_compile **compile_hash, static inline int compare_clause_id(const void *a, const void *b) { - long long ret = *(const unsigned long long *)a - *(const unsigned long long *)b; + long long ret = *(const long long *)a - *(const long long *)b; if (0 == ret) { return 0; @@ -1242,7 +1244,7 @@ static int maat_compile_has_literal(struct maat_compile* compile, continue; } - tmp = (struct maat_literal_id*)utarray_find(clause_state->literal_ids, + tmp = (struct maat_literal_id*)utarray_find(clause_state->ut_literal_ids, literal_id, compare_literal_id); if (tmp) { assert(tmp->group_id == literal_id->group_id && @@ -1277,7 +1279,9 @@ size_t compile_runtime_get_new_hit_paths(struct compile_runtime *compile_rt, struct maat_literal_id literal_id = {0, 0}; struct maat_hit_path tmp_path; struct bool_expr_match *expr_match = compile_rt->expr_match_buff + compile_state->thread_id * MAX_SCANNER_HIT_COMPILE_NUM; + assert(compile_state->thread_id >= 0); + pthread_rwlock_rdlock(&compile_rt->rwlock); int bool_match_ret = bool_matcher_match(compile_rt->bm, (unsigned long long *)utarray_eltptr(compile_state->all_hit_clauses, 0), utarray_len(compile_state->all_hit_clauses), expr_match, @@ -1312,7 +1316,8 @@ size_t compile_runtime_get_new_hit_paths(struct compile_runtime *compile_rt, } } } - + pthread_rwlock_unlock(&compile_rt->rwlock); + return new_hit_path_cnt; } @@ -1345,7 +1350,7 @@ void maat_compile_state_update_hit_clause(struct maat_compile_state *compile_sta struct maat_clause_state *clause_state = NULL; struct maat_literal_id literal_id = {group_id, vtable_id}; struct maat_literal_id *tmp = NULL; - unsigned long long *clause_id = 0; + long long *clause_id = 0; struct compile_runtime *compile_rt = (struct compile_runtime *)compile_runtime; pthread_rwlock_rdlock(&compile_rt->rwlock); @@ -1359,7 +1364,7 @@ void maat_compile_state_update_hit_clause(struct maat_compile_state *compile_sta } size_t new_clause_idx = utarray_len(compile_state->this_scan_hit_clauses); - tmp = (struct maat_literal_id *)utarray_find(clause_state->literal_ids, + tmp = (struct maat_literal_id *)utarray_find(clause_state->ut_literal_ids, &literal_id, compare_literal_id); if (tmp) { //Deduplication @@ -1377,7 +1382,7 @@ void maat_compile_state_update_hit_clause(struct maat_compile_state *compile_sta utarray_len(compile_state->this_scan_hit_clauses) - new_clause_idx); for (i = new_clause_idx; i < utarray_len(compile_state->this_scan_hit_clauses); i++) { - clause_id = (unsigned long long *)utarray_eltptr(compile_state->this_scan_hit_clauses, i); + clause_id = (long long *)utarray_eltptr(compile_state->this_scan_hit_clauses, i); utarray_push_back(compile_state->all_hit_clauses, clause_id); } utarray_sort(compile_state->all_hit_clauses, compare_clause_id); @@ -1539,7 +1544,7 @@ int compile_runtime_update(void *compile_runtime, void *compile_schema, } else { maat_compile_hash_add(&(compile_rt->compile_hash), compile_id, compile); } - // printf("compile_runtime_update compile_id:%lu, compile->declared_clause_num:%d\n", + // printf("compile_runtime_update compile_id:%lld, compile->declared_clause_num:%d\n", // compile->compile_id, compile->declared_clause_num); pthread_rwlock_unlock(&compile_rt->rwlock); @@ -1621,23 +1626,17 @@ int compile_runtime_commit(void *compile_runtime, const char *table_name) } struct compile_runtime *compile_rt = (struct compile_runtime *)compile_runtime; - //TODO: no need add rdlock? - size_t compile_cnt = maat_compile_in_use_count(compile_rt->compile_hash); - if (0 == compile_cnt) { - return 0; - } - + struct bool_matcher *old_bool_matcher = NULL; struct bool_matcher *new_bool_matcher = NULL; + size_t compile_cnt = HASH_COUNT(compile_rt->compile_hash); log_info(compile_rt->logger, MODULE_COMPILE, "table[%s] committing %zu compile rules for rebuilding compile bool_matcher engine", table_name, compile_cnt); int ret = 0; - new_bool_matcher = maat_compile_bool_matcher_new(compile_rt->compile_hash, - &compile_rt->clause_id_generator, - compile_rt->logger); + new_bool_matcher = maat_compile_bool_matcher_new(compile_rt); if (NULL == new_bool_matcher) { log_error(compile_rt->logger, MODULE_COMPILE, "[%s:%d] table[%s] rebuild compile bool_matcher engine failed when update %zu compile rules", @@ -1711,12 +1710,8 @@ int compile_runtime_match(struct compile_runtime *compile_rt, long long *compile struct compile_rule *compile_rules[compile_ids_size]; // all hit clause_id -> compile_id - pthread_rwlock_rdlock(&compile_rt->rwlock); - size_t bool_match_ret = maat_compile_bool_matcher_match(compile_rt->bm, - is_last_scan, compile_state, - (void **)compile_rules, - compile_ids_size); - pthread_rwlock_unlock(&compile_rt->rwlock); + size_t bool_match_ret = maat_compile_bool_matcher_match(compile_rt, is_last_scan, compile_state, + (void **)compile_rules, compile_ids_size); if (bool_match_ret > 0) { qsort(compile_rules, bool_match_ret, sizeof(struct compile_rule *), compare_compile_rule); @@ -1729,7 +1724,7 @@ int compile_runtime_match(struct compile_runtime *compile_rt, long long *compile return MIN(bool_match_ret, compile_ids_size); } -int maat_compile_state_update(struct maat_item *item_hash, int vtable_id, +int maat_compile_state_update(struct rcu_hash_table *item_htable, int vtable_id, long long *hit_item_ids, size_t hit_item_cnt, size_t *n_hit_group_id, struct maat_state *state) { @@ -1745,7 +1740,7 @@ int maat_compile_state_update(struct maat_item *item_hash, int vtable_id, } for (size_t i = 0; i < hit_item_cnt; i++) { - HASH_FIND(hh, item_hash, &(hit_item_ids[i]), sizeof(long long), item); + item = (struct maat_item *)rcu_hash_find(item_htable, (char *)&(hit_item_ids[i]), sizeof(long long)); //assert(item != NULL); if (!item) { // item config has been deleted diff --git a/src/maat_config_monitor.c b/src/maat_config_monitor.c index cb12ada..f23670f 100644 --- a/src/maat_config_monitor.c +++ b/src/maat_config_monitor.c @@ -109,9 +109,9 @@ char *read_nxt_line_from_buff(const char *buff, size_t buff_size, return line; } -int cm_read_table_file(struct cm_table_info_t* index, +int cm_read_table_file(struct cm_table_info_t *index, int (*update_fn)(const char *, const char *, void *), - void* u_param) + void *u_param, struct log_handle *logger) { int cfg_num = 0,i =0; int ret = 0; @@ -123,7 +123,8 @@ int cm_read_table_file(struct cm_table_info_t* index, ret = load_file_to_memory(index->cfg_path, (unsigned char **)&table_file_buff, &file_sz); if (ret < 0) { - // log_error + log_error(logger, MODULE_CONFIG_MONITOR, "[%s:%d] open %s failed.", + __FUNCTION__, __LINE__, index->cfg_path); return -1; } @@ -131,7 +132,9 @@ int cm_read_table_file(struct cm_table_info_t* index, sscanf(line, "%d\n", &cfg_num); if(cfg_num != index->cfg_num) { - //log_error + FREE(table_file_buff); + log_error(logger, MODULE_CONFIG_MONITOR, "[%s:%d] file %s config num not matched", + __FUNCTION__, __LINE__, index->cfg_path); return -1; } @@ -140,17 +143,19 @@ int cm_read_table_file(struct cm_table_info_t* index, ret_str = read_nxt_line_from_buff(table_file_buff, file_sz, &file_offset, line, sizeof(line)); if (ret_str == NULL) { - //log_error + log_error(logger, MODULE_CONFIG_MONITOR, "[%s:%d] file %s line_num %d less than claimed %d", + __FUNCTION__, __LINE__, index->cfg_path, i, cfg_num); break; } if(line[sizeof(line) - 1] != '\0') { - //log_error + log_error(logger, MODULE_CONFIG_MONITOR, "[%s:%d] line size more than %u at of file %s:%d", + __FUNCTION__, __LINE__, sizeof(line), index->cfg_path, i); continue; } ret = update_fn(index->table_name, line, u_param); - if (ret<0) { + if (ret < 0) { break; } } @@ -220,7 +225,7 @@ int get_new_idx_path(long long current_version, const char *file_dir, char ***idx_path, size_t *idx_num, struct log_handle *logger) { struct dirent **namelist = NULL; - int update_type = MAAT_UPDATE_TYPE_NONE; + int update_type = MAAT_UPDATE_TYPE_INVALID; int n = my_scandir(file_dir, &namelist, filter_fn, (int (*)(const void*, const void*))alphasort); if (n < 0) { @@ -297,7 +302,7 @@ int get_new_idx_path(long long current_version, const char *file_dir, *idx_num = inc_idx_num; update_type = MAAT_UPDATE_TYPE_INC; } else { - update_type = MAAT_UPDATE_TYPE_NONE; + update_type = MAAT_UPDATE_TYPE_INVALID; } FREE(inc_file_idx); @@ -324,7 +329,7 @@ void config_monitor_traverse(long long current_version, const char *idx_dir, memset(table_array, 0, sizeof(table_array)); int update_type = get_new_idx_path(current_version, idx_dir, &idx_path_array, &idx_path_num, logger); - if (update_type != MAAT_UPDATE_TYPE_NONE) { + if (update_type != MAAT_UPDATE_TYPE_INVALID) { for (i = 0; i < idx_path_num; i++) { log_info(logger, MODULE_CONFIG_MONITOR, "load %s", idx_path_array[i]); int table_num = cm_read_cfg_index_file(idx_path_array[i], table_array, MAX_TABLE_NUM); @@ -343,7 +348,7 @@ void config_monitor_traverse(long long current_version, const char *idx_dir, } for (int j = 0; j < table_num; j++) { - cm_read_table_file(table_array + j, update_fn, u_param); + cm_read_table_file(table_array + j, update_fn, u_param, logger); } if (finish_fn != NULL) { @@ -424,9 +429,7 @@ int load_maat_json_file(struct maat *maat_instance, const char *json_filename, strlen(maat_instance->decrypt_key) ? maat_instance->decrypt_key : NULL, strlen(maat_instance->decrypt_algo) ? maat_instance->decrypt_algo : NULL, maat_instance->logger); - FREE(json_buff); - json_buff = NULL; if (ret < 0) { return -1; } diff --git a/src/maat_ex_data.c b/src/maat_ex_data.c index 83c478d..fe2a3cb 100644 --- a/src/maat_ex_data.c +++ b/src/maat_ex_data.c @@ -26,6 +26,7 @@ struct ex_data_runtime { struct rcu_hash_table *htable; struct ex_data_schema *ref_ex_schema; + struct maat_garbage_bin *ref_garbage_bin; int table_id; struct log_handle *logger; @@ -33,20 +34,21 @@ struct ex_data_runtime { void cache_row_free(void *p) { - FREE(*(char **)p); + free(*(char **)p); } UT_icd ut_cache_row_icd = {sizeof(char*), NULL, NULL, cache_row_free}; struct ex_data_runtime * ex_data_runtime_new(int table_id, rcu_hash_data_free_fn *data_free_fn, - struct log_handle *logger) + struct maat_garbage_bin *garbage_bin, struct log_handle *logger) { struct ex_data_runtime *ex_data_rt = ALLOC(struct ex_data_runtime, 1); utarray_new(ex_data_rt->cache_rows, &ut_cache_row_icd); ex_data_rt->htable = rcu_hash_new(data_free_fn); ex_data_rt->table_id = table_id; + ex_data_rt->ref_garbage_bin = garbage_bin; ex_data_rt->logger = logger; return ex_data_rt; @@ -137,7 +139,7 @@ struct ex_data_schema *ex_data_schema_new(maat_ex_new_func_t *new_func, void ex_data_schema_free(struct ex_data_schema *ex_schema) { - FREE(ex_schema); + free(ex_schema); } void ex_data_runtime_set_schema(struct ex_data_runtime *ex_data_rt, @@ -146,16 +148,16 @@ void ex_data_runtime_set_schema(struct ex_data_runtime *ex_data_rt, ex_data_rt->ref_ex_schema = schema; } -void ex_data_runtime_set_ex_container_ctx(struct ex_data_runtime *ex_data_rt, - struct ex_container_ctx *container_ctx) +void ex_data_runtime_set_ex_container_schema(struct ex_data_runtime *ex_data_rt, + struct ex_container_schema *container_schema) { - rcu_hash_set_user_ctx(ex_data_rt->htable, container_ctx); + rcu_hash_set_user_ctx(ex_data_rt->htable, container_schema); } -struct ex_container_ctx * -ex_data_runtime_get_ex_container_ctx(struct ex_data_runtime *ex_data_rt) +struct ex_container_schema * +ex_data_runtime_get_ex_container_schema(struct ex_data_runtime *ex_data_rt) { - return (struct ex_container_ctx *)rcu_hash_get_user_ctx(ex_data_rt->htable); + return (struct ex_container_schema *)rcu_hash_get_user_ctx(ex_data_rt->htable); } void *ex_data_runtime_row2ex_data(struct ex_data_runtime *ex_data_rt, const char *row, @@ -169,9 +171,9 @@ void *ex_data_runtime_row2ex_data(struct ex_data_runtime *ex_data_rt, const char return ex_data; } -struct ex_data_container *ex_data_container_new(void *ex_data, void *custom_data) +struct ex_container *ex_container_new(void *ex_data, void *custom_data) { - struct ex_data_container *ex_container = ALLOC(struct ex_data_container, 1); + struct ex_container *ex_container = ALLOC(struct ex_container, 1); ex_container->ex_data = ex_data; ex_container->custom_data = custom_data; @@ -179,26 +181,31 @@ struct ex_data_container *ex_data_container_new(void *ex_data, void *custom_data return ex_container; } -void ex_data_container_free(void *ctx, void *data) +void ex_container_free(void *schema, void *data) { - if (NULL == ctx || NULL == data) { + /* schema is NULL if not call ex_data_runtime_set_ex_container_schema */ + if (NULL == data) { return; } + //TODO: + if (NULL == schema) { - struct ex_container_ctx *container_ctx = (struct ex_container_ctx *)ctx; - long argl = container_ctx->ex_schema->argl; - void *argp = container_ctx->ex_schema->argp; + } + + struct ex_container_schema *container_schema = (struct ex_container_schema *)schema; + long argl = container_schema->ex_schema->argl; + void *argp = container_schema->ex_schema->argp; - struct ex_data_container *ex_container = (struct ex_data_container *)data; + struct ex_container *ex_container = (struct ex_container *)data; if (ex_container->ex_data != NULL - && container_ctx->ex_schema->free_func != NULL) { - container_ctx->ex_schema->free_func(container_ctx->table_id, + && container_schema->ex_schema->free_func != NULL) { + container_schema->ex_schema->free_func(container_schema->table_id, &(ex_container->ex_data), argl, argp); } if (ex_container->custom_data != NULL - && container_ctx->custom_data_free != NULL) { - container_ctx->custom_data_free(ex_container->custom_data); + && container_schema->custom_data_free != NULL) { + container_schema->custom_data_free(ex_container->custom_data); } FREE(ex_container); @@ -206,12 +213,12 @@ void ex_data_container_free(void *ctx, void *data) int ex_data_runtime_add_ex_container(struct ex_data_runtime *ex_data_rt, const char *key, size_t key_len, - struct ex_data_container *ex_container) + struct ex_container *ex_container) { - struct ex_data_container *tmp_container = NULL; + struct ex_container *tmp_container = NULL; - tmp_container = (struct ex_data_container *)rcu_hash_find(ex_data_rt->htable, - key, key_len); + tmp_container = (struct ex_container *)rcu_hash_find(ex_data_rt->htable, + key, key_len); if (tmp_container != NULL) { log_error(ex_data_rt->logger, MODULE_EX_DATA, "[%s:%d] ex_data_runtime add ex container error: already exist same key:%s", @@ -227,8 +234,8 @@ int ex_data_runtime_add_ex_container(struct ex_data_runtime *ex_data_rt, int ex_data_runtime_del_ex_container(struct ex_data_runtime *ex_data_rt, const char *key, size_t key_len) { - struct ex_data_container *tmp_container = NULL; - tmp_container = (struct ex_data_container *)rcu_hash_find(ex_data_rt->htable, + struct ex_container *tmp_container = NULL; + tmp_container = (struct ex_container *)rcu_hash_find(ex_data_rt->htable, key, key_len); if (NULL == tmp_container) { log_error(ex_data_rt->logger, MODULE_EX_DATA, @@ -238,16 +245,16 @@ int ex_data_runtime_del_ex_container(struct ex_data_runtime *ex_data_rt, } rcu_hash_del(ex_data_rt->htable, key, key_len); - + return 0; } void *ex_data_runtime_get_ex_data_by_key(struct ex_data_runtime *ex_data_rt, const char *key, size_t key_len) { - struct ex_data_container *ex_container = NULL; + struct ex_container *ex_container = NULL; - ex_container = (struct ex_data_container *)rcu_hash_find(ex_data_rt->htable, + ex_container = (struct ex_container *)rcu_hash_find(ex_data_rt->htable, key, key_len); if (NULL == ex_container) { return NULL; @@ -262,7 +269,7 @@ void *ex_data_runtime_get_ex_data_by_key(struct ex_data_runtime *ex_data_rt, } void *ex_data_runtime_get_ex_data_by_container(struct ex_data_runtime *ex_data_rt, - struct ex_data_container *ex_container) + struct ex_container *ex_container) { void *dup_ex_data = NULL; ex_data_rt->ref_ex_schema->dup_func(ex_data_rt->table_id, &dup_ex_data, @@ -275,8 +282,8 @@ void *ex_data_runtime_get_ex_data_by_container(struct ex_data_runtime *ex_data_r void *ex_data_runtime_get_custom_data(struct ex_data_runtime *ex_data_rt, const char *key, size_t key_len) { - struct ex_data_container *ex_container = NULL; - ex_container = (struct ex_data_container *)rcu_hash_find(ex_data_rt->htable, + struct ex_container *ex_container = NULL; + ex_container = (struct ex_container *)rcu_hash_find(ex_data_rt->htable, key, key_len); if (NULL == ex_container) { return NULL; @@ -290,8 +297,13 @@ size_t ex_data_runtime_ex_container_count(struct ex_data_runtime *ex_data_rt) return rcu_hash_count(ex_data_rt->htable); } -size_t ex_data_runtime_list_updating_ex_container(struct ex_data_runtime *ex_data_rt, - struct ex_data_container ***ex_container) +int ex_data_runtime_is_updating(struct ex_data_runtime *ex_data_rt) { - return rcu_hash_list_updating_data(ex_data_rt->htable, (void ***)ex_container); -} \ No newline at end of file + return rcu_hash_is_updating(ex_data_rt->htable); +} + +size_t ex_data_runtime_list_ex_container(struct ex_data_runtime *ex_data_rt, + struct ex_container ***ex_container) +{ + return rcu_hash_list(ex_data_rt->htable, (void ***)ex_container); +} diff --git a/src/maat_expr.c b/src/maat_expr.c index 26982de..8452518 100644 --- a/src/maat_expr.c +++ b/src/maat_expr.c @@ -60,7 +60,7 @@ enum match_method { struct expr_item { long long item_id; long long group_id; - int district_id; + long long district_id; char keywords[MAX_KEYWORDS_STR]; enum expr_type expr_type; enum hs_match_mode match_mode; @@ -73,14 +73,11 @@ struct expr_runtime { enum hs_pattern_type pattern_type; struct adapter_hs *hs; struct adapter_hs_stream *hs_stream; - struct rcu_hash_table *htable; + struct rcu_hash_table *htable; // store hs_expr rule for rebuild adapter_hs instance + struct rcu_hash_table *item_htable; // store this expr table's all maat_item which will be used in expr_runtime_scan struct group2group_runtime *ref_g2g_rt; uint32_t rule_num; - - struct maat_item *item_hash; - void (*item_user_data_free)(void *); - int n_worker_thread; struct maat_garbage_bin *ref_garbage_bin; struct log_handle *logger; @@ -401,7 +398,7 @@ void expr_schema_free(void *expr_schema) FREE(expr_schema); } -void expr_rule_free(and_expr_t *expr_rule) +void expr_rule_free(struct hs_expr *expr_rule) { if (NULL == expr_rule) { return; @@ -416,7 +413,7 @@ void expr_rule_free(and_expr_t *expr_rule) void expr_ex_data_free(void *user_ctx, void *data) { - and_expr_t *expr_rule = (and_expr_t *)data; + struct hs_expr *expr_rule = (struct hs_expr *)data; expr_rule_free(expr_rule); } @@ -432,9 +429,9 @@ void *expr_runtime_new(void *expr_schema, int max_thread_num, struct expr_runtime *expr_rt = ALLOC(struct expr_runtime, 1); expr_rt->htable = rcu_hash_new(expr_ex_data_free); + expr_rt->item_htable = rcu_hash_new(maat_item_free); expr_rt->scan_mode = schema->scan_mode; expr_rt->pattern_type = schema->pattern_type; - expr_rt->item_user_data_free = maat_item_inner_free; expr_rt->n_worker_thread = max_thread_num; expr_rt->ref_garbage_bin = garbage_bin; expr_rt->logger = logger; @@ -467,10 +464,9 @@ void expr_runtime_free(void *expr_runtime) expr_rt->htable = NULL; } - struct maat_item *item = NULL, *tmp = NULL; - HASH_ITER(hh, expr_rt->item_hash, item, tmp) { - HASH_DELETE(hh, expr_rt->item_hash, item); - maat_item_free(item, expr_rt->item_user_data_free); + if (expr_rt->item_htable != NULL) { + rcu_hash_free(expr_rt->item_htable); + expr_rt->item_htable = NULL; } if (expr_rt->hit_cnt != NULL) { @@ -486,9 +482,8 @@ void expr_runtime_free(void *expr_runtime) FREE(expr_rt); } -int expr_runtime_update_row(struct expr_runtime *expr_rt, char *key, - size_t key_len, long long item_id, and_expr_t *expr_rule, - int is_valid, struct log_handle *logger) +int expr_runtime_update_row(struct expr_runtime *expr_rt, char *key, size_t key_len, + long long item_id, struct hs_expr *expr_rule, int is_valid) { void *data = NULL; @@ -496,21 +491,34 @@ int expr_runtime_update_row(struct expr_runtime *expr_rt, char *key, //delete data = rcu_hash_find(expr_rt->htable, key, key_len); if (NULL == data) { - log_error(logger, MODULE_EXPR, - "[%s:%d] the key of expr rule not exist, can't be deleted, expr_id:%llu", + data = rcu_hash_updating_find(expr_rt->htable, key, key_len); + if (NULL == data) { + log_error(expr_rt->logger, MODULE_EXPR, + "[%s:%d] the key of expr rule(rule_id:%lld) not exist, can't be deleted from expr runtime htable", __FUNCTION__, __LINE__, item_id); - return -1; + return -1; + } } + rcu_hash_del(expr_rt->htable, key, key_len); } else { //add data = rcu_hash_find(expr_rt->htable, key, key_len); if (data != NULL) { - log_error(logger, MODULE_EXPR, - "[%s:%d] the key of expr rule already exist, can't be added, expr_id:%llu", + log_error(expr_rt->logger, MODULE_EXPR, + "[%s:%d] the key of expr rule(rule_id:%lld) already exist, can't be added to expr runtime htable", __FUNCTION__, __LINE__, item_id); return -1; } + + data = rcu_hash_updating_find(expr_rt->htable, key, key_len); + if (data != NULL) { + log_error(expr_rt->logger, MODULE_EXPR, + "[%s:%d] the key of expr rule(rule_id:%lld) already exist, can't be added to expr runtime htable", + __FUNCTION__, __LINE__, item_id); + return -1; + } + rcu_hash_add(expr_rt->htable, key, key_len, (void *)expr_rule); } @@ -565,7 +573,7 @@ static size_t hex2bin(char *hex, int hex_len, char *binary, size_t size) } #define MAAT_MAX_EXPR_ITEM_NUM 8 -and_expr_t *expr_item_to_expr_rule(struct expr_item *expr_item, void *user_data, +struct hs_expr *expr_item_to_expr_rule(struct expr_item *expr_item, void *user_data, struct log_handle *logger) { size_t i = 0; @@ -581,7 +589,7 @@ and_expr_t *expr_item_to_expr_rule(struct expr_item *expr_item, void *user_data, memset(key_left_offset, -1, sizeof(key_left_offset)); memset(key_right_offset, -1, sizeof(key_right_offset)); - and_expr_t *expr_rule = ALLOC(and_expr_t, 1); + struct hs_expr *expr_rule = ALLOC(struct hs_expr, 1); switch (expr_item->expr_type) { case EXPR_TYPE_AND: @@ -718,7 +726,7 @@ int expr_runtime_update(void *expr_runtime, void *expr_schema, int ret = -1; struct maat_item_inner *u_para = NULL; struct maat_item *item = NULL; - and_expr_t *expr_rule = NULL; + struct hs_expr *expr_rule = NULL; struct expr_schema *schema = (struct expr_schema *)expr_schema; struct expr_runtime *expr_rt = (struct expr_runtime *)expr_runtime; @@ -728,30 +736,34 @@ int expr_runtime_update(void *expr_runtime, void *expr_schema, } int is_valid = get_column_value(line, valid_column); + //printf(" item_id:%lld is_valid:%d\n", item_id, is_valid); if (is_valid < 0) { return -1; } else if (0 == is_valid) { //delete - HASH_FIND(hh, expr_rt->item_hash, &item_id, sizeof(long long), item); + item = (struct maat_item *)rcu_hash_find(expr_rt->item_htable, (char *)&item_id, sizeof(item_id)); if (NULL == item) { - return -1; + item = (struct maat_item *)rcu_hash_updating_find(expr_rt->item_htable, (char *)&item_id, sizeof(item_id)); + if (NULL == item) { + return -1; + } } - - u_para = (struct maat_item_inner *)item->user_data; - item->user_data = NULL; - - if (NULL == u_para) { - return -1; - } - HASH_DELETE(hh, expr_rt->item_hash, item); - maat_garbage_bagging(expr_rt->ref_garbage_bin, u_para, maat_item_inner_free); + rcu_hash_del(expr_rt->item_htable, (char *)&item_id, sizeof(item_id)); } else { //add - HASH_FIND(hh, expr_rt->item_hash, &item_id, sizeof(long long), item); + item = (struct maat_item *)rcu_hash_find(expr_rt->item_htable, (char *)&item_id, sizeof(item_id)); if (item) { log_error(expr_rt->logger, MODULE_EXPR, - "[%s:%d] expr runtime add item %llu to item_hash failed, already exist", + "[%s:%d] expr runtime add item(item_id:%lld) to item_htable failed, already exist", + __FUNCTION__, __LINE__, item_id); + return -1; + } + + item = (struct maat_item *)rcu_hash_updating_find(expr_rt->item_htable, (char *)&item_id, sizeof(item_id)); + if (item) { + log_error(expr_rt->logger, MODULE_EXPR, + "[%s:%d] expr runtime add item(item_id:%lld) to item_htable failed, already exist", __FUNCTION__, __LINE__, item_id); return -1; } @@ -762,33 +774,27 @@ int expr_runtime_update(void *expr_runtime, void *expr_schema, } u_para = maat_item_inner_new(expr_item->group_id, item_id, expr_item->district_id); - item = maat_item_new(item_id, expr_item->group_id, u_para); - HASH_ADD(hh, expr_rt->item_hash, item_id, sizeof(long long), item); + item = maat_item_new(item_id, expr_item->group_id, u_para, maat_item_inner_free); + rcu_hash_add(expr_rt->item_htable, (char *)&item_id, sizeof(item_id), item); expr_rule = expr_item_to_expr_rule(expr_item, u_para, expr_rt->logger); expr_item_free(expr_item); if (NULL == expr_rule) { log_error(expr_rt->logger, MODULE_EXPR, - "[%s:%d] transform expr table(table_id:%d) item to expr_rule failed, item_id:%llu", + "[%s:%d] transform expr table(table_id:%d) item(item_id:%lld) to expr_rule failed", __FUNCTION__, __LINE__, schema->table_id, item_id); return -1; } } char *key = (char *)&item_id; - ret = expr_runtime_update_row(expr_rt, key, sizeof(long long), item_id, - expr_rule, is_valid, expr_rt->logger); + ret = expr_runtime_update_row(expr_rt, key, sizeof(item_id), item_id, + expr_rule, is_valid); if (ret < 0) { if (expr_rule != NULL) { expr_rule_free(expr_rule); } return -1; - } else { - if (0 == is_valid) { - expr_rt->rule_num--; - } else { - expr_rt->rule_num++; - } } return 0; @@ -800,30 +806,39 @@ int expr_runtime_commit(void *expr_runtime, const char *table_name) return -1; } - int ret = 0; struct expr_runtime *expr_rt = (struct expr_runtime *)expr_runtime; - void **ex_data_array = NULL; - size_t rule_cnt = rcu_hash_list_updating_data(expr_rt->htable, &ex_data_array); - if (0 == rule_cnt) { - FREE(ex_data_array); + int updating_flag = rcu_hash_is_updating(expr_rt->htable); + if (0 == updating_flag) { return 0; } - and_expr_t *rules = ALLOC(and_expr_t, rule_cnt); - for (size_t i = 0 ; i < rule_cnt; i++) { - rules[i] = *(and_expr_t *)ex_data_array[i]; + rcu_hash_commit(expr_rt->htable); + + struct hs_expr *rules = NULL; + void **ex_data_array = NULL; + size_t rule_cnt = rcu_hash_list(expr_rt->htable, &ex_data_array); + //printf("rcu_hash_commit rule_cnt:%zu\n", rule_cnt); + if (rule_cnt > 0) { + rules = ALLOC(struct hs_expr, rule_cnt); + for (size_t i = 0; i < rule_cnt; i++) { + rules[i] = *(struct hs_expr *)ex_data_array[i]; + //printf("expr_id:%lld\n", rules[i].expr_id); + // for (size_t j = 0; j < rules[i].n_patterns; j++) { + // printf("rules[%zu].patterns[%zu]:%s\n", i, j, rules[i].patterns[j].pat); + // } + } } - - struct adapter_hs *new_adapter_hs = NULL; - struct adapter_hs *old_adapter_hs = NULL; log_info(expr_rt->logger, MODULE_EXPR, "table[%s] committing %zu expr rules for rebuilding adapter_hs engine", table_name, rule_cnt); + + int ret = 0; + struct adapter_hs *new_adapter_hs = NULL; + struct adapter_hs *old_adapter_hs = NULL; new_adapter_hs = adapter_hs_initialize(expr_rt->scan_mode, expr_rt->pattern_type, - expr_rt->n_worker_thread, - rules, rule_cnt, + expr_rt->n_worker_thread, rules, rule_cnt, expr_rt->logger); if (NULL == new_adapter_hs) { log_error(expr_rt->logger, MODULE_EXPR, @@ -834,15 +849,20 @@ int expr_runtime_commit(void *expr_runtime, const char *table_name) old_adapter_hs = expr_rt->hs; expr_rt->hs = new_adapter_hs; - - maat_garbage_bagging(expr_rt->ref_garbage_bin, old_adapter_hs, + if (old_adapter_hs != NULL) { + maat_garbage_bagging(expr_rt->ref_garbage_bin, old_adapter_hs, (void (*)(void*))adapter_hs_destroy); - rcu_hash_commit(expr_rt->htable); - rule_cnt = rcu_hash_updating_count(expr_rt->htable); - assert(rule_cnt == 0); + } + rcu_hash_commit(expr_rt->item_htable); + expr_rt->rule_num = rule_cnt; + + if (rules != NULL) { + FREE(rules); + } - FREE(rules); - FREE(ex_data_array); + if (ex_data_array != NULL) { + FREE(ex_data_array); + } return ret; } @@ -879,7 +899,7 @@ int expr_runtime_scan(struct expr_runtime *expr_rt, int thread_id, const char *d long long hit_item_ids[MAX_SCANNER_HIT_ITEM_NUM]; struct maat_item_inner *item = NULL; int real_hit_item_cnt = 0; - int district_id = state->district_id; + long long district_id = state->district_id; memset(hit_item_ids, 0, sizeof(hit_item_ids)); @@ -891,7 +911,7 @@ int expr_runtime_scan(struct expr_runtime *expr_rt, int thread_id, const char *d } size_t group_hit_cnt = 0; - ret = maat_compile_state_update(expr_rt->item_hash, vtable_id, hit_item_ids, + ret = maat_compile_state_update(expr_rt->item_htable, vtable_id, hit_item_ids, real_hit_item_cnt, &group_hit_cnt, state); if (ret < 0) { return -1; @@ -952,7 +972,7 @@ int expr_runtime_stream_scan(struct expr_runtime *expr_rt, const char *data, siz } size_t group_hit_cnt = 0; - ret = maat_compile_state_update(expr_rt->item_hash, vtable_id, hit_item_ids, + ret = maat_compile_state_update(expr_rt->item_htable, vtable_id, hit_item_ids, n_hit_item, &group_hit_cnt, state); if (ret < 0) { return -1; diff --git a/src/maat_flag.c b/src/maat_flag.c index 7bda84e..689a003 100644 --- a/src/maat_flag.c +++ b/src/maat_flag.c @@ -37,20 +37,16 @@ struct flag_schema { struct flag_item { long long item_id; long long group_id; - int district_id; + long long district_id; long long flag; long long flag_mask; }; struct flag_runtime { struct flag_matcher *matcher; - struct rcu_hash_table *htable; - + struct rcu_hash_table *htable; //store flag rule for rebuild flag_matcher instance + struct rcu_hash_table *item_htable; //store this flag table's all maat_item which will be used in flag_runtime_scan uint32_t rule_num; - - struct maat_item *item_hash; - void (*item_user_data_free)(void *); - struct maat_garbage_bin *ref_garbage_bin; struct log_handle *logger; @@ -143,7 +139,7 @@ void *flag_runtime_new(void *flag_schema, int max_thread_num, struct flag_runtime *flag_rt = ALLOC(struct flag_runtime, 1); flag_rt->htable = rcu_hash_new(flag_ex_data_free); - flag_rt->item_user_data_free = maat_item_inner_free; + flag_rt->item_htable = rcu_hash_new(maat_item_free); flag_rt->ref_garbage_bin = garbage_bin; flag_rt->logger = logger; @@ -165,10 +161,14 @@ void flag_runtime_free(void *flag_runtime) flag_rt->htable = NULL; } - struct maat_item *item = NULL, *tmp = NULL; - HASH_ITER(hh, flag_rt->item_hash, item, tmp) { - HASH_DELETE(hh, flag_rt->item_hash, item); - maat_item_free(item, flag_rt->item_user_data_free); + if (flag_rt->item_htable != NULL) { + rcu_hash_free(flag_rt->item_htable); + flag_rt->item_htable = NULL; + } + + if (flag_rt->matcher != NULL) { + flag_matcher_free(flag_rt->matcher); + flag_rt->matcher = NULL; } if (flag_rt->hit_cnt != NULL) { @@ -193,21 +193,34 @@ int flag_runtime_update_row(struct flag_runtime *flag_rt, char *key, size_t key_ //delete data = rcu_hash_find(flag_rt->htable, key, key_len); if (NULL == data) { - log_error(flag_rt->logger, MODULE_FLAG, - "[%s:%d] the key of flag rule not exist, can't be deleted, item_id:%llu", + data = rcu_hash_updating_find(flag_rt->htable, key, key_len); + if (NULL == data) { + log_error(flag_rt->logger, MODULE_FLAG, + "[%s:%d] the key of flag rule(rule_id:%lld) not exist, can't be deleted from flag runtime htable", __FUNCTION__, __LINE__, item_id); - return -1; + return -1; + } } + rcu_hash_del(flag_rt->htable, key, key_len); } else { //add data = rcu_hash_find(flag_rt->htable, key, key_len); if (data != NULL) { log_error(flag_rt->logger, MODULE_FLAG, - "[%s:%d] the key of flag rule already exist, can't be added, item_id:%llu", + "[%s:%d] the key of flag rule(rule_id:%lld) already exist, can't be added to flag runtime htable", __FUNCTION__, __LINE__, item_id); return -1; } + + data = rcu_hash_updating_find(flag_rt->htable, key, key_len); + if (data != NULL) { + log_error(flag_rt->logger, MODULE_FLAG, + "[%s:%d] the key of flag rule(rule_id:%lld) already exist, can't be added to flag runtime htable", + __FUNCTION__, __LINE__, item_id); + return -1; + } + rcu_hash_add(flag_rt->htable, key, key_len, (void *)rule); } @@ -334,26 +347,29 @@ int flag_runtime_update(void *flag_runtime, void *flag_schema, return -1; } else if (0 == is_valid) { //delete - HASH_FIND(hh, flag_rt->item_hash, &item_id, sizeof(long long), item); + item = (struct maat_item *)rcu_hash_find(flag_rt->item_htable, (char *)&item_id, sizeof(item_id)); if (NULL == item) { + item = (struct maat_item *)rcu_hash_updating_find(flag_rt->item_htable, (char *)&item_id, sizeof(item_id)); + if (NULL == item) { + return -1; + } + } + + rcu_hash_del(flag_rt->item_htable, (char *)&item_id, sizeof(item_id)); + } else { + //add + item = (struct maat_item *)rcu_hash_find(flag_rt->item_htable, (char *)&item_id, sizeof(item_id)); + if (item) { + log_error(flag_rt->logger, MODULE_FLAG, + "[%s:%d] flag runtime add item(item_id:%lld) to item_htable failed, already exist", + __FUNCTION__, __LINE__, item_id); return -1; } - u_para = (struct maat_item_inner *)item->user_data; - item->user_data = NULL; - - if (NULL == u_para) { - return -1; - } - - HASH_DELETE(hh, flag_rt->item_hash, item); - maat_garbage_bagging(flag_rt->ref_garbage_bin, u_para, maat_item_inner_free); - } else { - //add - HASH_FIND(hh, flag_rt->item_hash, &item_id, sizeof(long long), item); + item = (struct maat_item *)rcu_hash_updating_find(flag_rt->item_htable, (char *)&item_id, sizeof(item_id)); if (item) { log_error(flag_rt->logger, MODULE_FLAG, - "[%s:%d] flag runtime add item %d to item_hash failed, already exist", + "[%s:%d] flag runtime add item(item_id:%lld) to item_htable failed, already exist", __FUNCTION__, __LINE__, item_id); return -1; } @@ -364,14 +380,14 @@ int flag_runtime_update(void *flag_runtime, void *flag_schema, } u_para = maat_item_inner_new(flag_item->group_id, item_id, flag_item->district_id); - item = maat_item_new(item_id, flag_item->group_id, u_para); - HASH_ADD(hh, flag_rt->item_hash, item_id, sizeof(long long), item); + item = maat_item_new(item_id, flag_item->group_id, u_para, maat_item_inner_free); + rcu_hash_add(flag_rt->item_htable, (char *)&(item_id), sizeof(item_id), item); flag_rule = flag_item_to_flag_rule(flag_item, u_para); flag_item_free(flag_item); if (NULL == flag_rule) { log_error(flag_rt->logger, MODULE_FLAG, - "[%s:%d] transform flag table(table_id:%d) item to flag_rule failed, item_id:%d", + "[%s:%d] transform flag table(table_id:%d) item(item_id:%lld) to flag_rule failed", __FUNCTION__, __LINE__, schema->table_id, item_id); return -1; } @@ -402,28 +418,32 @@ int flag_runtime_commit(void *flag_runtime, const char *table_name) return -1; } - int ret = 0; struct flag_runtime *flag_rt = (struct flag_runtime *)flag_runtime; - void **ex_data_array = NULL; - size_t rule_cnt = rcu_hash_list_updating_data(flag_rt->htable, &ex_data_array); - if (0 == rule_cnt) { - FREE(ex_data_array); + int updating_flag = rcu_hash_is_updating(flag_rt->htable); + if (0 == updating_flag) { return 0; } - struct flag_rule *rules = ALLOC(struct flag_rule, rule_cnt); + rcu_hash_commit(flag_rt->htable); - for (size_t i = 0; i < rule_cnt; i++) { - rules[i] = *(struct flag_rule *)ex_data_array[i]; + struct flag_rule *rules = NULL; + void **ex_data_array = NULL; + size_t rule_cnt = rcu_hash_list(flag_rt->htable, &ex_data_array); + if (rule_cnt > 0) { + rules = ALLOC(struct flag_rule, rule_cnt); + for (size_t i = 0; i < rule_cnt; i++) { + rules[i] = *(struct flag_rule *)ex_data_array[i]; + } } - struct flag_matcher *new_flag_matcher = NULL; - struct flag_matcher *old_flag_matcher = NULL; - log_info(flag_rt->logger, MODULE_FLAG, "table[%s] committing %zu flag rules for rebuilding flag_matcher engine", table_name, rule_cnt); + + int ret = 0; + struct flag_matcher *new_flag_matcher = NULL; + struct flag_matcher *old_flag_matcher = NULL; new_flag_matcher = flag_matcher_new(rules, rule_cnt); if (NULL == new_flag_matcher) { log_error(flag_rt->logger, MODULE_FLAG, @@ -434,15 +454,20 @@ int flag_runtime_commit(void *flag_runtime, const char *table_name) old_flag_matcher = flag_rt->matcher; flag_rt->matcher = new_flag_matcher; - - maat_garbage_bagging(flag_rt->ref_garbage_bin, old_flag_matcher, + if (old_flag_matcher != NULL) { + maat_garbage_bagging(flag_rt->ref_garbage_bin, old_flag_matcher, (void (*)(void*))flag_matcher_free); - rcu_hash_commit(flag_rt->htable); - rule_cnt = rcu_hash_updating_count(flag_rt->htable); - assert(rule_cnt == 0); + } + rcu_hash_commit(flag_rt->item_htable); + flag_rt->rule_num = rule_cnt; + + if (rules != NULL) { + FREE(rules); + } - FREE(rules); - FREE(ex_data_array); + if (ex_data_array != NULL) { + FREE(ex_data_array); + } return ret; } @@ -473,7 +498,7 @@ int flag_runtime_scan(struct flag_runtime *flag_rt, int thread_id, long long hit_item_ids[MAX_SCANNER_HIT_ITEM_NUM]; struct maat_item_inner *item = NULL; int real_hit_item_cnt = 0; - int district_id = state->district_id; + long long district_id = state->district_id; memset(hit_item_ids, 0, sizeof(hit_item_ids)); @@ -485,7 +510,7 @@ int flag_runtime_scan(struct flag_runtime *flag_rt, int thread_id, } size_t group_hit_cnt = 0; - int ret = maat_compile_state_update(flag_rt->item_hash, vtable_id, hit_item_ids, + int ret = maat_compile_state_update(flag_rt->item_htable, vtable_id, hit_item_ids, n_hit_item, &group_hit_cnt, state); if (ret < 0) { return -1; diff --git a/src/maat_fqdn_plugin.c b/src/maat_fqdn_plugin.c index 4fbd380..1d81b3f 100644 --- a/src/maat_fqdn_plugin.c +++ b/src/maat_fqdn_plugin.c @@ -112,7 +112,7 @@ void fqdn_plugin_schema_free(void *fqdn_plugin_schema) schema->ex_schema = NULL; } - FREE(schema); + free(schema); } /* fqdn plugin table ex data API */ @@ -162,8 +162,8 @@ void *fqdn_plugin_runtime_new(void *fqdn_plugin_schema, int max_thread_num, struct fqdn_plugin_schema *schema = (struct fqdn_plugin_schema *)fqdn_plugin_schema; struct fqdn_plugin_runtime *fqdn_plugin_rt = ALLOC(struct fqdn_plugin_runtime, 1); - fqdn_plugin_rt->ex_data_rt = ex_data_runtime_new(schema->table_id, ex_data_container_free, - logger); + fqdn_plugin_rt->ex_data_rt = ex_data_runtime_new(schema->table_id, ex_container_free, + garbage_bin, logger); fqdn_plugin_rt->ref_garbage_bin = garbage_bin; fqdn_plugin_rt->logger = logger; @@ -319,7 +319,7 @@ int fqdn_plugin_runtime_update_row(struct fqdn_plugin_runtime *fqdn_plugin_rt, } else { // add void *ex_data = ex_data_runtime_row2ex_data(ex_data_rt, row, key, key_len); - struct ex_data_container *ex_container = ex_data_container_new(ex_data, (void *)fqdn_plugin_rule); + struct ex_container *ex_container = ex_container_new(ex_data, (void *)fqdn_plugin_rule); ret = ex_data_runtime_add_ex_container(ex_data_rt, key, key_len, ex_container); if (ret < 0) { return -1; @@ -389,32 +389,39 @@ int fqdn_plugin_runtime_commit(void *fqdn_plugin_runtime, const char *table_name return -1; } - int ret = 0; - struct ex_data_container **ex_container = NULL; struct fqdn_plugin_runtime *fqdn_plugin_rt = (struct fqdn_plugin_runtime *)fqdn_plugin_runtime; struct ex_data_runtime *ex_data_rt = fqdn_plugin_rt->ex_data_rt; + if (NULL == ex_data_rt) { + return -1; + } - size_t rule_cnt = ex_data_runtime_list_updating_ex_container(ex_data_rt, &ex_container); - if (0 == rule_cnt) { - FREE(ex_container); + int updating_flag = ex_data_runtime_is_updating(ex_data_rt); + if (0 == updating_flag) { return 0; } - struct FQDN_rule *rules = ALLOC(struct FQDN_rule, rule_cnt); + ex_data_runtime_commit(ex_data_rt); - for (size_t i = 0; i < rule_cnt; i++) { - rules[i] = *(struct FQDN_rule *)ex_container[i]->custom_data; - assert(rules[i].user_tag == ex_container[i] || rules[i].user_tag == NULL); - rules[i].user_tag = ex_container[i]; + struct FQDN_rule *rules = NULL; + struct ex_container **ex_container = NULL; + size_t rule_cnt = ex_data_runtime_list_ex_container(ex_data_rt, &ex_container); + if (rule_cnt > 0) { + rules = ALLOC(struct FQDN_rule, rule_cnt); + for (size_t i = 0; i < rule_cnt; i++) { + rules[i] = *(struct FQDN_rule *)ex_container[i]->custom_data; + assert(rules[i].user_tag == ex_container[i] || rules[i].user_tag == NULL); + rules[i].user_tag = ex_container[i]; + } } - struct FQDN_engine *new_fqdn_engine = NULL; - struct FQDN_engine *old_fqdn_engine = NULL; - log_info(fqdn_plugin_rt->logger, MODULE_FQDN_PLUGIN, "table[%s] committing %zu fqdn_plugin rules for rebuilding FQDN engine", table_name, rule_cnt); + int ret = 0; + struct FQDN_engine *new_fqdn_engine = NULL; + struct FQDN_engine *old_fqdn_engine = NULL; + new_fqdn_engine = FQDN_engine_new(rules, rule_cnt); if (NULL == new_fqdn_engine) { log_error(fqdn_plugin_rt->logger, MODULE_FQDN_PLUGIN, @@ -425,12 +432,20 @@ int fqdn_plugin_runtime_commit(void *fqdn_plugin_runtime, const char *table_name old_fqdn_engine = fqdn_plugin_rt->engine; fqdn_plugin_rt->engine = new_fqdn_engine; - maat_garbage_bagging(fqdn_plugin_rt->ref_garbage_bin, old_fqdn_engine, - (void (*)(void*))FQDN_engine_free); - ex_data_runtime_commit(ex_data_rt); + if (old_fqdn_engine != NULL) { + maat_garbage_bagging(fqdn_plugin_rt->ref_garbage_bin, old_fqdn_engine, + (void (*)(void*))FQDN_engine_free); + } + + fqdn_plugin_rt->rule_num = rule_cnt; - FREE(rules); - FREE(ex_container); + if (rules != NULL) { + FREE(rules); + } + + if (ex_container != NULL) { + FREE(ex_container); + } return ret; } @@ -466,7 +481,7 @@ int fqdn_plugin_runtime_get_ex_data(void *fqdn_plugin_runtime, const char *query int n_result = FQDN_engine_search(fqdn_plugin_rt->engine, query_fqdn, strlen(query_fqdn), results, n_ex_data); for (int i = 0; i < n_result; i++) { ex_data_array[i] = ex_data_runtime_get_ex_data_by_container(fqdn_plugin_rt->ex_data_rt, - (struct ex_data_container *)results[i].user_tag); + (struct ex_container *)results[i].user_tag); } return n_result; diff --git a/src/maat_group.c b/src/maat_group.c index a77a256..434863f 100644 --- a/src/maat_group.c +++ b/src/maat_group.c @@ -149,8 +149,8 @@ void *group2group_runtime_new(void *g2g_schema, int max_thread_num, void group_vertex_free(struct maat_group *group) { - free(group->top_group_ids); - free(group); + FREE(group->top_group_ids); + FREE(group); } void maat_group_topology_free(struct maat_group_topology *group_topo) @@ -165,6 +165,7 @@ void maat_group_topology_free(struct maat_group_topology *group_topo) assert(group_topo->hash_group_by_id == NULL); igraph_destroy(&group_topo->group_graph); + FREE(group_topo); } void group2group_runtime_free(void *g2g_runtime) @@ -175,12 +176,14 @@ void group2group_runtime_free(void *g2g_runtime) struct group2group_runtime *g2g_rt = (struct group2group_runtime *)g2g_runtime; + pthread_rwlock_wrlock(&g2g_rt->rwlock); if (g2g_rt->group_topo != NULL) { maat_group_topology_free(g2g_rt->group_topo); + g2g_rt->group_topo = NULL; } - - //pthread_rwlock_unlock(&g2g_rt->rwlock); + pthread_rwlock_unlock(&g2g_rt->rwlock); pthread_rwlock_destroy(&g2g_rt->rwlock); + FREE(g2g_rt); } diff --git a/src/maat_interval.c b/src/maat_interval.c index fc3085a..62b2c8c 100644 --- a/src/maat_interval.c +++ b/src/maat_interval.c @@ -34,20 +34,16 @@ struct interval_schema { struct interval_item { long long item_id; long long group_id; - int district_id; + long long district_id; int low_bound; int up_bound; }; struct interval_runtime { struct interval_matcher *matcher; - struct rcu_hash_table *htable; - + struct rcu_hash_table *htable; //store interval rule for rebuild interval_matcher instance + struct rcu_hash_table *item_htable; //store this interval table's all maat_item which will be used in interval_runtime_scan uint32_t rule_num; - - struct maat_item *item_hash; - void (*item_user_data_free)(void *); - struct maat_garbage_bin *ref_garbage_bin; struct log_handle *logger; @@ -140,7 +136,7 @@ void *interval_runtime_new(void *interval_schema, int max_thread_num, struct interval_runtime *interval_rt = ALLOC(struct interval_runtime, 1); interval_rt->htable = rcu_hash_new(interval_ex_data_free); - interval_rt->item_user_data_free = maat_item_inner_free; + interval_rt->item_htable = rcu_hash_new(maat_item_free); interval_rt->ref_garbage_bin = garbage_bin; interval_rt->logger = logger; @@ -162,10 +158,14 @@ void interval_runtime_free(void *interval_runtime) interval_rt->htable = NULL; } - struct maat_item *item = NULL, *tmp = NULL; - HASH_ITER(hh, interval_rt->item_hash, item, tmp) { - HASH_DELETE(hh, interval_rt->item_hash, item); - maat_item_free(item, interval_rt->item_user_data_free); + if (interval_rt->item_htable != NULL) { + rcu_hash_free(interval_rt->item_htable); + interval_rt->item_htable = NULL; + } + + if (interval_rt->matcher != NULL) { + interval_matcher_free(interval_rt->matcher); + interval_rt->matcher = NULL; } if (interval_rt->hit_cnt != NULL) { @@ -285,21 +285,34 @@ int interval_runtime_update_row(struct interval_runtime *interval_rt, char *key, //delete data = rcu_hash_find(interval_rt->htable, key, key_len); if (NULL == data) { - log_error(interval_rt->logger, MODULE_INTERVAL, - "[%s:%d] the key of interval rule not exist, can't be deleted, item_id:%llu", + data = rcu_hash_updating_find(interval_rt->htable, key, key_len); + if (NULL == data) { + log_error(interval_rt->logger, MODULE_INTERVAL, + "[%s:%d] the key of interval rule(rule_id:%lld) not exist, can't be deleted from interval runtime htable", __FUNCTION__, __LINE__, item_id); - return -1; + return -1; + } } + rcu_hash_del(interval_rt->htable, key, key_len); } else { //add data = rcu_hash_find(interval_rt->htable, key, key_len); if (data != NULL) { log_error(interval_rt->logger, MODULE_INTERVAL, - "[%s:%d] the key of interval rule already exist, can't be added, item_id:%llu", + "[%s:%d] the key of interval rule(rule_id:%lld) already exist, can't be added to interval runtime htable", __FUNCTION__, __LINE__, item_id); return -1; } + + data = rcu_hash_updating_find(interval_rt->htable, key, key_len); + if (data != NULL) { + log_error(interval_rt->logger, MODULE_INTERVAL, + "[%s:%d] the key of interval rule(rule_id:%lld) already exist, can't be added to interval runtime htable", + __FUNCTION__, __LINE__, item_id); + return -1; + } + rcu_hash_add(interval_rt->htable, key, key_len, (void *)rule); } @@ -330,26 +343,33 @@ int interval_runtime_update(void *interval_runtime, void *interval_schema, return -1; } else if (0 == is_valid) { //delete - HASH_FIND(hh, interval_rt->item_hash, &item_id, sizeof(long long), item); + item = (struct maat_item *)rcu_hash_find(interval_rt->item_htable, + (char *)&item_id, sizeof(item_id)); if (NULL == item) { + item = (struct maat_item *)rcu_hash_updating_find(interval_rt->item_htable, + (char *)&item_id, sizeof(item_id)); + if (NULL == item) { + return -1; + } + } + + rcu_hash_del(interval_rt->item_htable, (char *)&item_id, sizeof(item_id)); + } else { + //add + item = (struct maat_item *)rcu_hash_find(interval_rt->item_htable, + (char *)&item_id, sizeof(item_id));; + if (item) { + log_error(interval_rt->logger, MODULE_INTERVAL, + "[%s:%d] interval runtime add item(item_id:%lld) to item_htable failed, already exist", + __FUNCTION__, __LINE__, item_id); return -1; } - u_para = (struct maat_item_inner *)item->user_data; - item->user_data = NULL; - - if (NULL == u_para) { - return -1; - } - - HASH_DELETE(hh, interval_rt->item_hash, item); - maat_garbage_bagging(interval_rt->ref_garbage_bin, u_para, maat_item_inner_free); - } else { - //add - HASH_FIND(hh, interval_rt->item_hash, &item_id, sizeof(long long), item); + item = (struct maat_item *)rcu_hash_updating_find(interval_rt->item_htable, + (char *)&item_id, sizeof(item_id));; if (item) { log_error(interval_rt->logger, MODULE_INTERVAL, - "[%s:%d] interval runtime add item %llu to item_hash failed, already exist", + "[%s:%d] interval runtime add item(item_id:%lld) to item_htable failed, already exist", __FUNCTION__, __LINE__, item_id); return -1; } @@ -360,14 +380,14 @@ int interval_runtime_update(void *interval_runtime, void *interval_schema, } u_para = maat_item_inner_new(interval_item->group_id, item_id, interval_item->district_id); - item = maat_item_new(item_id, interval_item->group_id, u_para); - HASH_ADD(hh, interval_rt->item_hash, item_id, sizeof(long long), item); + item = maat_item_new(item_id, interval_item->group_id, u_para, maat_item_inner_free); + rcu_hash_add(interval_rt->item_htable, (char *)&(item_id), sizeof(item_id), item); interval_rule = interval_item_to_interval_rule(interval_item, u_para); interval_item_free(interval_item); if (NULL == interval_rule) { log_error(interval_rt->logger, MODULE_INTERVAL, - "[%s:%d] transform interval table(table_id:%d) item to interval_rule failed, item_id:%llu", + "[%s:%d] transform interval table(table_id:%d) item(item_id:%lld) to interval_rule failed", __FUNCTION__, __LINE__, schema->table_id, item_id); return -1; } @@ -381,12 +401,6 @@ int interval_runtime_update(void *interval_runtime, void *interval_schema, interval_rule = NULL; } return -1; - } else { - if (0 == is_valid) { - interval_rt->rule_num--; - } else { - interval_rt->rule_num++; - } } return 0; @@ -398,29 +412,33 @@ int interval_runtime_commit(void *interval_runtime, const char *table_name) return -1; } - int ret = 0; struct interval_runtime *interval_rt = (struct interval_runtime *)interval_runtime; - void **ex_data_array = NULL; - size_t rule_cnt = rcu_hash_list_updating_data(interval_rt->htable, &ex_data_array); - if (0 == rule_cnt) { - FREE(ex_data_array); + int updating_flag = rcu_hash_is_updating(interval_rt->htable); + if (0 == updating_flag) { return 0; } - struct interval_rule *rules = ALLOC(struct interval_rule, rule_cnt); + rcu_hash_commit(interval_rt->htable); - for (size_t i = 0; i < rule_cnt; i++) { - rules[i] = *(struct interval_rule *)ex_data_array[i]; + void **ex_data_array = NULL; + struct interval_rule *rules = NULL; + size_t rule_cnt = rcu_hash_list(interval_rt->htable, &ex_data_array); + if (rule_cnt > 0) { + rules = ALLOC(struct interval_rule, rule_cnt); + for (size_t i = 0; i < rule_cnt; i++) { + rules[i] = *(struct interval_rule *)ex_data_array[i]; + } } - struct interval_matcher *new_interval_matcher = NULL; - struct interval_matcher *old_interval_matcher = NULL; - log_info(interval_rt->logger, MODULE_INTERVAL, "table[%s] committing %zu interval rules for rebuilding interval_matcher engine", table_name, rule_cnt); + int ret = 0; + struct interval_matcher *new_interval_matcher = NULL; + struct interval_matcher *old_interval_matcher = NULL; + new_interval_matcher = interval_matcher_new(rules, rule_cnt); if (NULL == new_interval_matcher) { log_error(interval_rt->logger, MODULE_INTERVAL, @@ -431,15 +449,20 @@ int interval_runtime_commit(void *interval_runtime, const char *table_name) old_interval_matcher = interval_rt->matcher; interval_rt->matcher = new_interval_matcher; - - maat_garbage_bagging(interval_rt->ref_garbage_bin, old_interval_matcher, - (void (*)(void*))interval_matcher_free); - rcu_hash_commit(interval_rt->htable); - rule_cnt = rcu_hash_updating_count(interval_rt->htable); - assert(rule_cnt == 0); + if (old_interval_matcher != NULL) { + maat_garbage_bagging(interval_rt->ref_garbage_bin, old_interval_matcher, + (void (*)(void*))interval_matcher_free); + } + rcu_hash_commit(interval_rt->item_htable); + interval_rt->rule_num = rule_cnt; - FREE(rules); - FREE(ex_data_array); + if (rules != NULL) { + FREE(rules); + } + + if (ex_data_array != NULL) { + FREE(ex_data_array); + } return ret; } @@ -470,7 +493,7 @@ int interval_runtime_scan(struct interval_runtime *interval_rt, int thread_id, long long hit_item_ids[MAX_SCANNER_HIT_ITEM_NUM]; struct maat_item_inner *item = NULL; int real_hit_item_cnt = 0; - int district_id = state->district_id; + long long district_id = state->district_id; memset(hit_item_ids, 0, sizeof(hit_item_ids)); @@ -482,7 +505,7 @@ int interval_runtime_scan(struct interval_runtime *interval_rt, int thread_id, } size_t group_hit_cnt = 0; - int ret = maat_compile_state_update(interval_rt->item_hash, vtable_id, hit_item_ids, + int ret = maat_compile_state_update(interval_rt->item_htable, vtable_id, hit_item_ids, n_hit_item, &group_hit_cnt, state); if (ret < 0) { return -1; diff --git a/src/maat_ip.c b/src/maat_ip.c index a21b170..a510044 100644 --- a/src/maat_ip.c +++ b/src/maat_ip.c @@ -56,13 +56,9 @@ struct ip_item { struct ip_runtime { struct ip_matcher* ip_matcher; - struct ex_data_runtime* ex_data_rt; - + struct rcu_hash_table* htable; //store ip rule for rebuild ip_matcher instance + struct rcu_hash_table *item_htable; //store this ip table's all maat_item which will be used in ip_runtime_scan uint32_t rule_num; - - struct maat_item *item_hash; - void (*item_user_data_free)(void *); - struct maat_garbage_bin *ref_garbage_bin; struct log_handle *logger; @@ -144,64 +140,6 @@ void ip_schema_free(void *ip_schema) FREE(ip_schema); } -void *ip_runtime_new(void *ip_schema, int max_thread_num, - struct maat_garbage_bin *garbage_bin, - struct log_handle *logger) -{ - if (NULL == ip_schema) { - return NULL; - } - - struct ip_schema *schema = (struct ip_schema *)ip_schema; - struct ip_runtime *ip_rt = ALLOC(struct ip_runtime, 1); - - ip_rt->ex_data_rt = ex_data_runtime_new(schema->table_id, - ex_data_container_free, - logger); - ip_rt->item_user_data_free = maat_item_inner_free; - ip_rt->ref_garbage_bin = garbage_bin; - ip_rt->logger = logger; - - ip_rt->hit_cnt = alignment_int64_array_alloc(max_thread_num); - ip_rt->scan_cnt = alignment_int64_array_alloc(max_thread_num); - - return ip_rt; -} - -void ip_runtime_free(void *ip_runtime) -{ - if (NULL == ip_runtime) { - return; - } - - struct ip_runtime *ip_rt = (struct ip_runtime *)ip_runtime; - if (ip_rt->ip_matcher != NULL) { - ip_matcher_free(ip_rt->ip_matcher); - } - - if (ip_rt->ex_data_rt != NULL) { - ex_data_runtime_free(ip_rt->ex_data_rt); - } - - struct maat_item *item = NULL, *tmp_item = NULL; - HASH_ITER(hh, ip_rt->item_hash, item, tmp_item) { - HASH_DELETE(hh, ip_rt->item_hash, item); - maat_item_free(item, ip_rt->item_user_data_free); - } - - if (ip_rt->hit_cnt != NULL) { - alignment_int64_array_free(ip_rt->hit_cnt); - ip_rt->hit_cnt = NULL; - } - - if (ip_rt->scan_cnt != NULL) { - alignment_int64_array_free(ip_rt->scan_cnt); - ip_rt->scan_cnt = NULL; - } - - FREE(ip_rt); -} - struct ip_item *ip_item_new(const char *line, struct ip_schema *ip_schema, struct log_handle *logger) { @@ -311,6 +249,64 @@ void ip_item_free(struct ip_item *ip_item) FREE(ip_item); } +void ip_ex_data_free(void *user_ctx, void *data) +{ + struct ip_item *item = (struct ip_item *)data; + ip_item_free(item); +} + +void *ip_runtime_new(void *ip_schema, int max_thread_num, + struct maat_garbage_bin *garbage_bin, + struct log_handle *logger) +{ + struct ip_runtime *ip_rt = ALLOC(struct ip_runtime, 1); + + ip_rt->htable = rcu_hash_new(ip_ex_data_free); + ip_rt->item_htable = rcu_hash_new(maat_item_free); + ip_rt->ref_garbage_bin = garbage_bin; + ip_rt->logger = logger; + + ip_rt->hit_cnt = alignment_int64_array_alloc(max_thread_num); + ip_rt->scan_cnt = alignment_int64_array_alloc(max_thread_num); + + return ip_rt; +} + +void ip_runtime_free(void *ip_runtime) +{ + if (NULL == ip_runtime) { + return; + } + + struct ip_runtime *ip_rt = (struct ip_runtime *)ip_runtime; + if (ip_rt->ip_matcher != NULL) { + ip_matcher_free(ip_rt->ip_matcher); + ip_rt->ip_matcher = NULL; + } + + if (ip_rt->htable != NULL) { + rcu_hash_free(ip_rt->htable); + ip_rt->htable = NULL; + } + + if (ip_rt->item_htable != NULL) { + rcu_hash_free(ip_rt->item_htable); + ip_rt->item_htable = NULL; + } + + if (ip_rt->hit_cnt != NULL) { + alignment_int64_array_free(ip_rt->hit_cnt); + ip_rt->hit_cnt = NULL; + } + + if (ip_rt->scan_cnt != NULL) { + alignment_int64_array_free(ip_rt->scan_cnt); + ip_rt->scan_cnt = NULL; + } + + FREE(ip_rt); +} + void ip_item_to_ip_rule(struct ip_item *item, struct ip_rule *rule) { if (IPv4 == item->addr_type) { @@ -327,30 +323,44 @@ void ip_item_to_ip_rule(struct ip_item *item, struct ip_rule *rule) rule->rule_id = item->item_id; } -struct ex_data_runtime *ip_runtime_get_ex_data_rt(struct ip_runtime *ip_rt) -{ - return ip_rt->ex_data_rt; -} - -int ip_runtime_update_row(struct ip_runtime *rt, char *key, size_t key_len, +int ip_runtime_update_row(struct ip_runtime *ip_rt, char *key, size_t key_len, struct ip_item *item, int is_valid) { - int ret = -1; - struct ex_data_runtime *ex_data_rt = rt->ex_data_rt; + void *data = NULL; if (0 == is_valid) { // delete - ret = ex_data_runtime_del_ex_container(ex_data_rt, key, key_len); - if (ret < 0) { - return -1; + data = rcu_hash_find(ip_rt->htable, key, key_len); + if (NULL == data) { + data = rcu_hash_updating_find(ip_rt->htable, key, key_len); + if (NULL == data) { + log_error(ip_rt->logger, MODULE_IP, + "[%s:%d] the key of ip rule(rule_id:%lld) not exist, can't be deleted from ip runtime htable", + __FUNCTION__, __LINE__, item->item_id); + return -1; + } } + + rcu_hash_del(ip_rt->htable, key, key_len); } else { // add - struct ex_data_container *ex_container = ex_data_container_new(NULL, (void *)item); - ret = ex_data_runtime_add_ex_container(ex_data_rt, key, key_len, ex_container); - if (ret < 0) { + data = rcu_hash_find(ip_rt->htable, key, key_len); + if (data != NULL) { + log_error(ip_rt->logger, MODULE_IP, + "[%s:%d] the key of ip rule(rule_id:%lld) already exist, can't be added to ip runtime htable", + __FUNCTION__, __LINE__, item->item_id); return -1; } + + data = rcu_hash_updating_find(ip_rt->htable, key, key_len); + if (data != NULL) { + log_error(ip_rt->logger, MODULE_IP, + "[%s:%d] the key of ip rule(rule_id:%lld) already exist, can't be added to ip runtime htable", + __FUNCTION__, __LINE__, item->item_id); + return -1; + } + + rcu_hash_add(ip_rt->htable, key, key_len, (void *)item); } return 0; @@ -374,31 +384,40 @@ int ip_runtime_update(void *ip_runtime, void *ip_schema, } int is_valid = get_column_value(line, valid_column); + //printf(" item_id:%lld is_valid:%d\n", item_id, is_valid); if (is_valid < 0) { return -1; } else if (0 == is_valid) { //delete - HASH_FIND(hh, ip_rt->item_hash, &item_id, sizeof(long long), item); + item = (struct maat_item *)rcu_hash_find(ip_rt->item_htable, + (char *)&item_id, + sizeof(item_id)); if (NULL == item) { - return -1; + item = (struct maat_item *)rcu_hash_updating_find(ip_rt->item_htable, + (char *)&item_id, + sizeof(item_id)); + if (NULL == item) { + return -1; + } } - - u_para = (struct maat_item_inner *)item->user_data; - item->user_data = NULL; - - if (NULL == u_para) { - return -1; - } - HASH_DELETE(hh, ip_rt->item_hash, item); - maat_garbage_bagging(ip_rt->ref_garbage_bin, u_para, - (void (*)(void *))maat_item_inner_free); + rcu_hash_del(ip_rt->item_htable, (char *)&item_id, sizeof(item_id)); } else { //add - HASH_FIND(hh, ip_rt->item_hash, &item_id, sizeof(long long), item); + item = (struct maat_item *)rcu_hash_find(ip_rt->item_htable, + (char *)&item_id, sizeof(item_id));; if (item) { log_error(ip_rt->logger, MODULE_IP, - "[%s:%d] ip runtime add item %llu to item_hash failed, already exist", + "[%s:%d] ip runtime add item(item_id:%lld) to item_htable failed, already exist", + __FUNCTION__, __LINE__, item_id); + return -1; + } + + item = (struct maat_item *)rcu_hash_updating_find(ip_rt->item_htable, + (char *)&item_id, sizeof(item_id));; + if (item) { + log_error(ip_rt->logger, MODULE_IP, + "[%s:%d] ip runtime add item(item_id:%lld) to item_htable failed, already exist", __FUNCTION__, __LINE__, item_id); return -1; } @@ -409,8 +428,8 @@ int ip_runtime_update(void *ip_runtime, void *ip_schema, } u_para = maat_item_inner_new(ip_item->group_id, item_id, 0); - item = maat_item_new(item_id, ip_item->group_id, u_para); - HASH_ADD(hh, ip_rt->item_hash, item_id, sizeof(long long), item); + item = maat_item_new(item_id, ip_item->group_id, u_para, maat_item_inner_free); + rcu_hash_add(ip_rt->item_htable, (char *)&(item_id), sizeof(item_id), item); } char *key = (char *)&item_id; @@ -421,12 +440,6 @@ int ip_runtime_update(void *ip_runtime, void *ip_schema, ip_item = NULL; } return -1; - } else { - if (0 == is_valid) { - ip_rt->rule_num--; - } else { - ip_rt->rule_num++; - } } return 0; @@ -438,31 +451,36 @@ int ip_runtime_commit(void *ip_runtime, const char *table_name) return -1; } - int ret = 0; - struct ex_data_container **ex_container = NULL; struct ip_runtime *ip_rt = (struct ip_runtime *)ip_runtime; - struct ex_data_runtime *ex_data_rt = ip_rt->ex_data_rt; - size_t rule_cnt = ex_data_runtime_list_updating_ex_container(ex_data_rt, &ex_container); - if (0 == rule_cnt) { - FREE(ex_container); + int updating_flag = rcu_hash_is_updating(ip_rt->htable); + if (0 == updating_flag) { return 0; } - struct ip_rule *rules = ALLOC(struct ip_rule, rule_cnt); + rcu_hash_commit(ip_rt->htable); - for (size_t i = 0; i < rule_cnt; i++) { - struct ip_item *item = (struct ip_item *)ex_container[i]->custom_data; - ip_item_to_ip_rule(item, &rules[i]); + struct ip_rule *rules = NULL; + void **ex_data_array = NULL; + size_t rule_cnt = rcu_hash_list(ip_rt->htable, &ex_data_array); + if (rule_cnt > 0) { + rules = ALLOC(struct ip_rule, rule_cnt); + for (size_t i = 0; i < rule_cnt; i++) { + struct ip_item *item = (struct ip_item *)ex_data_array[i]; + //printf(" item_id:%lld\n", item->item_id); + ip_item_to_ip_rule(item, &rules[i]); + } } - struct ip_matcher *new_ip_matcher = NULL; - struct ip_matcher *old_ip_matcher = NULL; - size_t mem_used = 0; - log_info(ip_rt->logger, MODULE_IP, "table[%s] committing %zu ip rules for rebuilding ip_matcher engine", table_name, rule_cnt); + + int ret = 0; + size_t mem_used = 0; + struct ip_matcher *new_ip_matcher = NULL; + struct ip_matcher *old_ip_matcher = NULL; + new_ip_matcher = ip_matcher_new(rules, rule_cnt, &mem_used, ip_rt->logger); if (NULL == new_ip_matcher) { log_error(ip_rt->logger, MODULE_IP, @@ -473,12 +491,20 @@ int ip_runtime_commit(void *ip_runtime, const char *table_name) old_ip_matcher = ip_rt->ip_matcher; ip_rt->ip_matcher = new_ip_matcher; - maat_garbage_bagging(ip_rt->ref_garbage_bin, old_ip_matcher, - (void (*)(void*))ip_matcher_free); - ex_data_runtime_commit(ex_data_rt); - - FREE(rules); - FREE(ex_container); + if (old_ip_matcher != NULL) { + maat_garbage_bagging(ip_rt->ref_garbage_bin, old_ip_matcher, + (void (*)(void*))ip_matcher_free); + } + rcu_hash_commit(ip_rt->item_htable); + ip_rt->rule_num = rule_cnt; + + if (rules != NULL) { + FREE(rules); + } + + if (ex_data_array != NULL) { + FREE(ex_data_array); + } return ret; } @@ -525,7 +551,7 @@ int ip_runtime_scan(struct ip_runtime *ip_rt, int thread_id, int ip_type, } size_t group_hit_cnt = 0; - int ret = maat_compile_state_update(ip_rt->item_hash, vtable_id, hit_item_ids, n_hit_item, + int ret = maat_compile_state_update(ip_rt->item_htable, vtable_id, hit_item_ids, n_hit_item, &group_hit_cnt, state); if (ret < 0) { return -1; diff --git a/src/maat_ip_plugin.c b/src/maat_ip_plugin.c index ea31189..449a844 100644 --- a/src/maat_ip_plugin.c +++ b/src/maat_ip_plugin.c @@ -128,7 +128,7 @@ void ip_plugin_schema_free(void *ip_plugin_schema) schema->ex_schema = NULL; } - FREE(schema); + free(schema); } struct ex_data_schema *ip_plugin_table_get_ex_data_schema(void *ip_plugin_schema) @@ -326,7 +326,7 @@ int ip_plugin_runtime_update_row(struct ip_plugin_runtime *ip_plugin_rt, } else { // add void *ex_data = ex_data_runtime_row2ex_data(ex_data_rt, row, key, key_len); - struct ex_data_container *ex_container = ex_data_container_new(ex_data, (void *)ip_plugin_rule); + struct ex_container *ex_container = ex_container_new(ex_data, (void *)ip_plugin_rule); ret = ex_data_runtime_add_ex_container(ex_data_rt, key, key_len, ex_container); if (ret < 0) { return -1; @@ -347,8 +347,8 @@ void *ip_plugin_runtime_new(void *ip_plugin_schema, int max_thread_num, struct ip_plugin_schema *schema = (struct ip_plugin_schema *)ip_plugin_schema; struct ip_plugin_runtime *ip_plugin_rt = ALLOC(struct ip_plugin_runtime, 1); - ip_plugin_rt->ex_data_rt = ex_data_runtime_new(schema->table_id, - ex_data_container_free, logger); + ip_plugin_rt->ex_data_rt = ex_data_runtime_new(schema->table_id, ex_container_free, + garbage_bin, logger); ip_plugin_rt->ref_garbage_bin = garbage_bin; ip_plugin_rt->logger = logger; @@ -435,32 +435,40 @@ int ip_plugin_runtime_commit(void *ip_plugin_runtime, const char *table_name) return -1; } - int ret = 0; - struct ex_data_container **ex_container = NULL; struct ip_plugin_runtime *ip_plugin_rt = (struct ip_plugin_runtime *)ip_plugin_runtime; struct ex_data_runtime *ex_data_rt = ip_plugin_rt->ex_data_rt; + if (NULL == ex_data_rt) { + return -1; + } - size_t rule_cnt = ex_data_runtime_list_updating_ex_container(ex_data_rt, &ex_container); - if (0 == rule_cnt) { - FREE(ex_container); + int updating_flag = ex_data_runtime_is_updating(ex_data_rt); + if (0 == updating_flag) { return 0; } - - struct ip_rule *rules = ALLOC(struct ip_rule, rule_cnt); - for (size_t i = 0; i < rule_cnt; i++) { - rules[i] = *(struct ip_rule *)ex_container[i]->custom_data; - assert(rules[i].user_tag == ex_container[i] || rules[i].user_tag == NULL); - rules[i].user_tag = ex_container[i]; + ex_data_runtime_commit(ex_data_rt); + + struct ip_rule *rules = NULL; + struct ex_container **ex_container = NULL; + size_t rule_cnt = ex_data_runtime_list_ex_container(ex_data_rt, &ex_container); + if (rule_cnt > 0) { + rules = ALLOC(struct ip_rule, rule_cnt); + for (size_t i = 0; i < rule_cnt; i++) { + rules[i] = *(struct ip_rule *)ex_container[i]->custom_data; + assert(rules[i].user_tag == ex_container[i] || rules[i].user_tag == NULL); + rules[i].user_tag = ex_container[i]; + } } - struct ip_matcher *new_ip_matcher = NULL; - struct ip_matcher *old_ip_matcher = NULL; - size_t mem_used = 0; - log_info(ip_plugin_rt->logger, MODULE_IP_PLUGIN, "table[%s] committing %zu ip_plugin rules for rebuilding ip_matcher engine", table_name, rule_cnt); + + int ret = 0; + size_t mem_used = 0; + struct ip_matcher *new_ip_matcher = NULL; + struct ip_matcher *old_ip_matcher = NULL; + new_ip_matcher = ip_matcher_new(rules, rule_cnt, &mem_used, ip_plugin_rt->logger); if (NULL == new_ip_matcher) { log_error(ip_plugin_rt->logger, MODULE_IP_PLUGIN, @@ -471,12 +479,20 @@ int ip_plugin_runtime_commit(void *ip_plugin_runtime, const char *table_name) old_ip_matcher = ip_plugin_rt->ip_matcher; ip_plugin_rt->ip_matcher = new_ip_matcher; - maat_garbage_bagging(ip_plugin_rt->ref_garbage_bin, old_ip_matcher, - (void (*)(void*))ip_matcher_free); - ex_data_runtime_commit(ex_data_rt); + if (old_ip_matcher != NULL) { + maat_garbage_bagging(ip_plugin_rt->ref_garbage_bin, old_ip_matcher, + (void (*)(void*))ip_matcher_free); + } + + ip_plugin_rt->rule_num = rule_cnt; - FREE(rules); - FREE(ex_container); + if (rules != NULL) { + FREE(rules); + } + + if (ex_container != NULL) { + FREE(ex_container); + } return ret; } @@ -518,7 +534,7 @@ int ip_plugin_runtime_get_ex_data(void *ip_plugin_runtime, const struct ip_addr int n_result = ip_matcher_match(ip_plugin_rt->ip_matcher, &ip_data, results, n_ex_data); for (int i = 0; i < n_result; i++) { ex_data_array[i] = ex_data_runtime_get_ex_data_by_container(ip_plugin_rt->ex_data_rt, - (struct ex_data_container *)results[i].tag); + (struct ex_container *)results[i].tag); } return n_result; } \ No newline at end of file diff --git a/src/maat_kv.c b/src/maat_kv.c index 59c8443..990407c 100644 --- a/src/maat_kv.c +++ b/src/maat_kv.c @@ -19,7 +19,7 @@ struct maat_kv_pair { char* key; //must be lower case. size_t keylen; - int val; + long long val; UT_hash_handle hh; }; @@ -35,7 +35,7 @@ void strlowercase(const char* src, size_t src_len, char* dst, size_t dst_sz) } } -struct maat_kv_pair* maat_kv_pair_new(const char* key, size_t keylen, int value) +struct maat_kv_pair* maat_kv_pair_new(const char* key, size_t keylen, long long value) { struct maat_kv_pair *kv = ALLOC(struct maat_kv_pair, 1); @@ -58,7 +58,7 @@ void maat_kv_pair_free(struct maat_kv_pair* kv) struct maat_kv_store* maat_kv_store_new(void) { - struct maat_kv_store* store = ALLOC(struct maat_kv_store, 1); + struct maat_kv_store *store = ALLOC(struct maat_kv_store, 1); return store; } @@ -79,7 +79,7 @@ void maat_kv_store_free(struct maat_kv_store* store) FREE(store); } -int maat_kv_register_unNull(struct maat_kv_store* store, const char* key, size_t keylen, int value) +int maat_kv_register_unNull(struct maat_kv_store* store, const char *key, size_t keylen, long long value) { if (keylen > MAAT_KV_MAX_KEY_LEN) { return -1; @@ -99,7 +99,7 @@ int maat_kv_register_unNull(struct maat_kv_store* store, const char* key, size_t return 1; } -int maat_kv_register(struct maat_kv_store* store, const char* key, int value) +int maat_kv_register(struct maat_kv_store* store, const char *key, long long value) { int ret = 0; ret = maat_kv_register_unNull(store, key, strlen(key), value); @@ -107,7 +107,7 @@ int maat_kv_register(struct maat_kv_store* store, const char* key, int value) return ret; } -int maat_kv_read_unNull(struct maat_kv_store* store, const char* key, size_t keylen, int* value) +int maat_kv_read_unNull(struct maat_kv_store *store, const char* key, size_t keylen, long long *value) { struct maat_kv_pair *kv = NULL; char key_lowercase[MAAT_KV_MAX_KEY_LEN] = {0}; @@ -126,11 +126,37 @@ int maat_kv_read_unNull(struct maat_kv_store* store, const char* key, size_t key } } -int maat_kv_read(struct maat_kv_store * store, const char * key, int * value) +int maat_kv_read(struct maat_kv_store *store, const char * key, long long *value) { return maat_kv_read_unNull(store, key, strlen(key), value); } +int maat_kv_write_unNull(struct maat_kv_store* store, const char* key, size_t keylen, long long value) +{ + struct maat_kv_pair *kv = NULL; + char key_lowercase[MAAT_KV_MAX_KEY_LEN] = {0}; + + if (keylen > MAAT_KV_MAX_KEY_LEN) { + return -1; + } + + strlowercase(key, keylen, key_lowercase, sizeof(key_lowercase)); + HASH_FIND(hh, store->hash, key_lowercase, keylen, kv); + if (kv) { + kv->val = value; + } else { + kv = maat_kv_pair_new(key, keylen, value); + HASH_ADD_KEYPTR(hh, store->hash, kv->key, keylen, kv); + } + + return 1; +} + +int maat_kv_write(struct maat_kv_store *store, const char *key, long long value) +{ + return maat_kv_write_unNull(store, key, strlen(key), value); +} + struct maat_kv_store *maat_kv_store_duplicate(struct maat_kv_store *origin_map) { struct maat_kv_store *target = maat_kv_store_new(); diff --git a/src/maat_plugin.c b/src/maat_plugin.c index ac1f7cc..125aa06 100644 --- a/src/maat_plugin.c +++ b/src/maat_plugin.c @@ -142,7 +142,7 @@ void plugin_schema_free(void *plugin_schema) schema->ex_schema = NULL; } - FREE(schema); + free(schema); } int plugin_table_add_callback(void *plugin_schema, int table_id, @@ -240,7 +240,8 @@ void *plugin_runtime_new(void *plugin_schema, int max_thread_num, struct plugin_schema *schema = (struct plugin_schema *)plugin_schema; struct plugin_runtime *plugin_rt = ALLOC(struct plugin_runtime, 1); plugin_rt->ex_data_rt = ex_data_runtime_new(schema->table_id, - ex_data_container_free, logger); + ex_container_free, + garbage_bin, logger); plugin_rt->ref_garbage_bin = garbage_bin; plugin_rt->logger = logger; @@ -281,7 +282,7 @@ int plugin_runtime_update_row(struct plugin_runtime *plugin_rt, } else { // add void *ex_data = ex_data_runtime_row2ex_data(plugin_rt->ex_data_rt, row, key, key_len); - struct ex_data_container *ex_container = ex_data_container_new(ex_data, NULL); + struct ex_container *ex_container = ex_container_new(ex_data, NULL); ret = ex_data_runtime_add_ex_container(plugin_rt->ex_data_rt, key, key_len, ex_container); if (ret < 0) { return -1; @@ -388,16 +389,18 @@ int plugin_runtime_commit(void *plugin_runtime, const char *table_name) } struct plugin_runtime *plugin_rt = (struct plugin_runtime *)plugin_runtime; - - struct ex_data_container **ex_container = NULL; - size_t rule_cnt = ex_data_runtime_list_updating_ex_container(plugin_rt->ex_data_rt, - &ex_container); - if (rule_cnt == 0) { - FREE(ex_container); + struct ex_data_runtime *ex_data_rt = plugin_rt->ex_data_rt; + if (NULL == ex_data_rt) { + return -1; + } + + int updating_flag = ex_data_runtime_is_updating(ex_data_rt); + if (0 == updating_flag) { return 0; } - ex_data_runtime_commit(plugin_rt->ex_data_rt); + ex_data_runtime_commit(ex_data_rt); + plugin_rt->rule_num = ex_data_runtime_ex_container_count(ex_data_rt); return 0; } diff --git a/src/maat_redis_monitor.c b/src/maat_redis_monitor.c index be85f51..73fe261 100644 --- a/src/maat_redis_monitor.c +++ b/src/maat_redis_monitor.c @@ -65,7 +65,7 @@ void _get_foregin_keys(struct serial_rule *p_rule, int *foreign_columns, &foreign_key_size); if (NULL == p_foreign) { log_error(logger, MODULE_REDIS_MONITOR, - "[%s:%d] Get %s,%lu foreign keys failed: No %dth column", + "[%s:%d] Get %s,%lld foreign keys failed: No %dth column", __FUNCTION__, __LINE__, p_rule->table_name, p_rule->rule_id, foreign_columns[i]); continue; @@ -78,7 +78,7 @@ void _get_foregin_keys(struct serial_rule *p_rule, int *foreign_columns, if (0 != strncmp(p_foreign, foreign_source_prefix, strlen(foreign_source_prefix))) { log_error(logger, MODULE_REDIS_MONITOR, - "[%s:%d] Get %s,%lu foreign key failed: Invalid source prefix %s", + "[%s:%d] Get %s,%lld foreign key failed: Invalid source prefix %s", __FUNCTION__, __LINE__, p_rule->table_name, p_rule->rule_id, p_foreign); continue; } @@ -89,7 +89,7 @@ void _get_foregin_keys(struct serial_rule *p_rule, int *foreign_columns, if (0 != strncmp(p_foreign, foreign_key_prefix, strlen(foreign_key_prefix))) { log_info(logger, MODULE_REDIS_MONITOR, - "[%s:%d] %s, %lu foreign key prefix %s is not recommended", + "[%s:%d] %s, %lld foreign key prefix %s is not recommended", __FUNCTION__, __LINE__, p_rule->table_name, p_rule->rule_id, p_foreign); } @@ -201,7 +201,7 @@ int _get_maat_redis_value(redisContext *c, struct serial_rule *rule_list, int ru ret = maat_cmd_wrap_redis_get_reply(c, &reply); if (ret == REDIS_ERR) { log_error(logger, MODULE_REDIS_MONITOR, - "[%s:%d] Redis GET %s:%s,%lu failed, redis server error", + "[%s:%d] Redis GET %s:%s,%lld failed, redis server error", __FUNCTION__, __LINE__, mr_key_prefix[rule_list[i].op], rule_list[i].table_name, rule_list[i].rule_id); error_happened = 1; @@ -216,7 +216,7 @@ int _get_maat_redis_value(redisContext *c, struct serial_rule *rule_list, int ru failed_cnt++; } else { log_error(logger, MODULE_REDIS_MONITOR, - "[%s:%d] Redis GET %s:%s,%lu failed", + "[%s:%d] Redis GET %s:%s,%lld failed", __FUNCTION__, __LINE__, mr_key_prefix[rule_list[i].op], rule_list[i].table_name, rule_list[i].rule_id); error_happened = 1; @@ -774,7 +774,7 @@ void _get_foreign_conts(redisContext *c, struct serial_rule *rule_list, ret = maat_cmd_wrap_redis_get_reply(c, &reply); if (ret == REDIS_ERR) { log_error(logger, MODULE_REDIS_MONITOR, - "[%s:%d] Get %s,%lu foreign key %s content failed, redis server error", + "[%s:%d] Get %s,%lld foreign key %s content failed, redis server error", __FUNCTION__, __LINE__, rule_list[track[i].rule_idx].table_name, rule_list[track[i].rule_idx].rule_id, @@ -784,7 +784,7 @@ void _get_foreign_conts(redisContext *c, struct serial_rule *rule_list, if (reply->type != REDIS_REPLY_STRING) { log_error(logger, MODULE_REDIS_MONITOR, - "[%s:%d] Get %s,%lu foreign key %s content failed", + "[%s:%d] Get %s,%lld foreign key %s content failed", __FUNCTION__, __LINE__, rule_list[track[i].rule_idx].table_name, rule_list[track[i].rule_idx].rule_id, @@ -993,7 +993,7 @@ void exec_serial_rule(redisContext *c, const char *transaction_list, for (i = 0; i < rule_num; i++) { switch (s_rule[i].op) { case MAAT_OP_ADD: - redisAppendCommand(c, "SET %s:%s,%lu %s", + redisAppendCommand(c, "SET %s:%s,%lld %s", mr_key_prefix[MAAT_OP_ADD], s_rule[i].table_name, s_rule[i].rule_id, @@ -1002,7 +1002,7 @@ void exec_serial_rule(redisContext *c, const char *transaction_list, (*cnt)++; append_cmd_cnt++; //Allowing add duplicated members for rule id recycling. - redisAppendCommand(c, "RPUSH %s ADD,%s,%lu", + redisAppendCommand(c, "RPUSH %s ADD,%s,%lld", transaction_list, s_rule[i].table_name, s_rule[i].rule_id); @@ -1010,7 +1010,7 @@ void exec_serial_rule(redisContext *c, const char *transaction_list, (*cnt)++; append_cmd_cnt++; if (s_rule[i].timeout > 0) { - redisAppendCommand(c, "ZADD %s %lld %s,%lu", + redisAppendCommand(c, "ZADD %s %lld %s,%lld", mr_expire_sset, s_rule[i].timeout, s_rule[i].table_name, @@ -1022,7 +1022,7 @@ void exec_serial_rule(redisContext *c, const char *transaction_list, } break; case MAAT_OP_DEL: - redisAppendCommand(c, "RENAME %s:%s,%lu %s:%s,%lu", + redisAppendCommand(c, "RENAME %s:%s,%lld %s:%s,%lld", mr_key_prefix[MAAT_OP_ADD], s_rule[i].table_name, s_rule[i].rule_id, @@ -1033,7 +1033,7 @@ void exec_serial_rule(redisContext *c, const char *transaction_list, (*cnt)++; append_cmd_cnt++; - redisAppendCommand(c, "EXPIRE %s:%s,%lu %d", + redisAppendCommand(c, "EXPIRE %s:%s,%lld %d", mr_key_prefix[MAAT_OP_DEL], s_rule[i].table_name, s_rule[i].rule_id, @@ -1043,7 +1043,7 @@ void exec_serial_rule(redisContext *c, const char *transaction_list, append_cmd_cnt++; //NX: Don't update already exisiting elements. Always add new elements. - redisAppendCommand(c, "RPUSH %s DEL,%s,%lu", + redisAppendCommand(c, "RPUSH %s DEL,%s,%lld", transaction_list, s_rule[i].table_name, s_rule[i].rule_id); @@ -1052,7 +1052,7 @@ void exec_serial_rule(redisContext *c, const char *transaction_list, append_cmd_cnt++; // Try to remove from expiration sorted set, no matter wheather it exists or not. - redisAppendCommand(c, "ZREM %s %s,%lu", + redisAppendCommand(c, "ZREM %s %s,%lld", mr_expire_sset, s_rule[i].table_name, s_rule[i].rule_id); @@ -1061,7 +1061,7 @@ void exec_serial_rule(redisContext *c, const char *transaction_list, append_cmd_cnt++; // Try to remove from label sorted set, no matter wheather it exists or not. - redisAppendCommand(c, "ZREM %s %s,%lu", + redisAppendCommand(c, "ZREM %s %s,%lld", mr_label_sset, s_rule[i].table_name, s_rule[i].rule_id); @@ -1074,7 +1074,7 @@ void exec_serial_rule(redisContext *c, const char *transaction_list, continue; } //s_rule[i].timeout>0 was checked by caller. - redisAppendCommand(c, "ZADD %s %lld %s,%lu", + redisAppendCommand(c, "ZADD %s %lld %s,%lld", mr_expire_sset, s_rule[i].timeout, s_rule[i].table_name, @@ -1194,7 +1194,7 @@ int maat_cmd_write_rule(redisContext *c, struct serial_rule *s_rule, rule_seq = expected_reply[i].s_rule_seq; log_error(logger, MODULE_REDIS_MONITOR, - "[%s:%d] %s %s %lu failed, rule id maybe conflict or not exist", + "[%s:%d] %s %s %lld failed, rule id maybe conflict or not exist", __FUNCTION__, __LINE__, mr_op_str[s_rule[rule_seq].op], s_rule[rule_seq].table_name, s_rule[rule_seq].rule_id); success_cnt--; @@ -1219,7 +1219,7 @@ error_out: for (i = 0; i < (unsigned int)serial_rule_num; i++) { if (s_rule[i].op == MAAT_OP_RENEW_TIMEOUT) { log_error(logger, MODULE_REDIS_MONITOR, - "[%s:%d] %s %s %lu is not allowed due to lock contention", + "[%s:%d] %s %s %lld is not allowed due to lock contention", __FUNCTION__, __LINE__, mr_op_str[MAAT_OP_RENEW_TIMEOUT], s_rule[i].table_name, s_rule[i].rule_id); } diff --git a/src/maat_rule.c b/src/maat_rule.c index 77a57f2..d822bc5 100644 --- a/src/maat_rule.c +++ b/src/maat_rule.c @@ -31,21 +31,24 @@ #define MODULE_MAAT_RULE module_name_str("maat.rule") -struct maat_item *maat_item_new(long long item_id, long long group_id, void *user_data) +struct maat_item *maat_item_new(long long item_id, long long group_id, void *user_data, + void (*user_data_free)(void *)) { struct maat_item *item = NULL; item = ALLOC(struct maat_item, 1); item->group_id = group_id; item->item_id = item_id; item->user_data = user_data; + item->user_data_free = user_data_free; return item; } -void maat_item_free(struct maat_item *item, void (* item_user_data_free)(void *)) +void maat_item_free(void *user_ctx, void *maat_item) { - if (item_user_data_free && item->user_data) { - item_user_data_free(item->user_data); + struct maat_item *item = (struct maat_item *)maat_item; + if (item->user_data_free && item->user_data) { + item->user_data_free(item->user_data); item->user_data = NULL; } @@ -206,7 +209,7 @@ error: return ret; } -struct maat_item_inner *maat_item_inner_new(long long group_id, long long item_id, int district_id) +struct maat_item_inner *maat_item_inner_new(long long group_id, long long item_id, long long district_id) { struct maat_item_inner *item = ALLOC(struct maat_item_inner, 1); item->magic_num = ITEM_RULE_MAGIC; @@ -245,7 +248,7 @@ struct maat_runtime* maat_runtime_create(long long version, struct maat *maat_in maat_rt->ref_tbl_mgr = maat_instance->tbl_mgr; maat_rt->max_table_num = table_manager_table_count(maat_instance->tbl_mgr); - + maat_rt->sequence_map = maat_kv_store_new(); maat_rt->logger = maat_instance->logger; maat_rt->ref_garbage_bin = maat_instance->garbage_bin; maat_rt->ref_cnt = alignment_int64_array_alloc(maat_instance->nr_worker_thread); @@ -262,6 +265,27 @@ void maat_runtime_commit(struct maat_runtime *maat_rt, struct log_handle *logger maat_rt->last_update_time = time(NULL); } +long long maat_runtime_get_sequence(struct maat_runtime *maat_rt, const char *key) +{ + if (NULL == maat_rt || NULL == key) { + return -1; + } + + long long sequence = 0; + int map_ret = maat_kv_read(maat_rt->sequence_map, key, &sequence); + if (map_ret < 0) { + maat_kv_register(maat_rt->sequence_map, key, sequence); + } else { + sequence++; + int ret = maat_kv_write(maat_rt->sequence_map, key, sequence); + if (ret < 0) { + return -1; + } + } + + return sequence; +} + void maat_runtime_destroy(struct maat_runtime *maat_rt) { if (NULL == maat_rt) { @@ -273,22 +297,46 @@ void maat_runtime_destroy(struct maat_runtime *maat_rt) maat_rt->ref_tbl_mgr = NULL; } + if (maat_rt->sequence_map != NULL) { + maat_kv_store_free(maat_rt->sequence_map); + maat_rt->sequence_map = NULL; + } + + if (maat_rt->ref_cnt != NULL) { + alignment_int64_array_free(maat_rt->ref_cnt); + maat_rt->ref_cnt = NULL; + } + FREE(maat_rt); } void maat_start_cb(long long new_version, int update_type, void *u_param) { struct maat *maat_instance = (struct maat *)u_param; - + maat_instance->new_version = new_version; + if (update_type == MAAT_UPDATE_TYPE_FULL) { maat_instance->creating_maat_rt = maat_runtime_create(new_version, maat_instance); } else { maat_instance->maat_version = new_version; } + struct maat_runtime *maat_rt = NULL; + if (maat_instance->creating_maat_rt != NULL) { + maat_rt = maat_instance->creating_maat_rt; + } else { + maat_rt = maat_instance->maat_rt; + } + size_t table_cnt = table_manager_table_count(maat_instance->tbl_mgr); for (size_t i = 0; i < table_cnt; i++) { enum table_type table_type = table_manager_get_table_type(maat_instance->tbl_mgr, i); + if (table_type == TABLE_TYPE_COMPILE) { + //compile runtime need a reference to maat runtime + void *compile_rt = table_manager_get_runtime(maat_instance->tbl_mgr, i); + compile_runtime_init(compile_rt, maat_rt); + } + if (table_type != TABLE_TYPE_PLUGIN) { continue; } @@ -327,7 +375,7 @@ int maat_update_cb(const char *table_name, const char *line, void *u_param) } else { maat_rt = maat_instance->maat_rt; } - + table_manager_update_runtime(maat_rt->ref_tbl_mgr, table_id, line); return 0; @@ -386,7 +434,12 @@ void maat_finish_cb(void *u_param) "Inc config version %llu load %d entries complete", maat_instance->maat_rt->version, maat_instance->maat_rt->rule_num); + } else { + log_info(maat_instance->logger, MODULE_MAAT_RULE, + "Version %d has no valid rules, plugin callback complete.", maat_instance->maat_version); } + + maat_instance->new_version = INVALID_VERSION; } void *rule_monitor_loop(void *arg) @@ -524,7 +577,8 @@ void *rule_monitor_loop(void *arg) } } + log_handle_destroy(maat_instance->logger); FREE(maat_instance); return NULL; -} +} \ No newline at end of file diff --git a/src/maat_table.c b/src/maat_table.c index c57295d..2bc5a56 100644 --- a/src/maat_table.c +++ b/src/maat_table.c @@ -372,7 +372,7 @@ struct maat_table *maat_table_new(cJSON *json, struct maat_kv_store *reserved_wo goto error; } - ret = maat_kv_read(reserved_word_map, item->valuestring, (int*)&(ptable->table_type)); + ret = maat_kv_read(reserved_word_map, item->valuestring, (long long *)&(ptable->table_type)); if (ret < 0) { log_error(logger, MODULE_TABLE, "[%s:%d] table:%s table_type %s is illegal", @@ -462,22 +462,17 @@ table_manager_create(const char *table_info_path, const char *accept_tags, log_error(logger, MODULE_TABLE, "[%s:%d] invalid json content in %s", __FUNCTION__, __LINE__, table_info_path); - free(json_buff); + FREE(json_buff); + cJSON_Delete(root); return NULL; } - struct maat_kv_store *reserved_word_map = maat_kv_store_new(); - register_reserved_word(reserved_word_map); - struct table_manager *tbl_mgr = ALLOC(struct table_manager, 1); tbl_mgr->n_accept_tag = parse_accept_tag(accept_tags, &tbl_mgr->accept_tags, logger); tbl_mgr->logger = logger; tbl_mgr->tablename2id_map = maat_kv_store_new(); tbl_mgr->ref_garbage_bin = garbage_bin; - int default_compile_table_id = MAX_TABLE_NUM; - int g2g_table_id = MAX_TABLE_NUM; - for (int i = 0; i < json_array_size; i++) { json = cJSON_GetArrayItem(root, i); @@ -485,13 +480,25 @@ table_manager_create(const char *table_info_path, const char *accept_tags, ret = register_tablename2id(json, tbl_mgr->tablename2id_map, logger); if (ret < 0) { log_error(logger, MODULE_TABLE, - "[%s:%d] register_tablename2id failed", - __FUNCTION__, __LINE__); + "[%s:%d] register_tablename2id failed", __FUNCTION__, __LINE__); + FREE(json_buff); + cJSON_Delete(root); + maat_kv_store_free(tbl_mgr->tablename2id_map); + for (int idx = 0; idx < tbl_mgr->n_accept_tag; idx++) { + FREE(tbl_mgr->accept_tags[idx].tag_name); + FREE(tbl_mgr->accept_tags[idx].tag_val); + } + FREE(tbl_mgr); return NULL; } } } + int default_compile_table_id = MAX_TABLE_NUM; + int g2g_table_id = MAX_TABLE_NUM; + struct maat_kv_store *reserved_word_map = maat_kv_store_new(); + register_reserved_word(reserved_word_map); + for (int i = 0; i < json_array_size; i++) { json = cJSON_GetArrayItem(root, i); @@ -534,9 +541,9 @@ table_manager_create(const char *table_info_path, const char *accept_tags, log_info(logger, MODULE_TABLE, "default compile table id: %d", default_compile_table_id); log_info(logger, MODULE_TABLE, "group2group table id: %d", g2g_table_id); + FREE(json_buff); maat_kv_store_free(reserved_word_map); cJSON_Delete(root); - FREE(json_buff); return tbl_mgr; } @@ -645,6 +652,9 @@ void table_manager_destroy(struct table_manager *tbl_mgr) enum table_type table_type = table_manager_get_table_type(tbl_mgr, i); maat_table_schema_free(schema, table_type); tbl_mgr->tbl[i]->schema = NULL; + + maat_table_free(tbl_mgr->tbl[i]); + tbl_mgr->tbl[i] = NULL; } for (size_t i = 0; i < tbl_mgr->n_accept_tag; i++) { @@ -668,7 +678,7 @@ int table_manager_get_table_id(struct table_manager *tbl_mgr, const char *name) return -1; } - int table_id = -1; + long long table_id = -1; int ret = maat_kv_read(tbl_mgr->tablename2id_map, name, &table_id); if (ret < 0) { log_error(tbl_mgr->logger, MODULE_TABLE, @@ -677,7 +687,7 @@ int table_manager_get_table_id(struct table_manager *tbl_mgr, const char *name) return -1; } - return table_id; + return (int)table_id; } enum table_type table_manager_get_table_type(struct table_manager *tbl_mgr, int table_id) @@ -746,14 +756,14 @@ int table_manager_accept_tags_match(struct table_manager *tbl_mgr, const char *t } int table_manager_set_scan_district(struct table_manager *tbl_mgr, const char *district, - size_t district_len, int *district_id) + size_t district_len, long long *district_id) { return maat_kv_read_unNull(tbl_mgr->district_map, district, district_len, district_id); } -int table_manager_get_district_id(struct table_manager *tbl_mgr, const char *district) +long long table_manager_get_district_id(struct table_manager *tbl_mgr, const char *district) { - int district_id = DISTRICT_ANY; + long long district_id = DISTRICT_ANY; int map_ret = maat_kv_read(tbl_mgr->district_map, district, &district_id); if (map_ret < 0) { diff --git a/src/rcu_hash.c b/src/rcu_hash.c index 5df0a43..321b8cd 100644 --- a/src/rcu_hash.c +++ b/src/rcu_hash.c @@ -12,6 +12,7 @@ #include #include #include +#include #include "rcu_hash.h" #include "maat_utils.h" @@ -52,7 +53,7 @@ struct rcu_hash_node { UT_hash_handle hh_b; }; -void rcu_hash_garbage_queue_free(struct rcu_hash_garbage_q* garbage_q) +void rcu_hash_garbage_queue_free(struct rcu_hash_garbage_q *garbage_q) { struct rcu_hash_garbage_bag *p = NULL; @@ -83,7 +84,7 @@ void rcu_hash_node_free(struct rcu_hash_node *node) if (NULL == node) { return; } - + if (node->key != NULL) { FREE(node->key); } @@ -179,28 +180,29 @@ void rcu_hash_add(struct rcu_hash_table *htable, const char *key, return; } - struct rcu_hash_node *tmp = NULL; - struct rcu_hash_node *node = ALLOC(struct rcu_hash_node, 1); - - node->key = (char *)malloc(sizeof(char) * key_len); - memcpy(node->key, key, key_len); - node->key_len = key_len; - node->data = data; - node->htable = htable; - if (!htable->is_updating) { rcu_hash_commit_prepare(htable); } + struct rcu_hash_node *tmp = NULL; if (htable->effective_hash == 'a') { HASH_FIND(hh_b, htable->hashmap_b, key, key_len, tmp); - if (NULL == tmp) { - HASH_ADD_KEYPTR(hh_b, htable->hashmap_b, key, key_len, node); - } } else { HASH_FIND(hh_a, htable->hashmap_a, key, key_len, tmp); - if (NULL == tmp) { - HASH_ADD_KEYPTR(hh_a, htable->hashmap_a, key, key_len, node); + } + + if (NULL == tmp) { + struct rcu_hash_node *node = ALLOC(struct rcu_hash_node, 1); + node->key = ALLOC(char, key_len); + memcpy(node->key, key, key_len); + node->key_len = key_len; + node->data = data; + node->htable = htable; + + if (htable->effective_hash == 'a') { + HASH_ADD_KEYPTR(hh_b, htable->hashmap_b, node->key, node->key_len, node); + } else { + HASH_ADD_KEYPTR(hh_a, htable->hashmap_a, node->key, node->key_len, node); } } } @@ -257,6 +259,28 @@ void *rcu_hash_find(struct rcu_hash_table *htable, const char *key, size_t key_l return NULL; } +void *rcu_hash_updating_find(struct rcu_hash_table *htable, const char *key, size_t key_len) +{ + if (NULL == htable || NULL == key || 0 == key_len) { + return NULL; + } + + struct rcu_hash_node *node = NULL; + if (htable->effective_hash == 'a') { + HASH_FIND(hh_b, htable->hashmap_b, key, key_len, node); + if (node != NULL) { + return node->data; + } + } else { + HASH_FIND(hh_a, htable->hashmap_a, key, key_len, node); + if (node != NULL) { + return node->data; + } + } + + return NULL; +} + size_t rcu_hash_count(struct rcu_hash_table *htable) { if (NULL == htable) { @@ -266,21 +290,17 @@ size_t rcu_hash_count(struct rcu_hash_table *htable) if (htable->effective_hash == 'a') { return HASH_CNT(hh_a, htable->hashmap_a); } else { - return HASH_CNT(hh_b, htable->hashmap_b); + return HASH_CNT(hh_b, htable->hashmap_b); } } -size_t rcu_hash_updating_count(struct rcu_hash_table *htable) +int rcu_hash_is_updating(struct rcu_hash_table *htable) { if (NULL == htable) { return 0; } - if (htable->effective_hash == 'a') { - return HASH_CNT(hh_b, htable->hashmap_b); - } else { - return HASH_CNT(hh_a, htable->hashmap_a); - } + return htable->is_updating; } void rcu_hash_commit(struct rcu_hash_table *htable) @@ -289,10 +309,6 @@ void rcu_hash_commit(struct rcu_hash_table *htable) return; } - if (!htable->is_updating) { - return; - } - struct rcu_hash_node *node = NULL; struct rcu_hash_node *tmp = NULL; @@ -324,23 +340,23 @@ void rcu_hash_commit(struct rcu_hash_table *htable) pthread_mutex_unlock(&htable->update_mutex); } -size_t rcu_hash_list_updating_data(struct rcu_hash_table *htable, void ***data_array) +size_t rcu_hash_list(struct rcu_hash_table *htable, void ***data_array) { size_t i = 0; size_t node_cnt = 0; struct rcu_hash_node *node = NULL, *tmp = NULL; if (htable->effective_hash == 'a') { - node_cnt = HASH_CNT(hh_b, htable->hashmap_b); + node_cnt = HASH_CNT(hh_a, htable->hashmap_a); *data_array = ALLOC(void *, node_cnt); - HASH_ITER(hh_b, htable->hashmap_b, node, tmp) { + HASH_ITER(hh_a, htable->hashmap_a, node, tmp) { (*data_array)[i] = node->data; i++; } } else { - node_cnt = HASH_CNT(hh_a, htable->hashmap_a); + node_cnt = HASH_CNT(hh_b, htable->hashmap_b); *data_array = ALLOC(void *, node_cnt); - HASH_ITER(hh_a, htable->hashmap_a, node, tmp) { + HASH_ITER(hh_b, htable->hashmap_b, node, tmp) { (*data_array)[i] = node->data; i++; } diff --git a/test/adapter_hs_gtest.cpp b/test/adapter_hs_gtest.cpp index 44f5520..680ca05 100644 --- a/test/adapter_hs_gtest.cpp +++ b/test/adapter_hs_gtest.cpp @@ -79,7 +79,7 @@ static size_t hex2bin(char *hex, int hex_len, char *binary, size_t size) return resultlen; } -int parse_config_file(const char *filename, and_expr_t exprs[], size_t *n_expr) +int parse_config_file(const char *filename, struct hs_expr exprs[], size_t *n_expr) { unsigned char *json_buff = NULL; size_t json_buff_size = 0; @@ -155,6 +155,7 @@ int parse_config_file(const char *filename, and_expr_t exprs[], size_t *n_expr) pat_str, pat_str_len); memcpy(exprs[i].patterns[j].pat, pat_str, pat_str_len); + free(pat_str); exprs[i].patterns[j].pat_len = pat_str_len; } else { memcpy(exprs[i].patterns[j].pat, item->valuestring, @@ -187,10 +188,11 @@ int parse_config_file(const char *filename, and_expr_t exprs[], size_t *n_expr) *n_expr = rule_cnt; next: cJSON_Delete(root); + FREE(json_buff); return ret; } -void expr_array_free(and_expr_t expr_array[], size_t n_expr_array) +void expr_array_free(struct hs_expr expr_array[], size_t n_expr_array) { for (size_t i = 0; i < n_expr_array; i++) { for (size_t j = 0; j < expr_array[i].n_patterns; j++) { @@ -205,7 +207,7 @@ void expr_array_free(and_expr_t expr_array[], size_t n_expr_array) TEST(block_mode_initialize, invalid_input_parameter) { struct adapter_hs *hs_instance = NULL; - and_expr_t exprs[64]; + struct hs_expr exprs[64]; /* case1: invalid scan_mode parameter */ hs_instance = adapter_hs_initialize(HS_SCAN_MODE_MAX, HS_PATTERN_TYPE_REG, @@ -225,7 +227,7 @@ TEST(block_mode_initialize, invalid_input_parameter) TEST(block_mode_scan, invalid_input_parameter) { - and_expr_t expr_array[64]; + struct hs_expr expr_array[64]; size_t n_expr_array = 0; struct adapter_hs *hs_instance = adapter_hs_initialize(HS_SCAN_MODE_BLOCK, HS_PATTERN_TYPE_REG, @@ -254,7 +256,7 @@ TEST(block_mode_scan, invalid_input_parameter) TEST(block_mode_scan, literal_sub_has_normal_offset) { - and_expr_t expr_array[64] = {0}; + struct hs_expr expr_array[64] = {0}; size_t n_expr_array = 0; int ret = parse_config_file("./literal_expr.conf", expr_array, &n_expr_array); @@ -295,7 +297,7 @@ TEST(block_mode_scan, literal_sub_has_normal_offset) TEST(block_mode_scan, literal_sub_has_left_unlimit_offset) { - and_expr_t expr_array[64] = {0}; + struct hs_expr expr_array[64] = {0}; size_t n_expr_array = 0; int ret = parse_config_file("./literal_expr.conf", expr_array, &n_expr_array); @@ -336,7 +338,7 @@ TEST(block_mode_scan, literal_sub_has_left_unlimit_offset) TEST(block_mode_scan, literal_sub_has_right_unlimit_offset) { - and_expr_t expr_array[64] = {0}; + struct hs_expr expr_array[64] = {0}; size_t n_expr_array = 0; int ret = parse_config_file("./literal_expr.conf", expr_array, &n_expr_array); @@ -392,7 +394,7 @@ TEST(block_mode_scan, literal_sub_has_right_unlimit_offset) TEST(block_mode_scan, literal_sub_with_no_offset) { - and_expr_t expr_array[64] = {0}; + struct hs_expr expr_array[64] = {0}; size_t n_expr_array = 0; int ret = parse_config_file("./literal_expr.conf", expr_array, &n_expr_array); @@ -440,7 +442,7 @@ TEST(block_mode_scan, literal_sub_with_no_offset) TEST(block_mode_scan, literal_exactly) { - and_expr_t expr_array[64] = {0}; + struct hs_expr expr_array[64] = {0}; size_t n_expr_array = 0; int ret = parse_config_file("./literal_expr.conf", expr_array, &n_expr_array); @@ -482,7 +484,7 @@ TEST(block_mode_scan, literal_exactly) TEST(block_mode_scan, literal_prefix) { - and_expr_t expr_array[64] = {0}; + struct hs_expr expr_array[64] = {0}; size_t n_expr_array = 0; int ret = parse_config_file("./literal_expr.conf", expr_array, &n_expr_array); @@ -533,7 +535,7 @@ TEST(block_mode_scan, literal_prefix) TEST(block_mode_scan, literal_suffix) { - and_expr_t expr_array[64] = {0}; + struct hs_expr expr_array[64] = {0}; size_t n_expr_array = 0; int ret = parse_config_file("./literal_expr.conf", expr_array, &n_expr_array); @@ -584,7 +586,7 @@ TEST(block_mode_scan, literal_suffix) TEST(block_mode_scan, literal_sub_with_hexbin) { - and_expr_t expr_array[64] = {0}; + struct hs_expr expr_array[64] = {0}; size_t n_expr_array = 0; int ret = parse_config_file("./literal_expr.conf", expr_array, &n_expr_array); @@ -617,7 +619,7 @@ TEST(block_mode_scan, literal_sub_with_hexbin) TEST(block_mode_scan, literal_with_chinese) { - and_expr_t expr_array[64] = {0}; + struct hs_expr expr_array[64] = {0}; size_t n_expr_array = 0; int ret = parse_config_file("./literal_expr.conf", expr_array, &n_expr_array); diff --git a/test/maat_api_gtest.cpp b/test/maat_api_gtest.cpp index 7e4421c..df9911d 100644 --- a/test/maat_api_gtest.cpp +++ b/test/maat_api_gtest.cpp @@ -1,10 +1,7 @@ #include "maat.h" - +#include "maat_rule.h" #include -TEST(EQ_Test, Always_True) { - EXPECT_EQ(1, 1); -} int main(int argc, char ** argv) { diff --git a/test/maat_ex_data_gtest.cpp b/test/maat_ex_data_gtest.cpp index f8eace5..f24e28d 100644 --- a/test/maat_ex_data_gtest.cpp +++ b/test/maat_ex_data_gtest.cpp @@ -4,6 +4,7 @@ #include "json2iris.h" #include "maat_command.h" #include "maat_ex_data.h" +#include "maat_garbage_collection.h" #include #include @@ -55,8 +56,8 @@ TEST(EXDataRuntime, Update) { ASSERT_GT(table_id, 0); int ex_data_counter = 0; - - struct ex_data_runtime *ex_data_rt = ex_data_runtime_new(table_id, ex_data_container_free, g_logger); + struct maat_garbage_bin *garbage_bin = maat_garbage_bin_new(10); + struct ex_data_runtime *ex_data_rt = ex_data_runtime_new(table_id, ex_container_free, garbage_bin, g_logger); struct ex_data_schema *ex_schema = ex_data_schema_new(ex_data_new_cb, ex_data_free_cb, ex_data_dup_cb, 0, &ex_data_counter); ex_data_runtime_set_schema(ex_data_rt, ex_schema); @@ -67,7 +68,7 @@ TEST(EXDataRuntime, Update) { void *ex_data = ex_data_runtime_row2ex_data(ex_data_rt, row1, key1, key1_len); EXPECT_EQ(ex_data_counter, 1); - struct ex_data_container *ex_container = ex_data_container_new(ex_data, NULL); + struct ex_container *ex_container = ex_container_new(ex_data, NULL); int ret = ex_data_runtime_add_ex_container(ex_data_rt, key1, key1_len, ex_container); EXPECT_EQ(ret, 0); @@ -75,7 +76,7 @@ TEST(EXDataRuntime, Update) { const char *key2 = "192.168.0.2"; size_t key2_len = strlen(key2); ex_data = ex_data_runtime_row2ex_data(ex_data_rt, row2, key2, key2_len); - ex_container = ex_data_container_new(ex_data, NULL); + ex_container = ex_container_new(ex_data, NULL); ret = ex_data_runtime_add_ex_container(ex_data_rt, key2, key2_len, ex_container); EXPECT_EQ(ret, 0); @@ -94,6 +95,10 @@ TEST(EXDataRuntime, Update) { info = (struct user_info *)res_data2; EXPECT_EQ(0, strcmp(info->name, "liyanhong")); EXPECT_EQ(info->id, 2); + + maat_garbage_bin_free(garbage_bin); + ex_data_runtime_free(ex_data_rt); + ex_data_schema_free(ex_schema); } int main(int argc, char ** argv) @@ -119,6 +124,7 @@ int main(int argc, char ** argv) ret = json2iris(json_buff, json_filename, NULL, tmp_iris_path, sizeof(tmp_iris_path), NULL, NULL, g_logger); + FREE(json_buff); EXPECT_NE(ret, -1); } diff --git a/test/maat_framework_gtest.cpp b/test/maat_framework_gtest.cpp index 8774e10..f49cddd 100644 --- a/test/maat_framework_gtest.cpp +++ b/test/maat_framework_gtest.cpp @@ -11,41 +11,316 @@ #define MODULE_FRAMEWORK_GTEST module_name_str("maat.framework_gtest") -#define ARRAY_SIZE 10 -#define HIT_PATH_SIZE 128 +#define ARRAY_SIZE 10 +#define HIT_PATH_SIZE 128 +#define WAIT_FOR_EFFECTIVE_S 1 const char *table_info_path = "./table_info.conf"; const char *json_path="./maat_json.json"; const char *json_filename = "maat_json.json"; -struct maat *g_maat_instance = NULL; + struct log_handle *g_logger = NULL; size_t g_thread_num = 4; extern int system_cmd_rmdir(const char *dir); +void wait_for_cmd_effective(struct maat *maat_instance, long long version_before) +{ + long long version_after = version_before; + int is_updating = 1; + long long wating_us = 0, sleep_us = 1000 * 100; + + while (is_updating || version_before == version_after) { + is_updating = maat_cmd_config_is_updating(maat_instance); + version_after = maat_cmd_get_config_version(maat_instance); + + usleep(sleep_us);//waiting for commands go into effect + wating_us += sleep_us; + } +} + +int count_line_num_cb(const char *table_name, const char *line, void *u_para) +{ + (*((unsigned int *)u_para))++; + return 0; +} + +int line_idx = 0; +long long absolute_expire_time=0; +int make_serial_rule(const char *table_name, const char *line, void *u_para) +{ + struct serial_rule *s_rule=(struct serial_rule *)u_para; + int rule_id = 0; + char *buff = ALLOC(char, strlen(line) + 1); + + memcpy(buff, line, strlen(line) + 1); + + while (buff[strlen(buff) - 1] == '\n' || buff[strlen(buff) - 1] == '\t') { + buff[strlen(buff) - 1] = '\0'; + } + + int j = 0; + char *str1 = NULL; + char *token = NULL; + char *saveptr1 = NULL; + + for (j = 0,str1 = buff; ; j++, str1 = NULL) { + token = strtok_r(str1, "\t ", &saveptr1); + if (token == NULL) + break; + if (j == 0) { + sscanf(token,"%d", &rule_id); + } + } + + memcpy(buff, line, strlen(line)+1); + while(buff[strlen(buff)-1]=='\n'||buff[strlen(buff)-1]=='\t') { + buff[strlen(buff)-1]='\0'; + } + + maat_cmd_set_serial_rule(s_rule + line_idx, MAAT_OP_ADD, rule_id, table_name, buff, absolute_expire_time); + line_idx++; + + FREE(buff); + + return 0; +} + +int write_config_to_redis(char *redis_ip, int redis_port, int redis_db, + struct log_handle *logger) +{ + char json_iris_path[512] = {0}; + + snprintf(json_iris_path, sizeof(json_iris_path), "./%s_iris_tmp", json_filename); + + redisContext *c = maat_cmd_connect_redis(redis_ip, redis_port, redis_db, logger); + if (NULL == c) { + return -1; + } + + redisReply *reply = maat_cmd_wrap_redis_command(c, "flushdb"); + if (NULL == reply) { + return -1; + } + + if (0 == access(json_iris_path, F_OK)) { + system_cmd_rmdir(json_iris_path); + } + + if (access(json_iris_path, F_OK) < 0) { + char tmp_iris_path[128] = {0}; + char *json_buff = NULL; + size_t json_buff_sz = 0; + + int ret = load_file_to_memory(json_filename, (unsigned char **)&json_buff, &json_buff_sz); + if (ret < 0) { + return -1; + } + + ret = json2iris(json_buff, json_filename, c, tmp_iris_path, + sizeof(tmp_iris_path), NULL, NULL, logger); + FREE(json_buff); + if (ret < 0) { + return -1; + } + } + + size_t total_line_cnt = 0; + char tmp_iris_full_idx_path[PATH_MAX] = {0}; + snprintf(tmp_iris_full_idx_path, sizeof(tmp_iris_full_idx_path), "%s/index", json_iris_path); + config_monitor_traverse(0, tmp_iris_full_idx_path, NULL, count_line_num_cb, NULL, &total_line_cnt, logger); + + struct serial_rule *s_rule = ALLOC(struct serial_rule, total_line_cnt); + long long server_time = maat_cmd_redis_server_time_s(c); + if (server_time < 0) { + return -1; + } + + absolute_expire_time = server_time + 300; + config_monitor_traverse(0, tmp_iris_full_idx_path, NULL, make_serial_rule, NULL, s_rule, logger); + line_idx = 0; + absolute_expire_time = 0; + + int success_cnt = 0; + do { + success_cnt = maat_cmd_write_rule(c, s_rule, total_line_cnt, server_time, logger); + } while (success_cnt < 0); + + EXPECT_EQ(success_cnt, (int)total_line_cnt); + + for (size_t i = 0; i < total_line_cnt; i++) { + maat_cmd_clear_rule_cache(s_rule + i); + } + FREE(s_rule); + redisFree(c); + + return 0; +} + +int compile_table_set_line(struct maat *maat_instance, const char *table_name, + enum maat_operation op, long long compile_id, + const char *user_region, int clause_num, int expire_after) +{ + char table_line[1024 * 16] = {0}; + sprintf(table_line, "%lld\t0\t0\t0\t0\t0\t%s\t%d\t%d\t0.0", + compile_id, user_region, op, clause_num); + + struct maat_cmd_line line_rule; + line_rule.rule_id = compile_id; + line_rule.table_line = table_line; + line_rule.table_name = table_name; + line_rule.expire_after = expire_after; + + return maat_cmd_set_line(maat_instance, &line_rule); +} + +int group2compile_table_set_line(struct maat *maat_instance, const char *table_name, + enum maat_operation op, long long group_id, long long compile_id, + int not_flag, const char *vtable_name, int clause_num, + int expire_after) +{ + char table_line[128] = {0}; + sprintf(table_line, "%lld\t%lld\t%d\t%d\t%s\t%d", + group_id, compile_id, op, not_flag, vtable_name, clause_num); + + struct maat_cmd_line line_rule; + line_rule.rule_id = group_id; + line_rule.table_line = table_line; + line_rule.table_name = table_name; + line_rule.expire_after = expire_after; + + return maat_cmd_set_line(maat_instance, &line_rule); +} + +int group2group_table_set_line(struct maat *maat_instance, const char *table_name, enum maat_operation op, + long long group_id, long long superior_group_id, int expire_after) +{ + char table_line[128] = {0}; + sprintf(table_line, "%lld\t%lld\t%d", group_id, superior_group_id, op); + + struct maat_cmd_line line_rule; + line_rule.rule_id = group_id; + line_rule.table_line = table_line; + line_rule.table_name = table_name; + line_rule.expire_after = expire_after; + + return maat_cmd_set_line(maat_instance, &line_rule); +} + +int expr_table_set_line(struct maat *maat_instance, const char *table_name, enum maat_operation op, + long long item_id, long long group_id, const char *keywords, const char *district, + int expr_type, int match_method, int is_hexbin, int expire_after) +{ + char table_line[1024] = {0}; + int table_id = maat_get_table_id(maat_instance, table_name); + if (table_id < 0) { + return 0; + } + + enum table_type table_type = table_manager_get_table_type(maat_instance->tbl_mgr, table_id); + assert(table_type == TABLE_TYPE_EXPR || table_type == TABLE_TYPE_EXPR_PLUS); + + if (table_type == TABLE_TYPE_EXPR_PLUS) { + sprintf(table_line, "%lld\t%lld\t%s\t%s\t%d\t%d\t%d\t%d", item_id, group_id, district, + keywords, expr_type, match_method, is_hexbin, op); + } else { + sprintf(table_line, "%lld\t%lld\t%s\t%d\t%d\t%d\t%d", item_id, group_id, keywords, expr_type, + match_method, is_hexbin, op); + } + + struct maat_cmd_line line_rule; + line_rule.rule_id = item_id; + line_rule.table_line = table_line; + line_rule.table_name = table_name; + line_rule.expire_after = expire_after; + + return maat_cmd_set_line(maat_instance, &line_rule); +} + +int ip_table_set_line(struct maat *maat_instance, const char *table_name, enum maat_operation op, + long long item_id, long long group_id, enum IP_TYPE type, const char *saddr_format, + const char *src_ip, const char *src_ip_mask, int expire_after) +{ + char table_line[1024] = {0}; + int table_id = maat_get_table_id(maat_instance, table_name); + if (table_id < 0) { + return 0; + } + + enum table_type table_type = table_manager_get_table_type(maat_instance->tbl_mgr, table_id); + assert(table_type == TABLE_TYPE_IP_PLUS); + + int ip_type = 4; + if (type == IPv6) { + ip_type = 6; + } + + sprintf(table_line, "%lld\t%lld\t%d\t%s\t%s\t%s\t%s\t0\t65535\t%s\t%s\t%s\t%s\t0\t65535\t6\t0\t%d", + item_id, group_id, ip_type, saddr_format, src_ip, src_ip_mask, "range", "range", "0.0.0.0", + "255.255.255.255", "range", op); + struct maat_cmd_line line_rule; + line_rule.rule_id = item_id; + line_rule.table_line = table_line; + line_rule.table_name = table_name; + line_rule.expire_after = expire_after; + + return maat_cmd_set_line(maat_instance, &line_rule); +} + class MaatFlagScan : public testing::Test { protected: static void SetUpTestCase() { + const char *accept_tags = "{\"tags\":[{\"tag\":\"location\",\"value\":\"北京/朝阳/华严北里/甲22号\"},{\"tag\":\"isp\",\"value\":\"移动\"},{\"tag\":\"location\",\"value\":\"Astana\"}]}"; + char redis_ip[64] = "127.0.0.1"; + int redis_port = 6379; + int redis_db = 0; + g_logger = log_handle_create("./maat_framework_gtest.log", 0); + assert(g_logger != NULL); + + int ret = write_config_to_redis(redis_ip, redis_port, redis_db, g_logger); + if (ret < 0) { + log_error(g_logger, MODULE_FRAMEWORK_GTEST, + "[%s:%d] write config to redis failed.", __FUNCTION__, __LINE__); + } + + struct maat_options *opts = maat_options_new(); + maat_options_set_redis(opts, redis_ip, redis_port, redis_db); + maat_options_set_logger(opts, g_logger); + maat_options_set_accept_tags(opts, accept_tags); + + _shared_maat_instance = maat_new(opts, table_info_path); + maat_options_free(opts); + if (NULL == _shared_maat_instance) { + log_error(g_logger, MODULE_FRAMEWORK_GTEST, + "[%s:%d] create maat instance in MaatFlagScan failed.", + __FUNCTION__, __LINE__); + } } static void TearDownTestCase() { - + maat_free(_shared_maat_instance); + g_logger = NULL; } + static struct maat *_shared_maat_instance; }; +struct maat *MaatFlagScan::_shared_maat_instance; + TEST_F(MaatFlagScan, basic) { const char *flag_table_name = "FLAG_CONFIG"; - int flag_table_id = maat_get_table_id(g_maat_instance, flag_table_name); + struct maat *maat_instance = MaatFlagScan::_shared_maat_instance; + + int flag_table_id = maat_get_table_id(maat_instance, flag_table_name); //compile_id:192 flag: 0000 0001 mask: 0000 0011 //scan_data: 0000 1001 or 0000 1101 should hit long long scan_data = 9; long long results[ARRAY_SIZE] = {0}; size_t n_hit_result = 0; struct maat_state *state = NULL; - int ret = maat_scan_flag(g_maat_instance, flag_table_id, 0, scan_data, results, + int ret = maat_scan_flag(maat_instance, flag_table_id, 0, scan_data, results, ARRAY_SIZE, &n_hit_result, &state); EXPECT_EQ(ret, MAAT_SCAN_HIT); EXPECT_EQ(n_hit_result, 1); @@ -53,14 +328,14 @@ TEST_F(MaatFlagScan, basic) { struct maat_hit_path hit_path[HIT_PATH_SIZE] = {0}; int n_read = 0; - n_read = maat_state_get_hit_paths(g_maat_instance, &state, hit_path, HIT_PATH_SIZE); + n_read = maat_state_get_hit_paths(maat_instance, &state, hit_path, HIT_PATH_SIZE); EXPECT_NE(n_read, 0); maat_state_free(&state); scan_data = 13; memset(results, 0, sizeof(results)); n_hit_result = 0; - ret = maat_scan_flag(g_maat_instance, flag_table_id, 0, scan_data, results, + ret = maat_scan_flag(maat_instance, flag_table_id, 0, scan_data, results, ARRAY_SIZE, &n_hit_result, &state); EXPECT_EQ(ret, MAAT_SCAN_HIT); EXPECT_EQ(n_hit_result, 1); @@ -70,7 +345,7 @@ TEST_F(MaatFlagScan, basic) { scan_data = 6; memset(results, 0, sizeof(results)); n_hit_result = 0; - ret = maat_scan_flag(g_maat_instance, flag_table_id, 0, scan_data, results, + ret = maat_scan_flag(maat_instance, flag_table_id, 0, scan_data, results, ARRAY_SIZE, &n_hit_result, &state); EXPECT_EQ(ret, MAAT_SCAN_HALF_HIT); EXPECT_EQ(n_hit_result, 0); @@ -80,8 +355,10 @@ TEST_F(MaatFlagScan, basic) { TEST_F(MaatFlagScan, withExprRegion) { const char *flag_table_name = "FLAG_CONFIG"; const char *expr_table_name = "HTTP_URL_LITERAL"; - int flag_table_id = maat_get_table_id(g_maat_instance, flag_table_name); - int expr_table_id = maat_get_table_id(g_maat_instance, expr_table_name); + struct maat *maat_instance = MaatFlagScan::_shared_maat_instance; + + int flag_table_id = maat_get_table_id(maat_instance, flag_table_name); + int expr_table_id = maat_get_table_id(maat_instance, expr_table_name); //compile_id:193 flag: 0000 0010 mask: 0000 0011 //scan_data: 0000 0010 or 0000 0100 should hit long long flag_scan_data = 2; @@ -89,18 +366,18 @@ TEST_F(MaatFlagScan, withExprRegion) { size_t n_hit_result = 0; struct maat_state *state = NULL; - int ret = maat_scan_flag(g_maat_instance, flag_table_id, 0, flag_scan_data, results, + int ret = maat_scan_flag(maat_instance, flag_table_id, 0, flag_scan_data, results, ARRAY_SIZE, &n_hit_result, &state); EXPECT_EQ(ret, MAAT_SCAN_HALF_HIT); EXPECT_EQ(n_hit_result, 0); struct maat_hit_path hit_path[HIT_PATH_SIZE] = {0}; int n_read = 0; - n_read = maat_state_get_hit_paths(g_maat_instance, &state, hit_path, HIT_PATH_SIZE); + n_read = maat_state_get_hit_paths(maat_instance, &state, hit_path, HIT_PATH_SIZE); EXPECT_NE(n_read, 0); const char *expr_scan_data = "hello world"; - ret = maat_scan_string(g_maat_instance, expr_table_id, 0, expr_scan_data, + ret = maat_scan_string(maat_instance, expr_table_id, 0, expr_scan_data, strlen(expr_scan_data), results, ARRAY_SIZE, &n_hit_result, &state); EXPECT_EQ(ret, MAAT_SCAN_HIT); @@ -111,7 +388,9 @@ TEST_F(MaatFlagScan, withExprRegion) { TEST_F(MaatFlagScan, hitMultiCompile) { const char *flag_table_name = "FLAG_CONFIG"; - int flag_table_id = maat_get_table_id(g_maat_instance, flag_table_name); + struct maat *maat_instance = MaatFlagScan::_shared_maat_instance; + + int flag_table_id = maat_get_table_id(maat_instance, flag_table_name); //compile_id:192 flag: 0000 0001 mask: 0000 0011 //compile_id:194 flag: 0001 0101 mask: 0001 1111 //scan_data: 0001 0101 should hit compile192 and compile194 @@ -120,7 +399,7 @@ TEST_F(MaatFlagScan, hitMultiCompile) { size_t n_hit_result = 0; struct maat_state *state = NULL; - int ret = maat_scan_flag(g_maat_instance, flag_table_id, 0, flag_scan_data, results, + int ret = maat_scan_flag(maat_instance, flag_table_id, 0, flag_scan_data, results, ARRAY_SIZE, &n_hit_result, &state); EXPECT_EQ(ret, MAAT_SCAN_HIT); EXPECT_EQ(n_hit_result, 2); @@ -128,13 +407,13 @@ TEST_F(MaatFlagScan, hitMultiCompile) { EXPECT_EQ(results[1], 192); memset(results, 0, sizeof(results)); - ret = maat_scan_flag(g_maat_instance, flag_table_id, 0, flag_scan_data, results, + ret = maat_scan_flag(maat_instance, flag_table_id, 0, flag_scan_data, results, ARRAY_SIZE, &n_hit_result, &state); EXPECT_EQ(ret, MAAT_SCAN_HALF_HIT); struct maat_hit_path hit_path[HIT_PATH_SIZE] = {0}; int n_read = 0; - n_read = maat_state_get_hit_paths(g_maat_instance, &state, hit_path, HIT_PATH_SIZE); + n_read = maat_state_get_hit_paths(maat_instance, &state, hit_path, HIT_PATH_SIZE); EXPECT_NE(n_read, 0); maat_state_free(&state); @@ -142,7 +421,9 @@ TEST_F(MaatFlagScan, hitMultiCompile) { TEST_F(MaatFlagScan, hitRepeatedCompile) { const char *flag_table_name = "FLAG_CONFIG"; - int flag_table_id = maat_get_table_id(g_maat_instance, flag_table_name); + struct maat *maat_instance = MaatFlagScan::_shared_maat_instance; + + int flag_table_id = maat_get_table_id(maat_instance, flag_table_name); long long results[ARRAY_SIZE] = {0}; size_t n_hit_result = 0; struct maat_state *state = NULL; @@ -150,7 +431,7 @@ TEST_F(MaatFlagScan, hitRepeatedCompile) { //compile_id:192 flag: 0000 0001 mask: 0000 0011 //scan_data: 0000 1001 or 0000 1101 should hit long long flag_scan_data1 = 9; - int ret = maat_scan_flag(g_maat_instance, flag_table_id, 0, flag_scan_data1, results, + int ret = maat_scan_flag(maat_instance, flag_table_id, 0, flag_scan_data1, results, ARRAY_SIZE, &n_hit_result, &state); EXPECT_EQ(ret, MAAT_SCAN_HIT); EXPECT_EQ(n_hit_result, 1); @@ -161,20 +442,20 @@ TEST_F(MaatFlagScan, hitRepeatedCompile) { //scan_data: 0001 0101 should hit compile192 and compile194 long long flag_scan_data2 = 21; memset(results, 0, sizeof(results)); - ret = maat_scan_flag(g_maat_instance, flag_table_id, 0, flag_scan_data2, results, + ret = maat_scan_flag(maat_instance, flag_table_id, 0, flag_scan_data2, results, ARRAY_SIZE, &n_hit_result, &state); EXPECT_EQ(ret, MAAT_SCAN_HIT); EXPECT_EQ(n_hit_result, 1); EXPECT_EQ(results[0], 194); memset(results, 0, sizeof(results)); - ret = maat_scan_flag(g_maat_instance, flag_table_id, 0, flag_scan_data2, results, + ret = maat_scan_flag(maat_instance, flag_table_id, 0, flag_scan_data2, results, ARRAY_SIZE, &n_hit_result, &state); EXPECT_EQ(ret, MAAT_SCAN_HALF_HIT); struct maat_hit_path hit_path[HIT_PATH_SIZE] = {0}; int n_read = 0; - n_read = maat_state_get_hit_paths(g_maat_instance, &state, hit_path, HIT_PATH_SIZE); + n_read = maat_state_get_hit_paths(maat_instance, &state, hit_path, HIT_PATH_SIZE); EXPECT_NE(n_read, 0); maat_state_free(&state); @@ -184,56 +465,91 @@ TEST_F(MaatFlagScan, hitRepeatedCompile) { TEST_F(MaatFlagScan, FlagPlus) { const char *flag_table_name = "FLAG_PLUS_CONFIG"; const char *region_name = "I love China"; - int flag_table_id = maat_get_table_id(g_maat_instance, flag_table_name); + struct maat *maat_instance = MaatFlagScan::_shared_maat_instance; + + int flag_table_id = maat_get_table_id(maat_instance, flag_table_name); //compile_id:196 flag: 0001 1111 mask: 0000 1111 //scan_data: 0000 1111 or 0001 1111 should hit long long scan_data1 = 15; long long results[ARRAY_SIZE] = {0}; size_t n_hit_result = 0; struct maat_state *state = NULL; - int ret = maat_scan_flag(g_maat_instance, flag_table_id, 0, scan_data1, results, + int ret = maat_scan_flag(maat_instance, flag_table_id, 0, scan_data1, results, ARRAY_SIZE, &n_hit_result, &state); EXPECT_EQ(ret, MAAT_SCAN_ERR); - ret = maat_state_set_scan_district(g_maat_instance, &state, region_name, strlen(region_name)); + ret = maat_state_set_scan_district(maat_instance, &state, region_name, strlen(region_name)); ASSERT_EQ(ret, 0); - ret = maat_scan_flag(g_maat_instance, flag_table_id, 0, scan_data1, results, + ret = maat_scan_flag(maat_instance, flag_table_id, 0, scan_data1, results, ARRAY_SIZE, &n_hit_result, &state); EXPECT_EQ(ret, MAAT_SCAN_HIT); EXPECT_EQ(n_hit_result, 1); EXPECT_EQ(results[0], 196); - ret = maat_scan_flag(g_maat_instance, flag_table_id, 0, scan_data1, results, + ret = maat_scan_flag(maat_instance, flag_table_id, 0, scan_data1, results, ARRAY_SIZE, &n_hit_result, &state); EXPECT_EQ(ret, MAAT_SCAN_HALF_HIT); struct maat_hit_path hit_path[HIT_PATH_SIZE] = {0}; int n_read = 0; - n_read = maat_state_get_hit_paths(g_maat_instance, &state, hit_path, HIT_PATH_SIZE); + n_read = maat_state_get_hit_paths(maat_instance, &state, hit_path, HIT_PATH_SIZE); EXPECT_NE(n_read, 0); maat_state_free(&state); } - +#if 1 class MaatStringScan : public testing::Test { protected: static void SetUpTestCase() { + const char *accept_tags = "{\"tags\":[{\"tag\":\"location\",\"value\":\"北京/朝阳/华严北里/甲22号\"},{\"tag\":\"isp\",\"value\":\"移动\"},{\"tag\":\"location\",\"value\":\"Astana\"}]}"; + char redis_ip[64] = "127.0.0.1"; + int redis_port = 6379; + int redis_db = 0; + + g_logger = log_handle_create("./maat_framework_gtest.log", 0); + assert(g_logger != NULL); + + int ret = write_config_to_redis(redis_ip, redis_port, redis_db, g_logger); + if (ret < 0) { + log_error(g_logger, MODULE_FRAMEWORK_GTEST, + "[%s:%d] write config to redis failed.", __FUNCTION__, __LINE__); + } + + struct maat_options *opts = maat_options_new(); + maat_options_set_redis(opts, redis_ip, redis_port, redis_db); + maat_options_set_logger(opts, g_logger); + maat_options_set_accept_tags(opts, accept_tags); + + _shared_maat_instance = maat_new(opts, table_info_path); + maat_options_free(opts); + if (NULL == _shared_maat_instance) { + log_error(g_logger, MODULE_FRAMEWORK_GTEST, + "[%s:%d] create maat instance in MaatFlagScan failed.", + __FUNCTION__, __LINE__); + } } static void TearDownTestCase() { - + maat_free(_shared_maat_instance); + g_logger = NULL; } + + static struct maat *_shared_maat_instance; }; +struct maat *MaatStringScan::_shared_maat_instance; + TEST_F(MaatStringScan, Expr8) { const char *table_name = "KEYWORDS_TABLE"; - int table_id = maat_get_table_id(g_maat_instance, table_name); + struct maat *maat_instance = MaatStringScan::_shared_maat_instance; + + int table_id = maat_get_table_id(maat_instance, table_name); char scan_data[128] = "string1, string2, string3, string4, string5, string6, string7, string8"; long long results[ARRAY_SIZE] = {0}; size_t n_hit_result = 0; struct maat_state *state = NULL; - int ret = maat_scan_string(g_maat_instance, table_id, 0, scan_data, strlen(scan_data), + int ret = maat_scan_string(maat_instance, table_id, 0, scan_data, strlen(scan_data), results, ARRAY_SIZE, &n_hit_result, &state); EXPECT_EQ(ret, MAAT_SCAN_HIT); EXPECT_EQ(n_hit_result, 1); @@ -241,7 +557,7 @@ TEST_F(MaatStringScan, Expr8) { struct maat_hit_path hit_path[HIT_PATH_SIZE] = {0}; int n_read = 0; - n_read = maat_state_get_hit_paths(g_maat_instance, &state, hit_path, HIT_PATH_SIZE); + n_read = maat_state_get_hit_paths(maat_instance, &state, hit_path, HIT_PATH_SIZE); EXPECT_NE(n_read, 0); maat_state_free(&state); } @@ -253,9 +569,10 @@ TEST_F(MaatStringScan, Regex) { struct maat_state *state = NULL; const char *cookie = "Cookie: Txa123aheadBCAxd"; const char *table_name = "HTTP_URL_REGEX"; - int table_id = maat_get_table_id(g_maat_instance, table_name); + struct maat *maat_instance = MaatStringScan::_shared_maat_instance; - ret = maat_scan_string(g_maat_instance, table_id, 0, cookie, strlen(cookie), + int table_id = maat_get_table_id(maat_instance, table_name); + ret = maat_scan_string(maat_instance, table_id, 0, cookie, strlen(cookie), results, ARRAY_SIZE, &n_hit_result, &state); EXPECT_EQ(ret, MAAT_SCAN_HIT); EXPECT_EQ(results[0], 146); @@ -293,23 +610,24 @@ TEST_F(MaatStringScan, ExprPlus) { const char *scan_data1 = "http://www.cyberessays.com/search_results.php?action=search&query=abckkk,1234567"; const char *scan_data2 = "Addis Sapphire Hotel"; const char *table_name = "HTTP_SIGNATURE"; - int table_id = maat_get_table_id(g_maat_instance, table_name); + struct maat *maat_instance = MaatStringScan::_shared_maat_instance; - int ret = maat_scan_string(g_maat_instance, table_id, 0, scan_data1, strlen(scan_data1), + int table_id = maat_get_table_id(maat_instance, table_name); + int ret = maat_scan_string(maat_instance, table_id, 0, scan_data1, strlen(scan_data1), results, ARRAY_SIZE, &n_hit_result, &state); EXPECT_EQ(ret, MAAT_SCAN_ERR);//Should return error for district not setting. - ret = maat_state_set_scan_district(g_maat_instance, &state, region_name1, strlen(region_name1)); + ret = maat_state_set_scan_district(maat_instance, &state, region_name1, strlen(region_name1)); ASSERT_EQ(ret, 0); - ret = maat_scan_string(g_maat_instance, table_id, 0, scan_data1, strlen(scan_data1), + ret = maat_scan_string(maat_instance, table_id, 0, scan_data1, strlen(scan_data1), results, ARRAY_SIZE, &n_hit_result, &state); EXPECT_EQ(ret, MAAT_SCAN_HIT); EXPECT_EQ(results[0], 128); maat_state_free(&state); - ret = maat_state_set_scan_district(g_maat_instance, &state, region_name2, strlen(region_name2)); + ret = maat_state_set_scan_district(maat_instance, &state, region_name2, strlen(region_name2)); ASSERT_EQ(ret, 0); - ret = maat_scan_string(g_maat_instance, table_id, 0, scan_data2, strlen(scan_data2), + ret = maat_scan_string(maat_instance, table_id, 0, scan_data2, strlen(scan_data2), results, ARRAY_SIZE, &n_hit_result, &state); EXPECT_EQ(ret, MAAT_SCAN_HIT); EXPECT_EQ(results[0], 190); @@ -324,20 +642,22 @@ TEST_F(MaatStringScan, ExprAndExprPlus) { const char *expr_plus_table_name = "HTTP_SIGNATURE"; const char *region_name = "I love China"; const char *scan_data = "today is Monday and yesterday is Tuesday"; - int expr_table_id = maat_get_table_id(g_maat_instance, expr_table_name); - int expr_plus_table_id = maat_get_table_id(g_maat_instance, expr_plus_table_name); + struct maat *maat_instance = MaatStringScan::_shared_maat_instance; - int ret = maat_scan_string(g_maat_instance, expr_plus_table_id, 0, scan_data, strlen(scan_data), + int expr_table_id = maat_get_table_id(maat_instance, expr_table_name); + int expr_plus_table_id = maat_get_table_id(maat_instance, expr_plus_table_name); + + int ret = maat_scan_string(maat_instance, expr_plus_table_id, 0, scan_data, strlen(scan_data), results, ARRAY_SIZE, &n_hit_result, &state); EXPECT_EQ(ret, MAAT_SCAN_ERR); - ret = maat_state_set_scan_district(g_maat_instance, &state, region_name, strlen(region_name)); + ret = maat_state_set_scan_district(maat_instance, &state, region_name, strlen(region_name)); ASSERT_EQ(ret, 0); - ret = maat_scan_string(g_maat_instance, expr_plus_table_id, 0, scan_data, strlen(scan_data), + ret = maat_scan_string(maat_instance, expr_plus_table_id, 0, scan_data, strlen(scan_data), results, ARRAY_SIZE, &n_hit_result, &state); EXPECT_EQ(ret, MAAT_SCAN_HALF_HIT); - ret = maat_scan_string(g_maat_instance, expr_table_id, 0, scan_data, strlen(scan_data), + ret = maat_scan_string(maat_instance, expr_table_id, 0, scan_data, strlen(scan_data), results, ARRAY_SIZE, &n_hit_result, &state); EXPECT_EQ(ret, MAAT_SCAN_HIT); EXPECT_EQ(results[0], 195); @@ -489,85 +809,68 @@ TEST_F(MaatStringScan, ExprPlusWithOffset) #endif TEST_F(MaatStringScan, dynamic_config) { const char *table_name = "HTTP_URL_LITERAL"; - int table_id = maat_get_table_id(g_maat_instance, table_name); - char data[128] = "hello world"; + char data[128] = "hello world, welcome to maat version4, it's funny."; long long results[ARRAY_SIZE] = {0}; size_t n_hit_result = 0; struct maat_state *state = NULL; - int ret = maat_scan_string(g_maat_instance, table_id, 0, data, strlen(data), results, + struct maat *maat_instance = MaatStringScan::_shared_maat_instance; + + int table_id = maat_get_table_id(maat_instance, table_name); + int ret = maat_scan_string(maat_instance, table_id, 0, data, strlen(data), results, ARRAY_SIZE, &n_hit_result, &state); EXPECT_EQ(ret, MAAT_SCAN_HALF_HIT); EXPECT_EQ(n_hit_result, 0); maat_state_free(&state); - /* add new line in GROUP2COMPILE table */ - const char *g2c_table_name = "GROUP2COMPILE"; - const char *g2c_table_line_add = "8888\t9999\t1\t0\tnull\t1"; - struct maat_cmd_line line_rule; - line_rule.rule_id = 8888; - line_rule.table_line = g2c_table_line_add; - line_rule.table_name = g2c_table_name; - ret = maat_cmd_set_line(g_maat_instance, &line_rule); - EXPECT_EQ(ret, 1); - - /* add new line in COMPILE table */ const char *compile_table_name = "COMPILE"; - const char *compile_table_line_add = "9999\t0\t0\t0\t0\t0\tanything\t1\t1\t0.0"; - memset(&line_rule, 0, sizeof(line_rule)); - line_rule.rule_id = 9999; - line_rule.table_line = compile_table_line_add; - line_rule.table_name = compile_table_name; - ret = maat_cmd_set_line(g_maat_instance, &line_rule); + const char *g2c_table_name = "GROUP2COMPILE"; + + /* compile table add line */ + long long compile_id = maat_cmd_incrby(maat_instance, "TEST_SEQ", 1); + ret = compile_table_set_line(maat_instance, compile_table_name, MAAT_OP_ADD, compile_id, NULL, 1, 0); EXPECT_EQ(ret, 1); - /* add new line in HTTP_URL_LITERAL table */ - const char *table_line_add = "9999\t8888\thello world\t0\t0\t0\t1\t"; - memset(&line_rule, 0, sizeof(line_rule)); - line_rule.rule_id = 9999; - line_rule.table_line = table_line_add; - line_rule.table_name = table_name; - ret = maat_cmd_set_line(g_maat_instance, &line_rule); + /* group2compile table add line */ + long long group_id = maat_cmd_incrby(maat_instance, "SEQUENCE_GROUP", 1); + ret = group2compile_table_set_line(maat_instance, g2c_table_name, MAAT_OP_ADD, group_id, + compile_id, 0, "null", 1, 0); EXPECT_EQ(ret, 1); - sleep(2); + /* expr table add line */ + long long item_id = maat_cmd_incrby(maat_instance, "SEQUENCE_REGION", 1); + const char *keywords = "welcome to maat"; + ret = expr_table_set_line(maat_instance, table_name, MAAT_OP_ADD, item_id, group_id, + keywords, NULL, 1, 0, 0, 0); /* EXPR_TYPE_AND MATCH_METHOD_SUB */ + EXPECT_EQ(ret, 1); + + sleep(WAIT_FOR_EFFECTIVE_S * 2); + state = NULL; - ret = maat_scan_string(g_maat_instance, table_id, 0, data, strlen(data), results, + ret = maat_scan_string(maat_instance, table_id, 0, data, strlen(data), results, ARRAY_SIZE, &n_hit_result, &state); EXPECT_EQ(ret, MAAT_SCAN_HIT); EXPECT_EQ(n_hit_result, 1); - EXPECT_EQ(results[0], 9999); + EXPECT_EQ(results[0], compile_id); maat_state_free(&state); - /* del new line from HTTP_URL_LITERAL table */ - const char *table_line_del = "9999\t8888\thello world\t0\t0\t0\t0\t"; - memset(&line_rule, 0, sizeof(line_rule)); - line_rule.rule_id = 9999; - line_rule.table_line = table_line_del; - line_rule.table_name = table_name; - ret = maat_cmd_set_line(g_maat_instance, &line_rule); + /* expr table del line */ + ret = expr_table_set_line(maat_instance, table_name, MAAT_OP_DEL, item_id, group_id, + keywords, NULL, 1, 0, 0, 0); /* EXPR_TYPE_AND MATCH_METHOD_SUB */ EXPECT_EQ(ret, 1); - /* del new line from COMPILE table */ - const char *compile_table_line_del = "9999\t0\t0\t0\t0\t0\tanything\t0\t1\t0.0"; - memset(&line_rule, 0, sizeof(line_rule)); - line_rule.rule_id = 9999; - line_rule.table_line = compile_table_line_del; - line_rule.table_name = compile_table_name; - ret = maat_cmd_set_line(g_maat_instance, &line_rule); + /* group2compile table del line */ + ret = group2compile_table_set_line(maat_instance, g2c_table_name, MAAT_OP_DEL, group_id, + compile_id, 0, "null", 1, 0); EXPECT_EQ(ret, 1); - /* del new line from GROUP2COMPILE table */ - const char *g2c_table_line_del = "8888\t9999\t0\t0\tnull\t1"; - memset(&line_rule, 0, sizeof(line_rule)); - line_rule.rule_id = 8888; - line_rule.table_line = g2c_table_line_del; - line_rule.table_name = g2c_table_name; - ret = maat_cmd_set_line(g_maat_instance, &line_rule); + /* compile table del line */ + ret = compile_table_set_line(maat_instance, compile_table_name, MAAT_OP_DEL, compile_id, NULL, 1, 0); EXPECT_EQ(ret, 1); - sleep(2); + sleep(WAIT_FOR_EFFECTIVE_S * 2); + state = NULL; - ret = maat_scan_string(g_maat_instance, table_id, 0, data, strlen(data), results, + ret = maat_scan_string(maat_instance, table_id, 0, data, strlen(data), results, ARRAY_SIZE, &n_hit_result, &state); EXPECT_EQ(ret, MAAT_SCAN_HALF_HIT); EXPECT_EQ(n_hit_result, 0); @@ -578,17 +881,49 @@ class MaatIPScan : public testing::Test { protected: static void SetUpTestCase() { - + const char *accept_tags = "{\"tags\":[{\"tag\":\"location\",\"value\":\"北京/朝阳/华严北里/甲22号\"},{\"tag\":\"isp\",\"value\":\"移动\"},{\"tag\":\"location\",\"value\":\"Astana\"}]}"; + char redis_ip[64] = "127.0.0.1"; + int redis_port = 6379; + int redis_db = 0; + + g_logger = log_handle_create("./maat_framework_gtest.log", 0); + assert(g_logger != NULL); + + int ret = write_config_to_redis(redis_ip, redis_port, redis_db, g_logger); + if (ret < 0) { + log_error(g_logger, MODULE_FRAMEWORK_GTEST, + "[%s:%d] write config to redis failed.", __FUNCTION__, __LINE__); + } + + struct maat_options *opts = maat_options_new(); + maat_options_set_redis(opts, redis_ip, redis_port, redis_db); + maat_options_set_logger(opts, g_logger); + maat_options_set_accept_tags(opts, accept_tags); + + _shared_maat_instance = maat_new(opts, table_info_path); + maat_options_free(opts); + if (NULL == _shared_maat_instance) { + log_error(g_logger, MODULE_FRAMEWORK_GTEST, + "[%s:%d] create maat instance in MaatFlagScan failed.", + __FUNCTION__, __LINE__); + } } static void TearDownTestCase() { - + maat_free(_shared_maat_instance); + g_logger = NULL; } + + static struct maat *_shared_maat_instance; }; +struct maat *MaatIPScan::_shared_maat_instance; + TEST_F(MaatIPScan, IPv4) { const char *table_name = "IP_PLUS_CONFIG"; - int table_id = maat_get_table_id(g_maat_instance, table_name); + struct maat *maat_instance = MaatIPScan::_shared_maat_instance; + + int table_id = maat_get_table_id(maat_instance, table_name); ASSERT_GT(table_id, 0); char ip_str[32] = "10.0.7.100"; @@ -599,7 +934,7 @@ TEST_F(MaatIPScan, IPv4) { long long results[ARRAY_SIZE] = {0}; size_t n_hit_result = 0; struct maat_state *state = NULL; - ret = maat_scan_ipv4(g_maat_instance, table_id, 0, sip, results, ARRAY_SIZE, + ret = maat_scan_ipv4(maat_instance, table_id, 0, sip, results, ARRAY_SIZE, &n_hit_result, &state); EXPECT_EQ(ret, MAAT_SCAN_HIT); EXPECT_EQ(n_hit_result, 1); @@ -609,7 +944,9 @@ TEST_F(MaatIPScan, IPv4) { TEST_F(MaatIPScan, IPv6) { const char *table_name = "IP_PLUS_CONFIG"; - int table_id = maat_get_table_id(g_maat_instance, table_name); + struct maat *maat_instance = MaatIPScan::_shared_maat_instance; + + int table_id = maat_get_table_id(maat_instance, table_name); char ip_str[32] = "1001:da8:205:1::101"; uint8_t sip[16]; int ret = inet_pton(AF_INET6, ip_str, &sip); @@ -618,7 +955,7 @@ TEST_F(MaatIPScan, IPv6) { long long results[ARRAY_SIZE] = {0}; size_t n_hit_result = 0; struct maat_state *state = NULL; - ret = maat_scan_ipv6(g_maat_instance, table_id, 0, sip, results, ARRAY_SIZE, + ret = maat_scan_ipv6(maat_instance, table_id, 0, sip, results, ARRAY_SIZE, &n_hit_result, &state); EXPECT_EQ(ret, MAAT_SCAN_HIT); EXPECT_EQ(n_hit_result, 1); @@ -629,7 +966,9 @@ TEST_F(MaatIPScan, IPv6) { TEST_F(MaatIPScan, dynamic_config) { const char *table_name = "IP_PLUS_CONFIG"; - int table_id = maat_get_table_id(g_maat_instance, table_name); + struct maat *maat_instance = MaatIPScan::_shared_maat_instance; + + int table_id = maat_get_table_id(maat_instance, table_name); char ip_str[32] = "100.100.100.100"; uint32_t sip; int ret = inet_pton(AF_INET, ip_str, &sip); @@ -638,105 +977,122 @@ TEST_F(MaatIPScan, dynamic_config) { long long results[ARRAY_SIZE] = {0}; size_t n_hit_result = 0; struct maat_state *state = NULL; - ret = maat_scan_ipv4(g_maat_instance, table_id, 0, sip, results, ARRAY_SIZE, + ret = maat_scan_ipv4(maat_instance, table_id, 0, sip, results, ARRAY_SIZE, &n_hit_result, &state); EXPECT_EQ(ret, MAAT_SCAN_OK); EXPECT_EQ(n_hit_result, 0); maat_state_free(&state); - /* add new line in GROUP2COMPILE table */ - const char *g2c_table_name = "GROUP2COMPILE"; - const char *g2c_table_line_add = "8887\t9998\t1\t0\tnull\t1"; - struct maat_cmd_line line_rule; - line_rule.rule_id = 8887; - line_rule.table_line = g2c_table_line_add; - line_rule.table_name = g2c_table_name; - ret = maat_cmd_set_line(g_maat_instance, &line_rule); - EXPECT_EQ(ret, 1); - - /* add new line in COMPILE table */ const char *compile_table_name = "COMPILE"; - const char *compile_table_line_add = "9998\t0\t0\t0\t0\t0\tanything\t1\t1\t0.0"; - memset(&line_rule, 0, sizeof(line_rule)); - line_rule.rule_id = 9998; - line_rule.table_line = compile_table_line_add; - line_rule.table_name = compile_table_name; - ret = maat_cmd_set_line(g_maat_instance, &line_rule); + const char *g2c_table_name = "GROUP2COMPILE"; + + /* compile table add line */ + long long compile_id = maat_cmd_incrby(maat_instance, "TEST_SEQ", 1); + ret = compile_table_set_line(maat_instance, compile_table_name, MAAT_OP_ADD, compile_id, NULL, 1, 0); EXPECT_EQ(ret, 1); - /* add new line in IP_PLUS_CONFIG */ - const char *table_line_add = "9998\t8887\t4\trange\t100.100.100.100\t100.100.100.100\trange\t0\t65535\trange\t10.0.6.201\t255.255.0.0\trange\t0\t65535\t6\t0\t1"; - memset(&line_rule, 0, sizeof(line_rule)); - line_rule.rule_id = 9998; - line_rule.table_line = table_line_add; - line_rule.table_name = table_name; - ret = maat_cmd_set_line(g_maat_instance, &line_rule); + /* group2compile table add line */ + long long group_id = maat_cmd_incrby(maat_instance, "SEQUENCE_GROUP", 1); + ret = group2compile_table_set_line(maat_instance, g2c_table_name, MAAT_OP_ADD, group_id, + compile_id, 0, "null", 1, 0); EXPECT_EQ(ret, 1); - sleep(2); + /* ip table add line */ + long long item_id = maat_cmd_incrby(maat_instance, "SEQUENCE_REGION", 1); + ret = ip_table_set_line(maat_instance, table_name, MAAT_OP_ADD, item_id, group_id, + IPv4, "range", "100.100.100.100", "100.100.100.100", 0); + EXPECT_EQ(ret, 1); + + sleep(WAIT_FOR_EFFECTIVE_S * 2); + state = NULL; - ret = maat_scan_ipv4(g_maat_instance, table_id, 0, sip, results, ARRAY_SIZE, + ret = maat_scan_ipv4(maat_instance, table_id, 0, sip, results, ARRAY_SIZE, &n_hit_result, &state); EXPECT_EQ(ret, MAAT_SCAN_HIT); EXPECT_EQ(n_hit_result, 1); - EXPECT_EQ(results[0], 9998); + EXPECT_EQ(results[0], compile_id); maat_state_free(&state); - /* del new line in IP_PLUS_CONFIG */ - const char *table_line_del = "9998\t8887\t4\trange\t100.100.100.100\t100.100.100.100\trange\t0\t65535\trange\t10.0.6.201\t255.255.0.0\trange\t0\t65535\t6\t0\t0"; - memset(&line_rule, 0, sizeof(line_rule)); - line_rule.rule_id = 9998; - line_rule.table_line = table_line_del; - line_rule.table_name = table_name; - ret = maat_cmd_set_line(g_maat_instance, &line_rule); + /* ip table del line */ + ret = ip_table_set_line(maat_instance, table_name, MAAT_OP_DEL, item_id, group_id, + IPv4, "range", "100.100.100.100", "100.100.100.100", 0); EXPECT_EQ(ret, 1); - /* del new line in COMPILE table */ - const char *compile_table_line_del = "9998\t0\t0\t0\t0\t0\tanything\t0\t1\t0.0"; - memset(&line_rule, 0, sizeof(line_rule)); - line_rule.rule_id = 9998; - line_rule.table_line = compile_table_line_del; - line_rule.table_name = compile_table_name; - ret = maat_cmd_set_line(g_maat_instance, &line_rule); + /* group2compile table del line */ + ret = group2compile_table_set_line(maat_instance, g2c_table_name, MAAT_OP_DEL, group_id, + compile_id, 0, "null", 1, 0); EXPECT_EQ(ret, 1); - /* del new line in GROUP2COMPILE table */ - const char *g2c_table_line_del = "8887\t9998\t0\t0\tnull\t1"; - memset(&line_rule, 0, sizeof(line_rule)); - line_rule.rule_id = 8887; - line_rule.table_line = g2c_table_line_del; - line_rule.table_name = g2c_table_name; - ret = maat_cmd_set_line(g_maat_instance, &line_rule); + /* compile table del line */ + ret = compile_table_set_line(maat_instance, compile_table_name, MAAT_OP_DEL, compile_id, NULL, 1, 0); EXPECT_EQ(ret, 1); + + sleep(WAIT_FOR_EFFECTIVE_S * 2); + + ret = maat_scan_ipv4(maat_instance, table_id, 0, sip, results, ARRAY_SIZE, + &n_hit_result, &state); + EXPECT_EQ(ret, MAAT_SCAN_OK); } class MaatIntervalScan : public testing::Test { protected: static void SetUpTestCase() { - + const char *accept_tags = "{\"tags\":[{\"tag\":\"location\",\"value\":\"北京/朝阳/华严北里/甲22号\"},{\"tag\":\"isp\",\"value\":\"移动\"},{\"tag\":\"location\",\"value\":\"Astana\"}]}"; + char redis_ip[64] = "127.0.0.1"; + int redis_port = 6379; + int redis_db = 0; + + g_logger = log_handle_create("./maat_framework_gtest.log", 0); + assert(g_logger != NULL); + + int ret = write_config_to_redis(redis_ip, redis_port, redis_db, g_logger); + if (ret < 0) { + log_error(g_logger, MODULE_FRAMEWORK_GTEST, + "[%s:%d] write config to redis failed.", __FUNCTION__, __LINE__); + } + + struct maat_options *opts = maat_options_new(); + maat_options_set_redis(opts, redis_ip, redis_port, redis_db); + maat_options_set_logger(opts, g_logger); + maat_options_set_accept_tags(opts, accept_tags); + + _shared_maat_instance = maat_new(opts, table_info_path); + maat_options_free(opts); + if (NULL == _shared_maat_instance) { + log_error(g_logger, MODULE_FRAMEWORK_GTEST, + "[%s:%d] create maat instance in MaatFlagScan failed.", + __FUNCTION__, __LINE__); + } } static void TearDownTestCase() { - + maat_free(_shared_maat_instance); + g_logger = NULL; } + + static struct maat *_shared_maat_instance; }; +struct maat *MaatIntervalScan::_shared_maat_instance; + TEST_F(MaatIntervalScan, Pure) { long long results[ARRAY_SIZE] = {0}; size_t n_hit_result = 0; struct maat_state *state = NULL; const char *table_name = "CONTENT_SIZE"; - int table_id = maat_get_table_id(g_maat_instance, table_name); + struct maat *maat_instance = MaatIntervalScan::_shared_maat_instance; + + int table_id = maat_get_table_id(maat_instance, table_name); unsigned int scan_data1 = 2015; - int ret = maat_scan_integer(g_maat_instance, table_id, 0, scan_data1, results, ARRAY_SIZE, + int ret = maat_scan_integer(maat_instance, table_id, 0, scan_data1, results, ARRAY_SIZE, &n_hit_result, &state); EXPECT_EQ(ret, MAAT_SCAN_HALF_HIT); maat_state_free(&state); unsigned int scan_data2 = 300; - ret = maat_scan_integer(g_maat_instance, table_id, 0, scan_data2, results, ARRAY_SIZE, + ret = maat_scan_integer(maat_instance, table_id, 0, scan_data2, results, ARRAY_SIZE, &n_hit_result, &state); EXPECT_EQ(ret, MAAT_SCAN_HALF_HIT); EXPECT_EQ(n_hit_result, 0); @@ -748,14 +1104,16 @@ TEST_F(MaatIntervalScan, IntervalPlus) { size_t n_hit_result = 0; struct maat_state *state = NULL; const char *table_name = "INTERGER_PLUS"; - int table_id = maat_get_table_id(g_maat_instance, table_name); + struct maat *maat_instance = MaatIntervalScan::_shared_maat_instance; + + int table_id = maat_get_table_id(maat_instance, table_name); const char *district_str = "interval.plus"; - int ret = maat_state_set_scan_district(g_maat_instance, &state, district_str, strlen(district_str)); + int ret = maat_state_set_scan_district(maat_instance, &state, district_str, strlen(district_str)); EXPECT_EQ(ret, 0); unsigned int scan_data1 = 2020; - ret = maat_scan_integer(g_maat_instance, table_id, 0, scan_data1, results, ARRAY_SIZE, + ret = maat_scan_integer(maat_instance, table_id, 0, scan_data1, results, ARRAY_SIZE, &n_hit_result, &state); EXPECT_EQ(ret, MAAT_SCAN_HIT); EXPECT_EQ(n_hit_result, 1); @@ -767,14 +1125,44 @@ class NOTLogic : public testing::Test { protected: static void SetUpTestCase() { - + const char *accept_tags = "{\"tags\":[{\"tag\":\"location\",\"value\":\"北京/朝阳/华严北里/甲22号\"},{\"tag\":\"isp\",\"value\":\"移动\"},{\"tag\":\"location\",\"value\":\"Astana\"}]}"; + char redis_ip[64] = "127.0.0.1"; + int redis_port = 6379; + int redis_db = 0; + + g_logger = log_handle_create("./maat_framework_gtest.log", 0); + assert(g_logger != NULL); + + int ret = write_config_to_redis(redis_ip, redis_port, redis_db, g_logger); + if (ret < 0) { + log_error(g_logger, MODULE_FRAMEWORK_GTEST, + "[%s:%d] write config to redis failed.", __FUNCTION__, __LINE__); + } + + struct maat_options *opts = maat_options_new(); + maat_options_set_redis(opts, redis_ip, redis_port, redis_db); + maat_options_set_logger(opts, g_logger); + maat_options_set_accept_tags(opts, accept_tags); + + _shared_maat_instance = maat_new(opts, table_info_path); + maat_options_free(opts); + if (NULL == _shared_maat_instance) { + log_error(g_logger, MODULE_FRAMEWORK_GTEST, + "[%s:%d] create maat instance in MaatFlagScan failed.", + __FUNCTION__, __LINE__); + } } static void TearDownTestCase() { - + maat_free(_shared_maat_instance); + g_logger = NULL; } + + static struct maat *_shared_maat_instance; }; +struct maat *NOTLogic::_shared_maat_instance; + TEST_F(NOTLogic, OneRegion) { const char *string_should_hit = "This string ONLY contains must-contained-string-of-rule-143."; const char *string_should_not_hit = "This string contains both must-contained-string-of-rule-143 and must-not-contained-string-of-rule-143."; @@ -782,10 +1170,13 @@ TEST_F(NOTLogic, OneRegion) { size_t n_hit_result = 0; struct maat_state *state = NULL; const char *table_name = "HTTP_URL_LITERAL"; + struct maat *maat_instance = NOTLogic::_shared_maat_instance; - int table_id = maat_get_table_id(g_maat_instance, table_name); - maat_state_set_last_scan(g_maat_instance, &state); - int ret = maat_scan_string(g_maat_instance, table_id, 0, string_should_hit, strlen(string_should_hit), + int table_id = maat_get_table_id(maat_instance, table_name); + ASSERT_GT(table_id, 0); + + maat_state_set_last_scan(maat_instance, &state); + int ret = maat_scan_string(maat_instance, table_id, 0, string_should_hit, strlen(string_should_hit), results, ARRAY_SIZE, &n_hit_result, &state); EXPECT_EQ(ret, MAAT_SCAN_HIT); @@ -793,8 +1184,8 @@ TEST_F(NOTLogic, OneRegion) { EXPECT_EQ(results[0], 143); maat_state_free(&state); - maat_state_set_last_scan(g_maat_instance, &state); - ret = maat_scan_string(g_maat_instance, table_id, 0, string_should_not_hit, strlen(string_should_not_hit), + maat_state_set_last_scan(maat_instance, &state); + ret = maat_scan_string(maat_instance, table_id, 0, string_should_not_hit, strlen(string_should_not_hit), results, ARRAY_SIZE, &n_hit_result, &state); EXPECT_EQ(ret, MAAT_SCAN_HALF_HIT); maat_state_free(&state); @@ -808,19 +1199,20 @@ TEST_F(NOTLogic, ScanNotAtLast) { struct maat_state *state = NULL; const char *hit_table_name = "HTTP_URL_LITERAL"; const char *not_hit_table_name = "KEYWORDS_TABLE"; + struct maat *maat_instance = NOTLogic::_shared_maat_instance; - int hit_table_id = maat_get_table_id(g_maat_instance, hit_table_name); + int hit_table_id = maat_get_table_id(maat_instance, hit_table_name); ASSERT_GT(hit_table_id, 0); - int ret = maat_scan_string(g_maat_instance, hit_table_id, 0, string_should_hit, strlen(string_should_hit), + int ret = maat_scan_string(maat_instance, hit_table_id, 0, string_should_hit, strlen(string_should_hit), results, ARRAY_SIZE, &n_hit_result, &state); EXPECT_EQ(ret, MAAT_SCAN_HALF_HIT); - int not_hit_table_id = maat_get_table_id(g_maat_instance, not_hit_table_name); + int not_hit_table_id = maat_get_table_id(maat_instance, not_hit_table_name); ASSERT_GT(not_hit_table_id, 0); - maat_state_set_last_scan(g_maat_instance, &state); - ret = maat_scan_string(g_maat_instance, not_hit_table_id, 0, string_should_not_hit, strlen(string_should_not_hit), + maat_state_set_last_scan(maat_instance, &state); + ret = maat_scan_string(maat_instance, not_hit_table_id, 0, string_should_not_hit, strlen(string_should_not_hit), results, ARRAY_SIZE, &n_hit_result, &state); EXPECT_EQ(ret, MAAT_SCAN_HALF_HIT); maat_state_free(&state); @@ -834,19 +1226,20 @@ TEST_F(NOTLogic, ScanIrrelavantAtLast) { struct maat_state *state = NULL; const char *hit_table_name = "HTTP_URL_LITERAL"; const char *not_hit_table_name = "KEYWORDS_TABLE"; + struct maat *maat_instance = NOTLogic::_shared_maat_instance; - int hit_table_id = maat_get_table_id(g_maat_instance, hit_table_name); + int hit_table_id = maat_get_table_id(maat_instance, hit_table_name); ASSERT_GT(hit_table_id, 0); - int ret = maat_scan_string(g_maat_instance, hit_table_id, 0, string_should_hit, strlen(string_should_hit), + int ret = maat_scan_string(maat_instance, hit_table_id, 0, string_should_hit, strlen(string_should_hit), results, ARRAY_SIZE, &n_hit_result, &state); EXPECT_EQ(ret, MAAT_SCAN_HALF_HIT); - int not_hit_table_id = maat_get_table_id(g_maat_instance, not_hit_table_name); + int not_hit_table_id = maat_get_table_id(maat_instance, not_hit_table_name); ASSERT_GT(hit_table_id, 0); - maat_state_set_last_scan(g_maat_instance, &state); - ret = maat_scan_string(g_maat_instance, not_hit_table_id, 0, string_irrelevant, strlen(string_irrelevant), + maat_state_set_last_scan(maat_instance, &state); + ret = maat_scan_string(maat_instance, not_hit_table_id, 0, string_irrelevant, strlen(string_irrelevant), results, ARRAY_SIZE, &n_hit_result, &state); EXPECT_EQ(ret, MAAT_SCAN_HIT); EXPECT_EQ(n_hit_result, 1); @@ -863,29 +1256,30 @@ TEST_F(NOTLogic, ScanHitAtLastEmptyExpr) { const char *not_hit_table_name = "HTTP_URL_LITERAL"; const char *hit_table_name = "IP_PLUS_CONFIG"; const char *empty_table_name = "EMPTY_KEYWORD"; + struct maat *maat_instance = NOTLogic::_shared_maat_instance; - int not_hit_table_id = maat_get_table_id(g_maat_instance, not_hit_table_name); + int not_hit_table_id = maat_get_table_id(maat_instance, not_hit_table_name); ASSERT_GT(not_hit_table_id, 0); - int ret = maat_scan_string(g_maat_instance, not_hit_table_id, 0, string_should_not_hit, strlen(string_should_not_hit), + int ret = maat_scan_string(maat_instance, not_hit_table_id, 0, string_should_not_hit, strlen(string_should_not_hit), results, ARRAY_SIZE, &n_hit_result, &state); EXPECT_EQ(ret, MAAT_SCAN_OK); uint32_t sip; inet_pton(AF_INET, "10.0.8.186", &sip); - int hit_table_id = maat_get_table_id(g_maat_instance, hit_table_name); + int hit_table_id = maat_get_table_id(maat_instance, hit_table_name); ASSERT_GT(hit_table_id, 0); - ret = maat_scan_ipv4(g_maat_instance, hit_table_id, 0, sip, + ret = maat_scan_ipv4(maat_instance, hit_table_id, 0, sip, results, ARRAY_SIZE, &n_hit_result, &state); EXPECT_EQ(ret, MAAT_SCAN_HALF_HIT); - int empty_table_id = maat_get_table_id(g_maat_instance, empty_table_name); + int empty_table_id = maat_get_table_id(maat_instance, empty_table_name); ASSERT_GT(empty_table_id, 0); - maat_state_set_last_scan(g_maat_instance, &state); - ret = maat_scan_string(g_maat_instance, empty_table_id, 0, string_match_no_region, strlen(string_match_no_region), + maat_state_set_last_scan(maat_instance, &state); + ret = maat_scan_string(maat_instance, empty_table_id, 0, string_match_no_region, strlen(string_match_no_region), results, ARRAY_SIZE, &n_hit_result, &state); EXPECT_EQ(ret, MAAT_SCAN_HIT); EXPECT_EQ(n_hit_result, 1); @@ -901,35 +1295,65 @@ TEST_F(NOTLogic, ScanHitAtLastEmptyInteger) { const char *not_hit_table_name = "HTTP_URL_LITERAL"; const char *hit_table_name = "IP_PLUS_CONFIG"; const char *empty_table_name = "EMPTY_INTERGER"; + struct maat *maat_instance = NOTLogic::_shared_maat_instance; - int not_hit_table_id = maat_get_table_id(g_maat_instance, not_hit_table_name); + int not_hit_table_id = maat_get_table_id(maat_instance, not_hit_table_name); ASSERT_GT(not_hit_table_id, 0); - int ret = maat_scan_string(g_maat_instance, not_hit_table_id, 0, string_should_not_hit, strlen(string_should_not_hit), + int ret = maat_scan_string(maat_instance, not_hit_table_id, 0, string_should_not_hit, strlen(string_should_not_hit), results, ARRAY_SIZE, &n_hit_result, &state); EXPECT_EQ(ret, MAAT_SCAN_OK); uint32_t sip; inet_pton(AF_INET, "10.0.8.187", &sip); - int hit_table_id = maat_get_table_id(g_maat_instance, hit_table_name); + int hit_table_id = maat_get_table_id(maat_instance, hit_table_name); ASSERT_GT(hit_table_id, 0); - ret = maat_scan_ipv4(g_maat_instance, hit_table_id, 0, sip, + ret = maat_scan_ipv4(maat_instance, hit_table_id, 0, sip, results, ARRAY_SIZE, &n_hit_result, &state); EXPECT_EQ(ret, MAAT_SCAN_HALF_HIT); - int empty_table_id = maat_get_table_id(g_maat_instance, empty_table_name); + int empty_table_id = maat_get_table_id(maat_instance, empty_table_name); ASSERT_GT(empty_table_id, 0); - maat_state_set_last_scan(g_maat_instance, &state); - ret = maat_scan_integer(g_maat_instance, empty_table_id, 0, 2015, + maat_state_set_last_scan(maat_instance, &state); + ret = maat_scan_integer(maat_instance, empty_table_id, 0, 2015, results, ARRAY_SIZE, &n_hit_result, &state); EXPECT_EQ(ret, MAAT_SCAN_HIT); EXPECT_EQ(results[0], 187); maat_state_free(&state); } +TEST_F(NOTLogic, ScanNotIP) { + const char *string_should_hit = "This string ONLY contains must-contained-string-of-rule-145."; + long long results[ARRAY_SIZE] = {0}; + size_t n_hit_result = 0; + struct maat_state *state = NULL; + const char *hit_table_name = "HTTP_URL_LITERAL"; + const char *not_hit_table_name = "IP_CONFIG"; + struct maat *maat_instance = NOTLogic::_shared_maat_instance; + + int hit_table_id = maat_get_table_id(maat_instance, hit_table_name); + ASSERT_GT(hit_table_id, 0); + + int ret = maat_scan_string(maat_instance, hit_table_id, 0, string_should_hit, strlen(string_should_hit), + results, ARRAY_SIZE, &n_hit_result, &state); + EXPECT_EQ(ret, MAAT_SCAN_HALF_HIT); + + uint32_t sip; + inet_pton(AF_INET, "10.0.6.205", &sip); + + int not_hit_table_id = maat_get_table_id(maat_instance, not_hit_table_name); + ASSERT_GT(not_hit_table_id, 0); + + maat_state_set_last_scan(maat_instance, &state); + ret = maat_scan_ipv4(maat_instance, not_hit_table_id, 0, sip, results, ARRAY_SIZE, + &n_hit_result, &state); + EXPECT_EQ(ret, MAAT_SCAN_HALF_HIT); + maat_state_free(&state); +} + void maat_read_entry_start_cb(int update_type, void *u_para) { @@ -971,23 +1395,54 @@ class PluginTable : public testing::Test { protected: static void SetUpTestCase() { - + const char *accept_tags = "{\"tags\":[{\"tag\":\"location\",\"value\":\"北京/朝阳/华严北里/甲22号\"},{\"tag\":\"isp\",\"value\":\"移动\"},{\"tag\":\"location\",\"value\":\"Astana\"}]}"; + char redis_ip[64] = "127.0.0.1"; + int redis_port = 6379; + int redis_db = 0; + + g_logger = log_handle_create("./maat_framework_gtest.log", 0); + assert(g_logger != NULL); + + int ret = write_config_to_redis(redis_ip, redis_port, redis_db, g_logger); + if (ret < 0) { + log_error(g_logger, MODULE_FRAMEWORK_GTEST, + "[%s:%d] write config to redis failed.", __FUNCTION__, __LINE__); + } + + struct maat_options *opts = maat_options_new(); + maat_options_set_redis(opts, redis_ip, redis_port, redis_db); + maat_options_set_logger(opts, g_logger); + maat_options_set_accept_tags(opts, accept_tags); + + _shared_maat_instance = maat_new(opts, table_info_path); + maat_options_free(opts); + if (NULL == _shared_maat_instance) { + log_error(g_logger, MODULE_FRAMEWORK_GTEST, + "[%s:%d] create maat instance in MaatFlagScan failed.", + __FUNCTION__, __LINE__); + } } static void TearDownTestCase() { - + maat_free(_shared_maat_instance); + g_logger = NULL; } + + static struct maat *_shared_maat_instance; }; +struct maat *PluginTable::_shared_maat_instance; + TEST_F(PluginTable, Callback) { const char *table_name = "QD_ENTRY_INFO"; - int table_id = maat_get_table_id(g_maat_instance, table_name); + struct maat *maat_instance = PluginTable::_shared_maat_instance; - int ret = maat_table_callback_register(g_maat_instance, table_id, + int table_id = maat_get_table_id(maat_instance, table_name); + int ret = maat_table_callback_register(maat_instance, table_id, maat_read_entry_start_cb, maat_read_entry_cb, maat_read_entry_finish_cb, - g_maat_instance); + maat_instance); EXPECT_EQ(ret, 0); } @@ -995,14 +1450,44 @@ class IPPluginTable : public testing::Test { protected: static void SetUpTestCase() { - + const char *accept_tags = "{\"tags\":[{\"tag\":\"location\",\"value\":\"北京/朝阳/华严北里/甲22号\"},{\"tag\":\"isp\",\"value\":\"移动\"},{\"tag\":\"location\",\"value\":\"Astana\"}]}"; + char redis_ip[64] = "127.0.0.1"; + int redis_port = 6379; + int redis_db = 0; + + g_logger = log_handle_create("./maat_framework_gtest.log", 0); + assert(g_logger != NULL); + + int ret = write_config_to_redis(redis_ip, redis_port, redis_db, g_logger); + if (ret < 0) { + log_error(g_logger, MODULE_FRAMEWORK_GTEST, + "[%s:%d] write config to redis failed.", __FUNCTION__, __LINE__); + } + + struct maat_options *opts = maat_options_new(); + maat_options_set_redis(opts, redis_ip, redis_port, redis_db); + maat_options_set_logger(opts, g_logger); + maat_options_set_accept_tags(opts, accept_tags); + + _shared_maat_instance = maat_new(opts, table_info_path); + maat_options_free(opts); + if (NULL == _shared_maat_instance) { + log_error(g_logger, MODULE_FRAMEWORK_GTEST, + "[%s:%d] create maat instance in MaatFlagScan failed.", + __FUNCTION__, __LINE__); + } } static void TearDownTestCase() { - + maat_free(_shared_maat_instance); + g_logger = NULL; } + + static struct maat *_shared_maat_instance; }; +struct maat *IPPluginTable::_shared_maat_instance; + struct ip_plugin_ud { long long rule_id; char *buffer; @@ -1050,9 +1535,12 @@ void ip_plugin_EX_dup_cb(int table_id, void **to, void **from, long argl, void * TEST_F(IPPluginTable, EX_DATA) { int ip_plugin_ex_data_counter = 0; const char *table_name = "TEST_IP_PLUGIN_WITH_EXDATA"; - int table_id = maat_get_table_id(g_maat_instance, table_name); + struct maat *maat_instance = IPPluginTable::_shared_maat_instance; - int ret = maat_plugin_table_ex_schema_register(g_maat_instance, table_id, + int table_id = maat_get_table_id(maat_instance, table_name); + ASSERT_GT(table_id, 0); + + int ret = maat_plugin_table_ex_schema_register(maat_instance, table_id, ip_plugin_EX_new_cb, ip_plugin_EX_free_cb, ip_plugin_EX_dup_cb, @@ -1066,7 +1554,7 @@ TEST_F(IPPluginTable, EX_DATA) { EXPECT_EQ(ret, 1); struct ip_plugin_ud *results[ARRAY_SIZE]; - ret = maat_ip_plugin_table_get_ex_data(g_maat_instance, table_id, &ipv4, + ret = maat_ip_plugin_table_get_ex_data(maat_instance, table_id, &ipv4, (void **)results, ARRAY_SIZE); EXPECT_EQ(ret, 2); EXPECT_EQ(results[0]->rule_id, 101); @@ -1082,7 +1570,7 @@ TEST_F(IPPluginTable, EX_DATA) { inet_pton(AF_INET6, "2001:db8:1234::5210", &(ipv6.ipv6)); memset(results, 0, sizeof(results)); - ret = maat_ip_plugin_table_get_ex_data(g_maat_instance, table_id, &ipv6, + ret = maat_ip_plugin_table_get_ex_data(maat_instance, table_id, &ipv6, (void**)results, ARRAY_SIZE); EXPECT_EQ(ret, 2); EXPECT_EQ(results[0]->rule_id, 104); @@ -1094,7 +1582,7 @@ TEST_F(IPPluginTable, EX_DATA) { //Reproduce BugReport-Liumengyan-20210515 inet_pton(AF_INET6, "240e:97c:4010:104::17", &(ipv6.ipv6)); - ret = maat_ip_plugin_table_get_ex_data(g_maat_instance, table_id, &ipv6, + ret = maat_ip_plugin_table_get_ex_data(maat_instance, table_id, &ipv6, (void**)results, ARRAY_SIZE); EXPECT_EQ(ret, 0); } @@ -1103,14 +1591,44 @@ class FQDNPluginTable : public testing::Test { protected: static void SetUpTestCase() { - + const char *accept_tags = "{\"tags\":[{\"tag\":\"location\",\"value\":\"北京/朝阳/华严北里/甲22号\"},{\"tag\":\"isp\",\"value\":\"移动\"},{\"tag\":\"location\",\"value\":\"Astana\"}]}"; + char redis_ip[64] = "127.0.0.1"; + int redis_port = 6379; + int redis_db = 0; + + g_logger = log_handle_create("./maat_framework_gtest.log", 0); + assert(g_logger != NULL); + + int ret = write_config_to_redis(redis_ip, redis_port, redis_db, g_logger); + if (ret < 0) { + log_error(g_logger, MODULE_FRAMEWORK_GTEST, + "[%s:%d] write config to redis failed.", __FUNCTION__, __LINE__); + } + + struct maat_options *opts = maat_options_new(); + maat_options_set_redis(opts, redis_ip, redis_port, redis_db); + maat_options_set_logger(opts, g_logger); + maat_options_set_accept_tags(opts, accept_tags); + + _shared_maat_instance = maat_new(opts, table_info_path); + maat_options_free(opts); + if (NULL == _shared_maat_instance) { + log_error(g_logger, MODULE_FRAMEWORK_GTEST, + "[%s:%d] create maat instance in MaatFlagScan failed.", + __FUNCTION__, __LINE__); + } } static void TearDownTestCase() { - + maat_free(_shared_maat_instance); + g_logger = NULL; } + + static struct maat *_shared_maat_instance; }; +struct maat *FQDNPluginTable::_shared_maat_instance; + #define FQDN_PLUGIN_EX_DATA struct fqdn_plugin_ud { @@ -1156,11 +1674,13 @@ void fqdn_plugin_ex_dup_cb(int table_id, void **to, void **from, long argl, void TEST_F(FQDNPluginTable, EX_DATA) { const char *table_name = "TEST_FQDN_PLUGIN_WITH_EXDATA"; - int table_id = maat_get_table_id(g_maat_instance, table_name); + struct maat *maat_instance = FQDNPluginTable::_shared_maat_instance; + + int table_id = maat_get_table_id(maat_instance, table_name); ASSERT_GT(table_id, 0); int fqdn_plugin_ex_data_counter = 0; - int ret = maat_plugin_table_ex_schema_register(g_maat_instance, table_id, + int ret = maat_plugin_table_ex_schema_register(maat_instance, table_id, fqdn_plugin_ex_new_cb, fqdn_plugin_ex_free_cb, fqdn_plugin_ex_dup_cb, @@ -1171,7 +1691,7 @@ TEST_F(FQDNPluginTable, EX_DATA) { int i = 0; struct fqdn_plugin_ud *result[4]; - ret = maat_fqdn_plugin_table_get_ex_data(g_maat_instance, table_id, "www.example1.com", (void**)result, 4); + ret = maat_fqdn_plugin_table_get_ex_data(maat_instance, table_id, "www.example1.com", (void**)result, 4); ASSERT_EQ(ret, 2); EXPECT_EQ(result[0]->rule_id, 201); EXPECT_EQ(result[1]->rule_id, 202); @@ -1180,10 +1700,10 @@ TEST_F(FQDNPluginTable, EX_DATA) { fqdn_plugin_ex_free_cb(0, (void**)&(result[i]), 0, NULL); } - ret = maat_fqdn_plugin_table_get_ex_data(g_maat_instance, table_id, "www.example3.com", (void**)result, 4); + ret = maat_fqdn_plugin_table_get_ex_data(maat_instance, table_id, "www.example3.com", (void**)result, 4); EXPECT_EQ(ret, 0); - ret = maat_fqdn_plugin_table_get_ex_data(g_maat_instance, table_id, "r3---sn-i3belne6.example2.com", (void**)result, 4); + ret = maat_fqdn_plugin_table_get_ex_data(maat_instance, table_id, "r3---sn-i3belne6.example2.com", (void**)result, 4); ASSERT_EQ(ret, 2); EXPECT_TRUE(result[0]->rule_id == 205 || result[0]->rule_id == 204); @@ -1237,22 +1757,53 @@ class BoolPluginTable : public testing::Test { protected: static void SetUpTestCase() { - + const char *accept_tags = "{\"tags\":[{\"tag\":\"location\",\"value\":\"北京/朝阳/华严北里/甲22号\"},{\"tag\":\"isp\",\"value\":\"移动\"},{\"tag\":\"location\",\"value\":\"Astana\"}]}"; + char redis_ip[64] = "127.0.0.1"; + int redis_port = 6379; + int redis_db = 0; + + g_logger = log_handle_create("./maat_framework_gtest.log", 0); + assert(g_logger != NULL); + + int ret = write_config_to_redis(redis_ip, redis_port, redis_db, g_logger); + if (ret < 0) { + log_error(g_logger, MODULE_FRAMEWORK_GTEST, + "[%s:%d] write config to redis failed.", __FUNCTION__, __LINE__); + } + + struct maat_options *opts = maat_options_new(); + maat_options_set_redis(opts, redis_ip, redis_port, redis_db); + maat_options_set_logger(opts, g_logger); + maat_options_set_accept_tags(opts, accept_tags); + + _shared_maat_instance = maat_new(opts, table_info_path); + maat_options_free(opts); + if (NULL == _shared_maat_instance) { + log_error(g_logger, MODULE_FRAMEWORK_GTEST, + "[%s:%d] create maat instance in MaatFlagScan failed.", + __FUNCTION__, __LINE__); + } } static void TearDownTestCase() { - + maat_free(_shared_maat_instance); + g_logger = NULL; } + + static struct maat *_shared_maat_instance; }; +struct maat *BoolPluginTable::_shared_maat_instance; + TEST_F(BoolPluginTable, EX_DATA) { int ex_data_counter = 0, i = 0; const char *table_name = "TEST_BOOL_PLUGIN_WITH_EXDATA"; + struct maat *maat_instance = BoolPluginTable::_shared_maat_instance; - int table_id = maat_get_table_id(g_maat_instance, table_name); + int table_id = maat_get_table_id(maat_instance, table_name); ASSERT_GT(table_id, 0); - int ret = maat_plugin_table_ex_schema_register(g_maat_instance, table_id, + int ret = maat_plugin_table_ex_schema_register(maat_instance, table_id, bool_plugin_ex_new_cb, bool_plugin_ex_free_cb, bool_plugin_ex_dup_cb, @@ -1262,14 +1813,14 @@ TEST_F(BoolPluginTable, EX_DATA) { struct bool_plugin_ud *result[6]; unsigned long long items_1[] = {999}; - ret = maat_bool_plugin_table_get_ex_data(g_maat_instance, table_id, items_1, 1, (void**)result, 6); + ret = maat_bool_plugin_table_get_ex_data(maat_instance, table_id, items_1, 1, (void**)result, 6); EXPECT_EQ(ret, 0); for (i = 0; i < ret; i++) { bool_plugin_ex_free_cb(0, (void**)&(result[i]), 0, NULL); } unsigned long long items_2[] = {1, 2, 1000}; - ret = maat_bool_plugin_table_get_ex_data(g_maat_instance, table_id, items_2, 3, (void**)result, 6); + ret = maat_bool_plugin_table_get_ex_data(maat_instance, table_id, items_2, 3, (void**)result, 6); EXPECT_EQ(ret, 1); EXPECT_EQ(result[0]->id, 301); for (i = 0; i < ret; i++) { @@ -1277,14 +1828,14 @@ TEST_F(BoolPluginTable, EX_DATA) { } unsigned long long items_3[]={101, 102, 1000}; - ret = maat_bool_plugin_table_get_ex_data(g_maat_instance, table_id, items_3, 3, (void**)result, 6); + ret = maat_bool_plugin_table_get_ex_data(maat_instance, table_id, items_3, 3, (void**)result, 6); EXPECT_EQ(ret, 4); for (i = 0; i < ret; i++) { bool_plugin_ex_free_cb(0, (void**)&(result[i]), 0, NULL); } unsigned long long items_4[]={7, 0, 1, 2, 3, 4, 5, 6, 7, 7, 7}; - ret = maat_bool_plugin_table_get_ex_data(g_maat_instance, table_id, items_4, sizeof(items_4)/sizeof(unsigned long long), (void**)result, 6); + ret = maat_bool_plugin_table_get_ex_data(maat_instance, table_id, items_4, sizeof(items_4)/sizeof(unsigned long long), (void**)result, 6); EXPECT_EQ(ret, 1); EXPECT_EQ(result[0]->id, 305); for (i = 0; i < ret; i++) { @@ -1296,23 +1847,55 @@ class VirtualTable : public testing::Test { protected: static void SetUpTestCase() { - + const char *accept_tags = "{\"tags\":[{\"tag\":\"location\",\"value\":\"北京/朝阳/华严北里/甲22号\"},{\"tag\":\"isp\",\"value\":\"移动\"},{\"tag\":\"location\",\"value\":\"Astana\"}]}"; + char redis_ip[64] = "127.0.0.1"; + int redis_port = 6379; + int redis_db = 0; + + g_logger = log_handle_create("./maat_framework_gtest.log", 0); + assert(g_logger != NULL); + + int ret = write_config_to_redis(redis_ip, redis_port, redis_db, g_logger); + if (ret < 0) { + log_error(g_logger, MODULE_FRAMEWORK_GTEST, + "[%s:%d] write config to redis failed.", __FUNCTION__, __LINE__); + } + + struct maat_options *opts = maat_options_new(); + maat_options_set_redis(opts, redis_ip, redis_port, redis_db); + maat_options_set_logger(opts, g_logger); + maat_options_set_accept_tags(opts, accept_tags); + + _shared_maat_instance = maat_new(opts, table_info_path); + maat_options_free(opts); + if (NULL == _shared_maat_instance) { + log_error(g_logger, MODULE_FRAMEWORK_GTEST, + "[%s:%d] create maat instance in MaatFlagScan failed.", + __FUNCTION__, __LINE__); + } } static void TearDownTestCase() { - + maat_free(_shared_maat_instance); + g_logger = NULL; } + + static struct maat *_shared_maat_instance; }; +struct maat *VirtualTable::_shared_maat_instance; + TEST_F(VirtualTable, basic) { long long results[ARRAY_SIZE] = {0}; size_t n_hit_result = 0; struct maat_state *state = NULL; const char *table_name = "HTTP_RESPONSE_KEYWORDS"; - int table_id = maat_get_table_id(g_maat_instance, table_name); + struct maat *maat_instance = VirtualTable::_shared_maat_instance; + + int table_id = maat_get_table_id(maat_instance, table_name); char scan_data[128] = "string1, string2, string3, string4, string5, string6, string7, string8"; - int ret = maat_scan_string(g_maat_instance, table_id, 0, scan_data, strlen(scan_data), + int ret = maat_scan_string(maat_instance, table_id, 0, scan_data, strlen(scan_data), results, ARRAY_SIZE, &n_hit_result, &state); EXPECT_EQ(ret, MAAT_SCAN_HALF_HIT); EXPECT_EQ(n_hit_result, 0); @@ -1323,14 +1906,44 @@ class CompileTable : public testing::Test { protected: static void SetUpTestCase() { - + const char *accept_tags = "{\"tags\":[{\"tag\":\"location\",\"value\":\"北京/朝阳/华严北里/甲22号\"},{\"tag\":\"isp\",\"value\":\"移动\"},{\"tag\":\"location\",\"value\":\"Astana\"}]}"; + char redis_ip[64] = "127.0.0.1"; + int redis_port = 6379; + int redis_db = 0; + + g_logger = log_handle_create("./maat_framework_gtest.log", 0); + assert(g_logger != NULL); + + int ret = write_config_to_redis(redis_ip, redis_port, redis_db, g_logger); + if (ret < 0) { + log_error(g_logger, MODULE_FRAMEWORK_GTEST, + "[%s:%d] write config to redis failed.", __FUNCTION__, __LINE__); + } + + struct maat_options *opts = maat_options_new(); + maat_options_set_redis(opts, redis_ip, redis_port, redis_db); + maat_options_set_logger(opts, g_logger); + maat_options_set_accept_tags(opts, accept_tags); + + _shared_maat_instance = maat_new(opts, table_info_path); + maat_options_free(opts); + if (NULL == _shared_maat_instance) { + log_error(g_logger, MODULE_FRAMEWORK_GTEST, + "[%s:%d] create maat instance in MaatFlagScan failed.", + __FUNCTION__, __LINE__); + } } static void TearDownTestCase() { - + maat_free(_shared_maat_instance); + g_logger = NULL; } + + static struct maat *_shared_maat_instance; }; +struct maat *CompileTable::_shared_maat_instance; + struct rule_ex_param { int ref_cnt; char name[NAME_MAX]; @@ -1389,42 +2002,17 @@ void compile_ex_param_dup(int table_id, void **to, void **from, long argl, void } TEST_F(CompileTable, CompileRuleUpdate) { - //9999 0 0 0 0 0 anything 1 1 0.0 + struct maat *maat_instance = CompileTable::_shared_maat_instance; + const char *compile_table_name = "COMPILE"; - const char *table_line_add = "9999\t0\t0\t0\t0\t0\tanything\t1\t1\t0.0"; - struct maat_cmd_line line_rule; - - memset(&line_rule, 0, sizeof(line_rule)); - line_rule.rule_id = 9999; - line_rule.table_line = table_line_add; - line_rule.table_name = compile_table_name; - int ret = maat_cmd_set_line(g_maat_instance, &line_rule); + long long compile_id = maat_cmd_incrby(maat_instance, "TEST_SEQ", 1); + int ret = compile_table_set_line(maat_instance, compile_table_name, MAAT_OP_ADD, compile_id, NULL, 1, 0); EXPECT_EQ(ret, 1); - sleep(1); - const char *table_line_del = "9999\t0\t0\t0\t0\t0\tanything\t0\t1\t0.0"; - memset(&line_rule, 0, sizeof(line_rule)); - line_rule.rule_id = 9999; - line_rule.table_line = table_line_del; - line_rule.table_name = compile_table_name; - ret = maat_cmd_set_line(g_maat_instance, &line_rule); - EXPECT_EQ(ret, 1); - - sleep(1); - memset(&line_rule, 0, sizeof(line_rule)); - line_rule.rule_id = 9999; - line_rule.table_line = table_line_add; - line_rule.table_name = compile_table_name; - ret = maat_cmd_set_line(g_maat_instance, &line_rule); - EXPECT_EQ(ret, 1); - - sleep(1); - memset(&line_rule, 0, sizeof(line_rule)); - line_rule.rule_id = 9999; - line_rule.table_line = table_line_del; - line_rule.table_name = compile_table_name; - ret = maat_cmd_set_line(g_maat_instance, &line_rule); + sleep(WAIT_FOR_EFFECTIVE_S); + ret = compile_table_set_line(maat_instance, compile_table_name, MAAT_OP_DEL, compile_id, NULL, 1, 0); EXPECT_EQ(ret, 1); + sleep(WAIT_FOR_EFFECTIVE_S); } TEST_F(CompileTable, Conjunction1) { @@ -1434,12 +2022,13 @@ TEST_F(CompileTable, Conjunction1) { const char *scan_data = "i.ytimg.com/vi/OtCNcustg_I/hqdefault.jpg?sqp=-oaymwEZCNACELwBSFXyq4qpAwsIARUAAIhCGAFwAQ==&rs=AOn4CLDOp_5fHMaCA9XZuJdCRv4DNDorMg"; const char *table_name = "HTTP_URL_LITERAL"; const char *compile_tables[2] = {"COMPILE", "COMPILE_ALIAS"}; + struct maat *maat_instance = CompileTable::_shared_maat_instance; - int table_id = maat_get_table_id(g_maat_instance, table_name); + int table_id = maat_get_table_id(maat_instance, table_name); ASSERT_GT(table_id, 0); - maat_state_set_scan_compile_tables(g_maat_instance, &state, compile_tables, 2); - int ret = maat_scan_string(g_maat_instance, table_id, 0, scan_data, strlen(scan_data), + maat_state_set_scan_compile_tables(maat_instance, &state, compile_tables, 2); + int ret = maat_scan_string(maat_instance, table_id, 0, scan_data, strlen(scan_data), results, ARRAY_SIZE, &n_hit_result, &state); EXPECT_EQ(ret, MAAT_SCAN_HIT); EXPECT_EQ(n_hit_result, 2); @@ -1447,7 +2036,7 @@ TEST_F(CompileTable, Conjunction1) { EXPECT_EQ(results[1], 141); struct maat_hit_path hit_path[HIT_PATH_SIZE] = {0}; - int n_read = maat_state_get_hit_paths(g_maat_instance, &state, hit_path, HIT_PATH_SIZE); + int n_read = maat_state_get_hit_paths(maat_instance, &state, hit_path, HIT_PATH_SIZE); EXPECT_EQ(n_read, 2); maat_state_free(&state); @@ -1460,29 +2049,30 @@ TEST_F(CompileTable, Conjunction2) { const char *scan_data = "i.ytimg.com/vi/OtCNcustg_I/hqdefault.jpg?sqp=-oaymwEZCNACELwBSFXyq4qpAwsIARUAAIhCGAFwAQ==&rs=AOn4CLDOp_5fHMaCA9XZuJdCRv4DNDorMg"; const char *table_name = "HTTP_URL_LITERAL"; const char *compile_tables[2] = {"COMPILE", "COMPILE_ALIAS"}; + struct maat *maat_instance = CompileTable::_shared_maat_instance; - int table_id = maat_get_table_id(g_maat_instance, table_name); + int table_id = maat_get_table_id(maat_instance, table_name); ASSERT_GT(table_id, 0); - int ret = maat_scan_string(g_maat_instance, table_id, 0, scan_data, strlen(scan_data), + int ret = maat_scan_string(maat_instance, table_id, 0, scan_data, strlen(scan_data), results, ARRAY_SIZE, &n_hit_result, &state); EXPECT_EQ(ret, MAAT_SCAN_HIT); EXPECT_EQ(n_hit_result, 1); EXPECT_EQ(results[0], 197); struct maat_hit_path hit_path[HIT_PATH_SIZE] = {0}; - int n_read = maat_state_get_hit_paths(g_maat_instance, &state, hit_path, HIT_PATH_SIZE); + int n_read = maat_state_get_hit_paths(maat_instance, &state, hit_path, HIT_PATH_SIZE); EXPECT_EQ(n_read, 2); - maat_state_set_scan_compile_tables(g_maat_instance, &state, compile_tables, 2); - ret = maat_scan_string(g_maat_instance, table_id, 0, scan_data, strlen(scan_data), + maat_state_set_scan_compile_tables(maat_instance, &state, compile_tables, 2); + ret = maat_scan_string(maat_instance, table_id, 0, scan_data, strlen(scan_data), results, ARRAY_SIZE, &n_hit_result, &state); EXPECT_EQ(ret, MAAT_SCAN_HIT); EXPECT_EQ(n_hit_result, 1); EXPECT_EQ(results[0], 141); memset(hit_path, 0, sizeof(hit_path)); - n_read = maat_state_get_hit_paths(g_maat_instance, &state, hit_path, HIT_PATH_SIZE); + n_read = maat_state_get_hit_paths(maat_instance, &state, hit_path, HIT_PATH_SIZE); EXPECT_EQ(n_read, 4); maat_state_free(&state); @@ -1492,14 +2082,44 @@ class Policy : public testing::Test { protected: static void SetUpTestCase() { - + const char *accept_tags = "{\"tags\":[{\"tag\":\"location\",\"value\":\"北京/朝阳/华严北里/甲22号\"},{\"tag\":\"isp\",\"value\":\"移动\"},{\"tag\":\"location\",\"value\":\"Astana\"}]}"; + char redis_ip[64] = "127.0.0.1"; + int redis_port = 6379; + int redis_db = 0; + + g_logger = log_handle_create("./maat_framework_gtest.log", 0); + assert(g_logger != NULL); + + int ret = write_config_to_redis(redis_ip, redis_port, redis_db, g_logger); + if (ret < 0) { + log_error(g_logger, MODULE_FRAMEWORK_GTEST, + "[%s:%d] write config to redis failed.", __FUNCTION__, __LINE__); + } + + struct maat_options *opts = maat_options_new(); + maat_options_set_redis(opts, redis_ip, redis_port, redis_db); + maat_options_set_logger(opts, g_logger); + maat_options_set_accept_tags(opts, accept_tags); + + _shared_maat_instance = maat_new(opts, table_info_path); + maat_options_free(opts); + if (NULL == _shared_maat_instance) { + log_error(g_logger, MODULE_FRAMEWORK_GTEST, + "[%s:%d] create maat instance in MaatFlagScan failed.", + __FUNCTION__, __LINE__); + } } static void TearDownTestCase() { - + maat_free(_shared_maat_instance); + g_logger = NULL; } + + static struct maat *_shared_maat_instance; }; +struct maat *Policy::_shared_maat_instance; + void accept_tags_entry_cb(int table_id, const char *table_line, void *u_para) { int* callback_times = (int *)u_para; @@ -1514,11 +2134,13 @@ void accept_tags_entry_cb(int table_id, const char *table_line, void *u_para) TEST_F(Policy, PluginRuleTags1) { const char *table_name = "TEST_EFFECTIVE_RANGE_TABLE"; - int table_id = maat_get_table_id(g_maat_instance, table_name); + struct maat *maat_instance = Policy::_shared_maat_instance; + + int table_id = maat_get_table_id(maat_instance, table_name); ASSERT_GT(table_id, 0); int callback_times=0; - int ret = maat_table_callback_register(g_maat_instance, table_id, + int ret = maat_table_callback_register(maat_instance, table_id, NULL, accept_tags_entry_cb, NULL, @@ -1535,11 +2157,13 @@ void accept_tags_entry2_cb(int table_id, const char *table_line, void *u_para) TEST_F(Policy, PluginRuleTags2) { const char *table_name = "IR_INTERCEPT_IP"; - int table_id = maat_get_table_id(g_maat_instance, table_name); + struct maat *maat_instance = Policy::_shared_maat_instance; + + int table_id = maat_get_table_id(maat_instance, table_name); ASSERT_GT(table_id, 0); int callback_times = 0; - int ret = maat_table_callback_register(g_maat_instance, table_id, + int ret = maat_table_callback_register(maat_instance, table_id, NULL, accept_tags_entry2_cb, NULL, @@ -1555,16 +2179,17 @@ TEST_F(Policy, CompileRuleTags) { const char *should_hit = "string bbb should hit"; const char *should_not_hit = "string aaa should not hit"; const char *table_name = "HTTP_URL_LITERAL"; - - int table_id = maat_get_table_id(g_maat_instance, table_name); + struct maat *maat_instance = Policy::_shared_maat_instance; + + int table_id = maat_get_table_id(maat_instance, table_name); ASSERT_GT(table_id, 0); - int ret = maat_scan_string(g_maat_instance, table_id, 0, should_not_hit, + int ret = maat_scan_string(maat_instance, table_id, 0, should_not_hit, strlen(should_not_hit), results, ARRAY_SIZE, &n_hit_result, &state); EXPECT_EQ(ret, MAAT_SCAN_HALF_HIT); - ret = maat_scan_string(g_maat_instance, table_id, 0, should_hit, + ret = maat_scan_string(maat_instance, table_id, 0, should_hit, strlen(should_hit), results, ARRAY_SIZE, &n_hit_result, &state); EXPECT_EQ(ret, MAAT_SCAN_HIT); @@ -1580,11 +2205,13 @@ TEST_F(Policy, CompileEXData) { const char *table_name = "HTTP_URL_LITERAL"; const char *compile_table_name = "COMPILE_ALIAS"; const char *expect_name = "I have a name"; - int table_id = maat_get_table_id(g_maat_instance, table_name); - int compile_table_id = maat_get_table_id(g_maat_instance, compile_table_name); + struct maat *maat_instance = Policy::_shared_maat_instance; + + int table_id = maat_get_table_id(maat_instance, table_name); + int compile_table_id = maat_get_table_id(maat_instance, compile_table_name); int ex_data_counter = 0; - int ret = maat_plugin_table_ex_schema_register(g_maat_instance, compile_table_id, + int ret = maat_plugin_table_ex_schema_register(maat_instance, compile_table_id, compile_ex_param_new, compile_ex_param_free, compile_ex_param_dup, @@ -1592,16 +2219,16 @@ TEST_F(Policy, CompileEXData) { ASSERT_TRUE(ret == 0); EXPECT_EQ(ex_data_counter, 1); - ret = maat_state_set_scan_compile_tables(g_maat_instance, &state, &compile_table_name, 1); + ret = maat_state_set_scan_compile_tables(maat_instance, &state, &compile_table_name, 1); EXPECT_EQ(ret, 0); - ret = maat_scan_string(g_maat_instance, table_id, 0, url, strlen(url), + ret = maat_scan_string(maat_instance, table_id, 0, url, strlen(url), results, ARRAY_SIZE, &n_hit_result, &state); EXPECT_EQ(ret, MAAT_SCAN_HIT); EXPECT_EQ(n_hit_result, 1); EXPECT_EQ(results[0], 141); - void *ex_data = maat_plugin_table_get_ex_data(g_maat_instance, compile_table_id, + void *ex_data = maat_plugin_table_get_ex_data(maat_instance, compile_table_id, (char *)&results[0]); ASSERT_TRUE(ex_data!=NULL); struct rule_ex_param *param = (struct rule_ex_param *)ex_data; @@ -1662,14 +2289,44 @@ class TableInfo : public testing::Test { protected: static void SetUpTestCase() { - + const char *accept_tags = "{\"tags\":[{\"tag\":\"location\",\"value\":\"北京/朝阳/华严北里/甲22号\"},{\"tag\":\"isp\",\"value\":\"移动\"},{\"tag\":\"location\",\"value\":\"Astana\"}]}"; + char redis_ip[64] = "127.0.0.1"; + int redis_port = 6379; + int redis_db = 0; + + g_logger = log_handle_create("./maat_framework_gtest.log", 0); + assert(g_logger != NULL); + + int ret = write_config_to_redis(redis_ip, redis_port, redis_db, g_logger); + if (ret < 0) { + log_error(g_logger, MODULE_FRAMEWORK_GTEST, + "[%s:%d] write config to redis failed.", __FUNCTION__, __LINE__); + } + + struct maat_options *opts = maat_options_new(); + maat_options_set_redis(opts, redis_ip, redis_port, redis_db); + maat_options_set_logger(opts, g_logger); + maat_options_set_accept_tags(opts, accept_tags); + + _shared_maat_instance = maat_new(opts, table_info_path); + maat_options_free(opts); + if (NULL == _shared_maat_instance) { + log_error(g_logger, MODULE_FRAMEWORK_GTEST, + "[%s:%d] create maat instance in MaatFlagScan failed.", + __FUNCTION__, __LINE__); + } } static void TearDownTestCase() { - - } + maat_free(_shared_maat_instance); + g_logger = NULL; + } + + static struct maat *_shared_maat_instance; }; +struct maat *TableInfo::_shared_maat_instance; + TEST_F(TableInfo, Conjunction) { long long results[ARRAY_SIZE] = {0}; size_t n_hit_result = 0; @@ -1677,14 +2334,15 @@ TEST_F(TableInfo, Conjunction) { const char *scan_data = "soq is using table conjunction function.http://www.3300av.com/novel/27122.txt"; const char *table_name = "HTTP_URL_LITERAL"; const char *conj_table_name = "HTTP_HOST_LITERAL"; + struct maat *maat_instance = TableInfo::_shared_maat_instance; - int table_id = maat_get_table_id(g_maat_instance, table_name); + int table_id = maat_get_table_id(maat_instance, table_name); ASSERT_GT(table_id, 0); - int conj_table_id = maat_get_table_id(g_maat_instance, conj_table_name); + int conj_table_id = maat_get_table_id(maat_instance, conj_table_name); ASSERT_GT(conj_table_id, 0); - int ret = maat_scan_string(g_maat_instance, conj_table_id, 0, scan_data, strlen(scan_data), + int ret = maat_scan_string(maat_instance, conj_table_id, 0, scan_data, strlen(scan_data), results, ARRAY_SIZE, &n_hit_result, &state); EXPECT_EQ(ret, MAAT_SCAN_HIT); EXPECT_EQ(n_hit_result, 2); @@ -1697,13 +2355,68 @@ class MaatCmdTest : public testing::Test { protected: static void SetUpTestCase() { - + char redis_ip[64] = "127.0.0.1"; + int redis_port = 6379; + int redis_db = 0; + + g_logger = log_handle_create("./maat_framework_gtest.log", 0); + assert(g_logger != NULL); + + struct maat_options *opts = maat_options_new(); + maat_options_set_redis(opts, redis_ip, redis_port, redis_db); + maat_options_set_logger(opts, g_logger); + + _shared_maat_instance = maat_new(opts, table_info_path); + if (NULL == _shared_maat_instance) { + log_error(g_logger, MODULE_FRAMEWORK_GTEST, + "[%s:%d] create maat instance in MaatFlagScan failed.", + __FUNCTION__, __LINE__); + } + maat_cmd_flushDB(_shared_maat_instance); + maat_free(_shared_maat_instance); + + g_logger = log_handle_create("./maat_framework_gtest.log", 0); + assert(g_logger != NULL); + maat_options_set_logger(opts, g_logger); + _shared_maat_instance = maat_new(opts, table_info_path); + maat_options_free(opts); } static void TearDownTestCase() { - - } + maat_free(_shared_maat_instance); + g_logger = NULL; + } + + static struct maat *_shared_maat_instance; }; + +struct maat *MaatCmdTest::_shared_maat_instance; + +int test_add_expr_command(struct maat *maat_instance, const char *expr_table, + long long compile_id, int timeout, const char *keywords) +{ + char huge_serv_def[1024 * 2] = {0}; + + memset(huge_serv_def, 's', sizeof(huge_serv_def) - 1); + huge_serv_def[sizeof(huge_serv_def) - 1] = '\0'; + + int ret = compile_table_set_line(maat_instance, "COMPILE", MAAT_OP_ADD, compile_id, + huge_serv_def, 1, timeout); + EXPECT_EQ(ret, 1); + + long long group_id = maat_cmd_incrby(maat_instance, "SEQUENCE_GROUP", 1); + ret = group2compile_table_set_line(maat_instance, "GROUP2COMPILE", MAAT_OP_ADD, group_id, + compile_id, 0, "null", 1, timeout); + EXPECT_EQ(ret, 1); + + long long item_id = maat_cmd_incrby(maat_instance, "SEQUENCE_REGION", 1); + ret = expr_table_set_line(maat_instance, expr_table, MAAT_OP_ADD, item_id, group_id, keywords, + "null", 1, 0, 0, 0); + EXPECT_EQ(ret, 1); + + return ret; +} + #if 0 TEST_F(MaatCmdTest, HitPath) { const char *g2g_table_name = "GROUP2GROUP"; @@ -1744,84 +2457,177 @@ TEST_F(MaatCmdTest, HitPath) { sprintf(table_line_add, "%lld\t%lld\t%s\t%s\t", item_id, group_id, "URL", "graph_theory"); } #endif -#if 0 + TEST_F(MaatCmdTest, SetIP) { - long long version_before = 0; + long long version_before = 0; + long long results[ARRAY_SIZE] = {0}; + size_t n_hit_result = 0; + struct maat_state *state = NULL; const char *ip_table_name = "IP_CONFIG"; const char *compile_table_name = "COMPILE"; - char compile_table_line_add[128] = {0}; - long long compile_id = 0; - - compile_id = maat_cmd_incrby(g_maat_instance, "TEST_SEQ", 1); - sprintf(compile_table_line_add, "%lld\t0\t0\t0\t0\t0\tanything\t1\t1\t0.0", compile_id); - struct maat_cmd_line line_rule; - line_rule.rule_id = compile_id; - line_rule.table_line = compile_table_line_add; - line_rule.table_name = compile_table_name; - int ret = maat_cmd_set_line(g_maat_instance, &line_rule); + const char *g2c_table_name = "GROUP2COMPILE"; + struct maat *maat_instance = MaatCmdTest::_shared_maat_instance; + + /* compile table add line */ + long long compile_id = maat_cmd_incrby(maat_instance, "TEST_SEQ", 1); + int ret = compile_table_set_line(maat_instance, compile_table_name, MAAT_OP_ADD, compile_id, + "null", 1, 0); EXPECT_EQ(ret, 1); - long long group_id = maat_cmd_incrby(g_maat_instance, "SEQUENCE_GROUP", 1); - memset(&line_rule, 0, sizeof(line_rule)); + /* group2compile table add line */ + long long group_id = maat_cmd_incrby(maat_instance, "SEQUENCE_GROUP", 1); + ret = group2compile_table_set_line(maat_instance, g2c_table_name, MAAT_OP_ADD, group_id, + compile_id, 0, "null", 1, 0); + EXPECT_EQ(ret, 1); + + /* item table add line */ + const char *src_ip = "172.0.0.1"; + const char *src_ip_mask = "255.255.255.255"; + long long item_id = maat_cmd_incrby(maat_instance, "SEQUENCE_REGION", 1); + ret = ip_table_set_line(maat_instance, ip_table_name, MAAT_OP_ADD, item_id, group_id, + IPv4, "mask", src_ip, src_ip_mask, 0); + EXPECT_EQ(ret, 1); + + //version_before = maat_cmd_get_config_version(maat_instance); + //wait_for_cmd_effective(maat_instance, version_before); + sleep(WAIT_FOR_EFFECTIVE_S * 2); + + uint32_t sip; + ret = inet_pton(AF_INET, src_ip, &sip); + EXPECT_EQ(ret, 1); + + int table_id = maat_get_table_id(maat_instance, ip_table_name); + ASSERT_GE(table_id, 0); + + ret = maat_scan_ipv4(maat_instance, table_id, 0, sip, results, ARRAY_SIZE, + &n_hit_result, &state); + EXPECT_EQ(ret, MAAT_SCAN_HIT); + EXPECT_EQ(n_hit_result, 1); + EXPECT_EQ(results[0], compile_id); + + maat_state_free(&state); +} + +TEST_F(MaatCmdTest, SetExpr) { + const char *scan_data = "Hiredis is a minimalistic C client library for the Redis database.\r\n"; + const char *table_name = "HTTP_URL_LITERAL"; + const char *keywords1 = "Hiredis"; + const char *keywords2 = "C Client"; + char escape_buff1[256], escape_buff2[256]; + char keywords[512]; - struct Maat_cmd_region region; - memset(®ion, 0, sizeof(region)); - region.region_type=REGION_IP; - region.table_name=region_table; - region.ip_rule.addr_type=ADDR_TYPE_IPv4; - region.ip_rule.direction=ADDR_DIR_DOUBLE; - region.ip_rule.src_ip="172.0.0.1"; - region.ip_rule.mask_src_ip="255.255.255.255"; - region.ip_rule.src_port=53331; - region.ip_rule.mask_src_port=0;//means any port should hit. + long long results[ARRAY_SIZE] = {0}; + size_t n_hit_result = 0; + struct maat_state *state = NULL; + struct maat *maat_instance = MaatCmdTest::_shared_maat_instance; + + maat_cmd_str_escape(escape_buff1, sizeof(escape_buff1), keywords1); + maat_cmd_str_escape(escape_buff2, sizeof(escape_buff2), keywords2); + snprintf(keywords, sizeof(keywords), "%s&%s", escape_buff1, escape_buff2); + long long compile_id = maat_cmd_incrby(maat_instance, "TEST_SEQ", 2); + + test_add_expr_command(maat_instance, table_name, compile_id - 1, 0, keywords); + test_add_expr_command(maat_instance, table_name, compile_id, 0, keywords); + long long version_before = maat_cmd_get_config_version(maat_instance); + + wait_for_cmd_effective(maat_instance, version_before); + + int table_id = maat_get_table_id(maat_instance, table_name); + ASSERT_GT(table_id, 0); + + memset(results, 0, sizeof(results)); + int ret = maat_scan_string(maat_instance, table_id, 0, scan_data, strlen(scan_data), + results, ARRAY_SIZE, &n_hit_result, &state); + EXPECT_EQ(ret, MAAT_SCAN_HIT); + EXPECT_TRUE(results[0] == compile_id || results[0] == (compile_id - 1)); + maat_state_free(&state); + + + ret = compile_table_set_line(maat_instance, "COMPILE", MAAT_OP_DEL, compile_id-1, + "null", 1, 0); + EXPECT_EQ(ret, 1); + ret = compile_table_set_line(maat_instance, "COMPILE", MAAT_OP_DEL, compile_id, + "null", 1, 0); + EXPECT_EQ(ret, 1); + sleep(WAIT_FOR_EFFECTIVE_S); + + ret = maat_scan_string(maat_instance, table_id, 0, scan_data, strlen(scan_data), + results, ARRAY_SIZE, &n_hit_result, &state); + EXPECT_EQ(ret, MAAT_SCAN_HALF_HIT); + maat_state_free(&state); + + int timeout = 1; + compile_id = maat_cmd_incrby(maat_instance, "TEST_SEQ", 1); + test_add_expr_command(maat_instance, table_name, compile_id, timeout, keywords); + sleep(timeout + 1); + ret = maat_scan_string(maat_instance, table_id, 0, scan_data, strlen(scan_data), + results, ARRAY_SIZE, &n_hit_result, &state); + EXPECT_EQ(ret, MAAT_SCAN_HALF_HIT); + maat_state_free(&state); +} +#if 0 +TEST_F(MaatCmdTest, SetExpr8) { + const char *scan_data8 = "string1, string2, string3, string4, string5, string6, string7, string8"; + const char *scan_data7 = "string1, string2, string3, string4, string5, string6, string7"; - region.ip_rule.dst_ip="172.0.0.2"; - region.ip_rule.mask_dst_ip="255.255.255.255"; - region.ip_rule.dst_port=80; - region.ip_rule.mask_dst_port=65535; - region.ip_rule.protocol=0;//means any protocol should hit. - ret=Maat_command_batch_set_region(batch, MAAT_OP_ADD, ®ion, g2c.group_id); - EXPECT_GE(ret, 0); - - - ret=Maat_read_state(feather,MAAT_STATE_VERSION, &version_before, sizeof(version_before)); - EXPECT_EQ(ret, 0); - Maat_command_batch_commit(batch); - - wait_for_cmd_effective(feather, version_before); + const char *compile_table_name = "COMPILE"; + const char *table_name = "KEYWORDS_TABLE"; + const char *g2c_table_name = "GROUP2COMPILE"; - struct ipaddr ipv4_addr; - struct stream_tuple4_v4 v4_addr; - ipv4_addr.addrtype=ADDR_TYPE_IPV4; - inet_pton(AF_INET,region.ip_rule.src_ip,&(v4_addr.saddr)); - v4_addr.source=htons(region.ip_rule.src_port+1);//Not use the exactly port for testing port mask. - inet_pton(AF_INET,region.ip_rule.dst_ip,&(v4_addr.daddr)); - v4_addr.dest=htons(region.ip_rule.dst_port); - ipv4_addr.v4=&v4_addr; - - int table_id=0; - struct Maat_rule_t result; - memset(&result, 0, sizeof(result)); - scan_status_t mid=NULL; - table_id=Maat_table_register(feather,region_table); - ASSERT_GE(table_id, 0); + const char *keywords8 = "string1&string2&string3&string4&string5&string6&string7&string8"; + const char *keywords7 = "string1&string2&string3&string4&string5&string6&string7"; - ret=Maat_scan_proto_addr(feather,table_id,&ipv4_addr,6,&result,1, &mid,0); - EXPECT_EQ(ret, 1); - EXPECT_EQ(result.config_id, config_id); + long long results[ARRAY_SIZE] = {0}; + size_t n_hit_result = 0; + struct maat_state *state = NULL; + struct maat *maat_instance = MaatCmdTest::_shared_maat_instance; - Maat_clean_status(&mid); - - ret=Maat_read_state(feather,MAAT_STATE_VERSION, &version_before, sizeof(version_before)); + /* compile table add line */ + long long compile_id = maat_cmd_incrby(maat_instance, "TEST_SEQ", 1); + int ret = compile_table_set_line(maat_instance, compile_table_name, MAAT_OP_ADD, compile_id, NULL, 1, 0); + EXPECT_EQ(ret, 1); - Maat_command_raw_set_compile(feather, MAAT_OP_RENEW_TIMEOUT, &compile, "COMPILE", NULL, 1, 0, timeout); - - sleep(timeout-1); - ret=Maat_scan_proto_addr(feather,table_id,&ipv4_addr,6,&result,1, &mid,0); - EXPECT_EQ(ret, 1); + /* group2compile table add line */ + long long group_id = maat_cmd_incrby(maat_instance, "SEQUENCE_GROUP", 1); + ret = group2compile_table_set_line(maat_instance, g2c_table_name, MAAT_OP_ADD, group_id, + compile_id, 0, "null", 1, 0); + EXPECT_EQ(ret, 1); - Maat_clean_status(&mid); + /* expr table add line */ + long long item_id = maat_cmd_incrby(maat_instance, "SEQUENCE_REGION", 1); + ret = expr_table_set_line(maat_instance, table_name, MAAT_OP_ADD, item_id, group_id, + keywords8, NULL, 1, 0, 0, 0); /* EXPR_TYPE_AND MATCH_METHOD_SUB */ + EXPECT_EQ(ret, 1); + + sleep(WAIT_FOR_EFFECTIVE_S); + + int table_id = maat_get_table_id(maat_instance, table_name); + ASSERT_GT(table_id, 0); + + ret = maat_scan_string(maat_instance, table_id, 0, scan_data8, strlen(scan_data8), + results, ARRAY_SIZE, &n_hit_result, &state); + EXPECT_EQ(ret, MAAT_SCAN_HIT); + EXPECT_EQ(n_hit_result, 1); + EXPECT_EQ(results[0], compile_id); + maat_state_free(&state); + + ret = expr_table_set_line(maat_instance, table_name, MAAT_OP_DEL, item_id, group_id, + keywords8, NULL, 1, 0, 0, 0); + EXPECT_EQ(ret, 1); + + ret = expr_table_set_line(maat_instance, table_name, MAAT_OP_ADD, item_id, group_id, + keywords7, NULL, 1, 0, 0, 0); + + sleep(WAIT_FOR_EFFECTIVE_S); + + memset(&results, 0, sizeof(results)); + ret = maat_scan_string(maat_instance, table_id, 0, scan_data7, strlen(scan_data7), + results, ARRAY_SIZE, &n_hit_result, &state); + EXPECT_EQ(ret, MAAT_SCAN_HIT); + EXPECT_EQ(n_hit_result, 1); + EXPECT_EQ(results[0], compile_id); + maat_state_free(&state); } #endif struct user_info { @@ -1859,61 +2665,69 @@ void plugin_ex_dup_cb(int table_id, void **to, void **from, long argl, void *arg *to = u; } -#if 0 TEST_F(MaatCmdTest, PluginEXData) { const char *table_name = "TEST_PLUGIN_EXDATA_TABLE"; const int TEST_CMD_LINE_NUM = 4; - struct maat_cmd_line line_rule; - const char *table_line[TEST_CMD_LINE_NUM] = {"1\t192.168.0.1\tmahuateng\t1\t0", + struct maat *maat_instance = MaatCmdTest::_shared_maat_instance; + const char *table_line_add[TEST_CMD_LINE_NUM] = {"1\t192.168.0.1\tmahuateng\t1\t0", "2\t192.168.0.2\tliuqiangdong\t1\t0", "3\t192.168.0.3\tmayun\t1\t0", "4\t192.168.0.4\tliyanhong\t1\t0"}; - int table_id = maat_get_table_id(g_maat_instance, table_name); + + const char *table_line_del[TEST_CMD_LINE_NUM] = {"1\t192.168.0.1\tmahuateng\t0\t0", + "2\t192.168.0.2\tliuqiangdong\t0\t0", + "3\t192.168.0.3\tmayun\t0\t0", + "4\t192.168.0.4\tliyanhong\t0\t0"}; + + int table_id = maat_get_table_id(maat_instance, table_name); ASSERT_GT(table_id, 0); + long long item_id = -1; + struct maat_cmd_line line_rule; /* 1st line */ memset(&line_rule, 0, sizeof(line_rule)); - line_rule.rule_id = maat_cmd_incrby(g_maat_instance, "TEST_PLUG_SEQ", 1); + line_rule.rule_id = maat_cmd_incrby(maat_instance, "TEST_PLUG_SEQ", 1); line_rule.table_name = table_name; - line_rule.table_line = table_line[0]; + line_rule.table_line = table_line_add[0]; line_rule.expire_after = 0; - int ret = maat_cmd_set_line(g_maat_instance, &line_rule); + int ret = maat_cmd_set_line(maat_instance, &line_rule); EXPECT_GT(ret, 0); /* 2nd line */ memset(&line_rule, 0, sizeof(line_rule)); - line_rule.rule_id = maat_cmd_incrby(g_maat_instance, "TEST_PLUG_SEQ", 1); + line_rule.rule_id = maat_cmd_incrby(maat_instance, "TEST_PLUG_SEQ", 1); line_rule.table_name = table_name; - line_rule.table_line = table_line[1]; + line_rule.table_line = table_line_add[1]; line_rule.expire_after = 0; + item_id = line_rule.rule_id; - ret = maat_cmd_set_line(g_maat_instance, &line_rule); + ret = maat_cmd_set_line(maat_instance, &line_rule); EXPECT_GT(ret, 0); /* 3rd line */ memset(&line_rule, 0, sizeof(line_rule)); - line_rule.rule_id = maat_cmd_incrby(g_maat_instance, "TEST_PLUG_SEQ", 1); + line_rule.rule_id = maat_cmd_incrby(maat_instance, "TEST_PLUG_SEQ", 1); line_rule.table_name = table_name; - line_rule.table_line = table_line[2]; + line_rule.table_line = table_line_add[2]; line_rule.expire_after = 0; - ret = maat_cmd_set_line(g_maat_instance, &line_rule); + ret = maat_cmd_set_line(maat_instance, &line_rule); EXPECT_GT(ret, 0); /* 4th line */ memset(&line_rule, 0, sizeof(line_rule)); - line_rule.rule_id = maat_cmd_incrby(g_maat_instance, "TEST_PLUG_SEQ", 1); + line_rule.rule_id = maat_cmd_incrby(maat_instance, "TEST_PLUG_SEQ", 1); line_rule.table_name = table_name; - line_rule.table_line = table_line[3]; + line_rule.table_line = table_line_add[3]; line_rule.expire_after = 0; - ret = maat_cmd_set_line(g_maat_instance, &line_rule); + ret = maat_cmd_set_line(maat_instance, &line_rule); EXPECT_GT(ret, 0); - sleep(1); + sleep(WAIT_FOR_EFFECTIVE_S); int ex_data_counter = 0; - ret = maat_plugin_table_ex_schema_register(g_maat_instance, table_id, + ret = maat_plugin_table_ex_schema_register(maat_instance, table_id, plugin_ex_new_cb, plugin_ex_free_cb, plugin_ex_dup_cb, @@ -1922,184 +2736,37 @@ TEST_F(MaatCmdTest, PluginEXData) { EXPECT_EQ(ex_data_counter, TEST_CMD_LINE_NUM); struct user_info *uinfo = NULL; - const char *key1 = "192.168.0.4"; - uinfo = (struct user_info *)maat_plugin_table_get_ex_data(g_maat_instance, table_id, - key1, strlen(key1)); + const char *key1 = "192.168.0.2"; + uinfo = (struct user_info *)maat_plugin_table_get_ex_data(maat_instance, + table_id, key1); ASSERT_TRUE(uinfo != NULL); EXPECT_EQ(0, strcmp(uinfo->name, "liuqiangdong")); EXPECT_EQ(uinfo->id, 2); plugin_ex_free_cb(table_id, (void**)&uinfo, 0, NULL); - - ret = maat_cmd_set_line(g_maat_instance, &line_rule + 1); + + memset(&line_rule, 0, sizeof(line_rule)); + line_rule.rule_id = item_id; + line_rule.table_name = table_name; + line_rule.table_line = table_line_del[1]; + line_rule.expire_after = 0; + + ret = maat_cmd_set_line(maat_instance, &line_rule); EXPECT_GT(ret, 0); - sleep(1); + sleep(WAIT_FOR_EFFECTIVE_S); const char *key2 = "192.168.0.2"; - uinfo = (struct user_info *)maat_plugin_table_get_ex_data(g_maat_instance, table_id, - key2, strlen(key2)); + uinfo = (struct user_info *)maat_plugin_table_get_ex_data(maat_instance, + table_id, key2); ASSERT_TRUE(uinfo == NULL); } #endif -int count_line_num_cb(const char *table_name, const char *line, void *u_para) -{ - (*((unsigned int *)u_para))++; - return 0; -} - -int line_idx = 0; -long long absolute_expire_time=0; -int make_serial_rule(const char *table_name, const char *line, void *u_para) -{ - struct serial_rule *s_rule=(struct serial_rule *)u_para; - int rule_id = 0; - char *buff = ALLOC(char, strlen(line) + 1); - - memcpy(buff, line, strlen(line) + 1); - - while (buff[strlen(buff) - 1] == '\n' || buff[strlen(buff) - 1] == '\t') { - buff[strlen(buff) - 1] = '\0'; - } - - int j = 0; - char *str1 = NULL; - char *token = NULL; - char *saveptr1 = NULL; - - for (j = 0,str1 = buff; ; j++, str1 = NULL) { - token = strtok_r(str1, "\t ", &saveptr1); - if (token == NULL) - break; - if (j == 0) { - sscanf(token,"%d", &rule_id); - } - } - - memcpy(buff, line, strlen(line)+1); - while(buff[strlen(buff)-1]=='\n'||buff[strlen(buff)-1]=='\t') { - buff[strlen(buff)-1]='\0'; - } - - maat_cmd_set_serial_rule(s_rule + line_idx, MAAT_OP_ADD, rule_id, table_name, buff, absolute_expire_time); - line_idx++; - FREE(str1); - - return 0; -} - -int write_config_to_redis(char *redis_ip, int redis_port, int redis_db, - struct log_handle *logger) -{ - char json_iris_path[512] = {0}; - - snprintf(json_iris_path, sizeof(json_iris_path), "./%s_iris_tmp", json_filename); - - redisContext *c = maat_cmd_connect_redis(redis_ip, redis_port, redis_db, logger); - if (NULL == c) { - return -1; - } - - redisReply *reply = maat_cmd_wrap_redis_command(c, "flushdb"); - if (NULL == reply) { - return -1; - } - - if (0 == access(json_iris_path, F_OK)) { - system_cmd_rmdir(json_iris_path); - } - - if (access(json_iris_path, F_OK) < 0) { - char tmp_iris_path[128] = {0}; - char *json_buff = NULL; - size_t json_buff_sz = 0; - - int ret = load_file_to_memory(json_filename, (unsigned char **)&json_buff, &json_buff_sz); - if (ret < 0) { - return -1; - } - - ret = json2iris(json_buff, json_filename, c, tmp_iris_path, - sizeof(tmp_iris_path), NULL, NULL, logger); - if (ret < 0) { - return -1; - } - } - - size_t total_line_cnt = 0; - char tmp_iris_full_idx_path[PATH_MAX] = {0}; - snprintf(tmp_iris_full_idx_path, sizeof(tmp_iris_full_idx_path), "%s/index", json_iris_path); - config_monitor_traverse(0, tmp_iris_full_idx_path, NULL, count_line_num_cb, NULL, &total_line_cnt, logger); - - struct serial_rule *s_rule = ALLOC(struct serial_rule, total_line_cnt); - long long server_time = maat_cmd_redis_server_time_s(c); - if (server_time < 0) { - return -1; - } - - absolute_expire_time = server_time + 300; - config_monitor_traverse(0, tmp_iris_full_idx_path, NULL, make_serial_rule, NULL, s_rule, logger); - - int success_cnt = 0; - do { - success_cnt = maat_cmd_write_rule(c, s_rule, total_line_cnt, server_time, logger); - } while (success_cnt < 0); - - EXPECT_EQ(success_cnt, (int)total_line_cnt); - - for (size_t i = 0; i < total_line_cnt; i++) { - maat_cmd_clear_rule_cache(s_rule + i); - } - FREE(s_rule); - redisFree(c); - - return 0; -} - int main(int argc, char ** argv) { int ret=0; ::testing::InitGoogleTest(&argc, argv); - g_logger = log_handle_create("./maat_framework_gtest.log", 0); - if (NULL == g_logger) { - printf("create log handle failed.\n"); - return -1; - } - - const char* accept_tags="{\"tags\":[{\"tag\":\"location\",\"value\":\"北京/朝阳/华严北里/甲22号\"},{\"tag\":\"isp\",\"value\":\"移动\"},{\"tag\":\"location\",\"value\":\"Astana\"}]}"; - char redis_ip[64] = "127.0.0.1"; - int redis_port = 6379; - int redis_db = 0; - - ret = write_config_to_redis(redis_ip, redis_port, redis_db, g_logger); - if (ret < 0) { - log_error(g_logger, MODULE_FRAMEWORK_GTEST, - "[%s:%d] write config to redis failed.", __FUNCTION__, __LINE__); - log_handle_destroy(g_logger); - g_logger = NULL; - return -1; - } - - struct maat_options *opts = maat_options_new(); - maat_options_set_redis(opts, redis_ip, redis_port, redis_db); - maat_options_set_logger(opts, g_logger); - maat_options_set_accept_tags(opts, accept_tags); - - g_maat_instance = maat_new(opts, table_info_path); - maat_options_free(opts); - if (NULL == g_maat_instance) { - log_error(g_logger, MODULE_FRAMEWORK_GTEST, - "[%s:%d] create maat instance in MaatStringScan failed.", - __FUNCTION__, __LINE__); - log_handle_destroy(g_logger); - return -1; - } - ret=RUN_ALL_TESTS(); - maat_free(g_maat_instance); - log_handle_destroy(g_logger); - g_logger = NULL; - return ret; } diff --git a/test/maat_input_mode_gtest.cpp b/test/maat_input_mode_gtest.cpp index 43a76e4..82eb96f 100644 --- a/test/maat_input_mode_gtest.cpp +++ b/test/maat_input_mode_gtest.cpp @@ -25,6 +25,7 @@ TEST(json_mode, maat_scan_string) { ret = json2iris(json_buff, json_filename, NULL, tmp_iris_path, sizeof(tmp_iris_path), NULL, NULL, g_logger); + FREE(json_buff); EXPECT_NE(ret, -1); } @@ -48,11 +49,18 @@ TEST(json_mode, maat_scan_string) { EXPECT_EQ(n_hit_result, 1); EXPECT_EQ(results[0], 182); + long long clause_id = maat_runtime_get_sequence(maat_instance->maat_rt, "test_seq"); + EXPECT_EQ(clause_id, 0); + clause_id = maat_runtime_get_sequence(maat_instance->maat_rt, "test_seq"); + EXPECT_EQ(clause_id, 1); + clause_id = maat_runtime_get_sequence(maat_instance->maat_rt, "test_seq"); + EXPECT_EQ(clause_id, 2); + maat_options_free(opts); maat_state_free(&state); maat_free(maat_instance); } - +#if 0 TEST(iris_mode, maat_scan_string) { char tmp_iris_path[512] = {0}; char json_iris_path[512] = {0}; @@ -67,6 +75,7 @@ TEST(iris_mode, maat_scan_string) { ret = json2iris(json_buff, json_filename, NULL, tmp_iris_path, sizeof(tmp_iris_path), NULL, NULL, g_logger); + FREE(json_buff); EXPECT_NE(ret, -1); } @@ -140,7 +149,8 @@ int make_serial_rule(const char *table_name, const char *line, void *u_para) maat_cmd_set_serial_rule(s_rule + line_idx, MAAT_OP_ADD, rule_id, table_name, buff, absolute_expire_time); line_idx++; - FREE(str1); + + FREE(buff); return 0; } @@ -169,6 +179,7 @@ TEST(redis_mode, maat_scan_string) { ret = json2iris(json_buff, json_filename, c, tmp_iris_path, sizeof(tmp_iris_path), NULL, NULL, g_logger); + FREE(json_buff); EXPECT_NE(ret, -1); } @@ -219,7 +230,7 @@ TEST(redis_mode, maat_scan_string) { maat_state_free(&state); maat_free(maat_instance); } - +#endif int main(int argc, char ** argv) { int ret=0; diff --git a/test/rcu_hash_gtest.cpp b/test/rcu_hash_gtest.cpp index 05c9b1d..31c53b5 100644 --- a/test/rcu_hash_gtest.cpp +++ b/test/rcu_hash_gtest.cpp @@ -3,6 +3,8 @@ #include +struct rcu_hash_table *g_htable = NULL; + struct user_data { int id; char name[32]; @@ -44,10 +46,15 @@ TEST(rcu_hash_add_one_node, single_thread) { EXPECT_EQ(ret, 0); void **data_array = NULL; - ret = rcu_hash_list_updating_data(htable, &data_array); + rcu_hash_commit(htable); + + ret = rcu_hash_list(htable, &data_array); EXPECT_EQ(ret, 1); - rcu_hash_commit(htable); + if (data_array != NULL) { + free(data_array); + data_array = NULL; + } /* find in hash after commit */ res = rcu_hash_find(htable, key, key_len); @@ -63,9 +70,6 @@ TEST(rcu_hash_add_one_node, single_thread) { ret = rcu_hash_garbage_queue_len(htable); EXPECT_EQ(ret, 0); - ret = rcu_hash_list_updating_data(htable, &data_array); - EXPECT_EQ(ret, 0); - rcu_hash_free(htable); } @@ -119,11 +123,22 @@ TEST(rcu_hash_add_multi_node, single_thread) { EXPECT_EQ(ret, 0); void **data_array = NULL; - ret = rcu_hash_list_updating_data(htable, &data_array); - EXPECT_EQ(ret, 4); + ret = rcu_hash_list(htable, &data_array); + EXPECT_EQ(ret, 0); + if (data_array != NULL) { + free(data_array); + data_array = NULL; + } rcu_hash_commit(htable); + ret = rcu_hash_list(htable, &data_array); + EXPECT_EQ(ret, 4); + if (data_array != NULL) { + free(data_array); + data_array = NULL; + } + /* find in hash after commit */ res = rcu_hash_find(htable, key0, key0_len); EXPECT_TRUE(res != NULL); @@ -155,9 +170,6 @@ TEST(rcu_hash_add_multi_node, single_thread) { ret = rcu_hash_garbage_queue_len(htable); EXPECT_EQ(ret, 0); - ret = rcu_hash_list_updating_data(htable, &data_array); - EXPECT_EQ(ret, 0); - rcu_hash_free(htable); } @@ -177,8 +189,12 @@ TEST(rcu_hash_del_one_node, single_thread) { rcu_hash_add(htable, key, key_len, (void *)data); void **data_array = NULL; - int ret = rcu_hash_list_updating_data(htable, &data_array); - EXPECT_EQ(ret, 1); + int ret = rcu_hash_list(htable, &data_array); + EXPECT_EQ(ret, 0); + if (data_array != NULL) { + free(data_array); + data_array = NULL; + } /* find in hash before commit */ void *res = rcu_hash_find(htable, key, key_len); @@ -192,9 +208,13 @@ TEST(rcu_hash_del_one_node, single_thread) { rcu_hash_del(htable, key, key_len); - ret = rcu_hash_list_updating_data(htable, &data_array); + ret = rcu_hash_list(htable, &data_array); EXPECT_EQ(ret, 0); - + if (data_array != NULL) { + free(data_array); + data_array = NULL; + } + rcu_hash_commit(htable); /* find in hash after commit */ @@ -297,10 +317,110 @@ TEST(rcu_hash_del_multi_node, single_thread) { rcu_hash_free(htable); } +TEST(global_rcu_hash_add, single_thread) { + EXPECT_TRUE(g_htable != NULL); + + struct user_data *data1 = ALLOC(struct user_data, 1); + data1->id = 101; + char name1[64] = "www.baidu.com"; + memcpy(data1->name, name1, strlen(name1)); + char key1[64] = "http_url"; + size_t key1_len = strlen(key1); + rcu_hash_add(g_htable, key1, key1_len, (void *)data1); + + int ret = rcu_hash_count(g_htable); + EXPECT_EQ(ret, 0); + + rcu_hash_commit(g_htable); + + ret = rcu_hash_count(g_htable); + EXPECT_EQ(ret, 1); + + void *res = rcu_hash_find(g_htable, key1, key1_len); + EXPECT_TRUE(res != NULL); +} + +TEST(global_rcu_hash_find, single_thread) { + EXPECT_TRUE(g_htable != NULL); + + const char *key = "http_url"; + size_t key_len = strlen(key); + + void *res = rcu_hash_find(g_htable, key, key_len); + EXPECT_TRUE(res != NULL); +} + +TEST(global_rcu_hash_del, single_thread) { + EXPECT_TRUE(g_htable != NULL); + + const char *key = "http_url"; + size_t key_len = strlen(key); + + void *res = rcu_hash_find(g_htable, key, key_len); + EXPECT_TRUE(res != NULL); + + rcu_hash_del(g_htable, key, key_len); + rcu_hash_commit(g_htable); + + res = rcu_hash_find(g_htable, key, key_len); + EXPECT_TRUE(res == NULL); +} + +TEST(rcu_hash_updating_find, single_thread) { + struct rcu_hash_table *htable = rcu_hash_new(data_free); + EXPECT_TRUE(htable != NULL); + + //add data1 + struct user_data *data1 = ALLOC(struct user_data, 1); + data1->id = 101; + char name1[64] = "www.baidu.com"; + memcpy(data1->name, name1, strlen(name1)); + char key[64] = "http_url"; + size_t key_len = strlen(key); + rcu_hash_add(htable, key, key_len, (void *)data1); + + void *res = rcu_hash_find(htable, key, key_len); + EXPECT_TRUE(res == NULL); + + res = rcu_hash_updating_find(htable, key, key_len); + EXPECT_TRUE(res != NULL); + + //del data1 + rcu_hash_del(htable, key, key_len); + + res = rcu_hash_updating_find(htable, key, key_len); + EXPECT_TRUE(res == NULL); + + //add data2 + struct user_data *data2 = ALLOC(struct user_data, 1); + data2->id = 102; + char name2[64] = "www.google.com"; + memcpy(data2->name, name2, strlen(name2)); + + rcu_hash_add(htable, key, key_len, (void *)data2); + res = rcu_hash_updating_find(htable, key, key_len); + EXPECT_TRUE(res != NULL); + + rcu_hash_commit(htable); + + res = rcu_hash_find(htable, key, key_len); + struct user_data *tmp_data = (struct user_data *)res; + EXPECT_EQ(tmp_data->id, 102); + EXPECT_STREQ(tmp_data->name, "www.google.com"); + + rcu_hash_free(htable); +} + int main(int argc, char ** argv) { int ret=0; ::testing::InitGoogleTest(&argc, argv); + + g_htable = rcu_hash_new(data_free); + ret=RUN_ALL_TESTS(); + + rcu_hash_free(g_htable); + return ret; } diff --git a/tools/maat_redis_tool.cpp b/tools/maat_redis_tool.cpp index 7e7dd35..9c3b085 100644 --- a/tools/maat_redis_tool.cpp +++ b/tools/maat_redis_tool.cpp @@ -231,7 +231,8 @@ int make_serial_rule(const char *table_name, const char *line, void *u_para) maat_cmd_set_serial_rule(s_rule + line_idx, MAAT_OP_ADD, rule_id, table_name, buff, absolute_expire_time); line_idx++; - FREE(str1); + + FREE(buff); return 0; } @@ -308,7 +309,8 @@ int main(int argc, char * argv[]) } ret = json2iris(json_buff, json_file, c, tmp_iris_path, sizeof(tmp_iris_path), NULL, NULL, logger); - if (ret < 0) { + FREE(json_buff); + if (ret < 0) { printf("Invalid json format.\n"); }